diff --git a/.cargo/config.offline.toml b/.cargo/config.offline.toml new file mode 100644 index 0000000..f1e7bb1 --- /dev/null +++ b/.cargo/config.offline.toml @@ -0,0 +1,37 @@ +# Offline build config for ivy_mooncake. +# +# NOT loaded automatically. Activated only by `make OFFLINE=1 ...`, which passes +# it via `cargo --config .cargo/config.offline.toml` (see Makefile). Online +# builds keep using the normal .cargo/config.toml and fetch from crates.io. +# +# It redirects every crate source -- crates.io plus the two pinned git deps +# (rust-postgres, iceberg-rust) -- to the local vendor/ tree produced by: +# +# cargo vendor --locked --manifest-path Cargo.toml \ +# --sync ivy_moonlink/Cargo.toml vendor/ +# +# Regenerate vendor/ (and re-copy the block below from cargo's output) whenever +# Cargo.lock changes. vendor/ is git-ignored and shipped in the offline bundle, +# not committed. +# +# Keep the macOS link flags from .cargo/config.toml in sync here: this file is +# used INSTEAD of that one during offline builds, not merged with it. + +[target.'cfg(target_os="macos")'] +rustflags = ["-Clink-arg=-Wl,-undefined,dynamic_lookup"] + +[source.crates-io] +replace-with = "vendored-sources" + +[source."git+https://github.com/Mooncake-Labs/rust-postgres.git?rev=14c8e599f5551a8caa96ff5023685a6f537a1455"] +git = "https://github.com/Mooncake-Labs/rust-postgres.git" +rev = "14c8e599f5551a8caa96ff5023685a6f537a1455" +replace-with = "vendored-sources" + +[source."git+https://github.com/apache/iceberg-rust?rev=4a6ea15a7dd73e7c68b0a24441820b13bdd22a92"] +git = "https://github.com/apache/iceberg-rust" +rev = "4a6ea15a7dd73e7c68b0a24441820b13bdd22a92" +replace-with = "vendored-sources" + +[source.vendored-sources] +directory = "vendor/" diff --git a/.gitignore b/.gitignore index 79d4fd0..2799620 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,16 @@ /tests/pg_regress/regression.out /tests/pg_regress/results/ +# Vendored crate sources for offline builds: large (~1.3G), regenerated from +# Cargo.lock via `cargo vendor`, and shipped in the offline bundle -- not git. +/vendor/ +# Transient backup created while `make OFFLINE=1` swaps in config.offline.toml. +/.cargo/config.toml.online.bak + +# Offline-bundle build inputs and output (regenerated by make offline-bundle). +/offline-deps/ +/dist/ + # Local-only: developer notes, machine-specific helpers, alt-base image docs/ Dockerfile.ivorysql-base diff --git a/Cargo.lock b/Cargo.lock index b5e02ce..ffab16b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3468,7 +3468,7 @@ dependencies = [ "native-tls", "nix", "parquet", - "postgres-native-tls 0.5.0 (git+https://github.com/Mooncake-Labs/rust-postgres.git?rev=14c8e599f5551a8caa96ff5023685a6f537a1455)", + "postgres-native-tls", "rand 0.9.4", "serde", "serde_json", @@ -3503,14 +3503,14 @@ dependencies = [ "parquet", "pg_escape", "pin-project-lite", - "postgres-native-tls 0.5.0 (git+https://github.com/Mooncake-Labs/rust-postgres.git?rev=14c8e599f5551a8caa96ff5023685a6f537a1455)", + "postgres-native-tls", "postgres-replication", "prost", "serde", "serde_json", "thiserror", "tokio", - "tokio-postgres 0.7.11 (git+https://github.com/Mooncake-Labs/rust-postgres.git?rev=14c8e599f5551a8caa96ff5023685a6f537a1455)", + "tokio-postgres", "tracing", "uuid", ] @@ -3534,7 +3534,7 @@ dependencies = [ "moonlink_error", "more-asserts", "native-tls", - "postgres-native-tls 0.5.0 (git+https://github.com/Mooncake-Labs/rust-postgres.git?rev=14c8e599f5551a8caa96ff5023685a6f537a1455)", + "postgres-native-tls", "serde", "serde_json", "sqlx", @@ -4219,7 +4219,7 @@ dependencies = [ "native-tls", "pgrx", "postgres", - "postgres-native-tls 0.5.0 (git+https://github.com/Mooncake-Labs/rust-postgres.git)", + "postgres-native-tls", "regex", "serde_json", "tokio", @@ -4461,14 +4461,14 @@ checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" [[package]] name = "postgres" version = "0.19.8" -source = "git+https://github.com/Mooncake-Labs/rust-postgres.git#14c8e599f5551a8caa96ff5023685a6f537a1455" +source = "git+https://github.com/Mooncake-Labs/rust-postgres.git?rev=14c8e599f5551a8caa96ff5023685a6f537a1455#14c8e599f5551a8caa96ff5023685a6f537a1455" dependencies = [ "bytes", "fallible-iterator", "futures-util", "log", "tokio", - "tokio-postgres 0.7.11 (git+https://github.com/Mooncake-Labs/rust-postgres.git)", + "tokio-postgres", ] [[package]] @@ -4479,18 +4479,7 @@ dependencies = [ "native-tls", "tokio", "tokio-native-tls", - "tokio-postgres 0.7.11 (git+https://github.com/Mooncake-Labs/rust-postgres.git?rev=14c8e599f5551a8caa96ff5023685a6f537a1455)", -] - -[[package]] -name = "postgres-native-tls" -version = "0.5.0" -source = "git+https://github.com/Mooncake-Labs/rust-postgres.git#14c8e599f5551a8caa96ff5023685a6f537a1455" -dependencies = [ - "native-tls", - "tokio", - "tokio-native-tls", - "tokio-postgres 0.7.11 (git+https://github.com/Mooncake-Labs/rust-postgres.git)", + "tokio-postgres", ] [[package]] @@ -4510,23 +4499,6 @@ dependencies = [ "stringprep", ] -[[package]] -name = "postgres-protocol" -version = "0.6.7" -source = "git+https://github.com/Mooncake-Labs/rust-postgres.git#14c8e599f5551a8caa96ff5023685a6f537a1455" -dependencies = [ - "base64", - "byteorder", - "bytes", - "fallible-iterator", - "hmac 0.12.1", - "md-5", - "memchr", - "rand 0.8.6", - "sha2 0.10.9", - "stringprep", -] - [[package]] name = "postgres-replication" version = "0.6.7" @@ -4537,9 +4509,9 @@ dependencies = [ "futures-util", "memchr", "pin-project-lite", - "postgres-protocol 0.6.7 (git+https://github.com/Mooncake-Labs/rust-postgres.git?rev=14c8e599f5551a8caa96ff5023685a6f537a1455)", - "postgres-types 0.2.7 (git+https://github.com/Mooncake-Labs/rust-postgres.git?rev=14c8e599f5551a8caa96ff5023685a6f537a1455)", - "tokio-postgres 0.7.11 (git+https://github.com/Mooncake-Labs/rust-postgres.git?rev=14c8e599f5551a8caa96ff5023685a6f537a1455)", + "postgres-protocol", + "postgres-types", + "tokio-postgres", ] [[package]] @@ -4549,21 +4521,11 @@ source = "git+https://github.com/Mooncake-Labs/rust-postgres.git?rev=14c8e599f55 dependencies = [ "bytes", "fallible-iterator", - "postgres-protocol 0.6.7 (git+https://github.com/Mooncake-Labs/rust-postgres.git?rev=14c8e599f5551a8caa96ff5023685a6f537a1455)", + "postgres-protocol", "serde", "serde_json", ] -[[package]] -name = "postgres-types" -version = "0.2.7" -source = "git+https://github.com/Mooncake-Labs/rust-postgres.git#14c8e599f5551a8caa96ff5023685a6f537a1455" -dependencies = [ - "bytes", - "fallible-iterator", - "postgres-protocol 0.6.7 (git+https://github.com/Mooncake-Labs/rust-postgres.git)", -] - [[package]] name = "potential_utf" version = "0.1.5" @@ -6283,33 +6245,8 @@ dependencies = [ "percent-encoding", "phf 0.11.3", "pin-project-lite", - "postgres-protocol 0.6.7 (git+https://github.com/Mooncake-Labs/rust-postgres.git?rev=14c8e599f5551a8caa96ff5023685a6f537a1455)", - "postgres-types 0.2.7 (git+https://github.com/Mooncake-Labs/rust-postgres.git?rev=14c8e599f5551a8caa96ff5023685a6f537a1455)", - "rand 0.8.6", - "socket2 0.5.10", - "tokio", - "tokio-util", - "whoami", -] - -[[package]] -name = "tokio-postgres" -version = "0.7.11" -source = "git+https://github.com/Mooncake-Labs/rust-postgres.git#14c8e599f5551a8caa96ff5023685a6f537a1455" -dependencies = [ - "async-trait", - "byteorder", - "bytes", - "fallible-iterator", - "futures-channel", - "futures-util", - "log", - "parking_lot", - "percent-encoding", - "phf 0.11.3", - "pin-project-lite", - "postgres-protocol 0.6.7 (git+https://github.com/Mooncake-Labs/rust-postgres.git)", - "postgres-types 0.2.7 (git+https://github.com/Mooncake-Labs/rust-postgres.git)", + "postgres-protocol", + "postgres-types", "rand 0.8.6", "socket2 0.5.10", "tokio", diff --git a/Cargo.toml b/Cargo.toml index c2230d9..7bb7889 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,8 +25,13 @@ moonlink_rpc.path = "ivy_moonlink/src/moonlink_rpc" moonlink_service = { path = "ivy_moonlink/src/moonlink_service", optional = true } native-tls = "0.2" pgrx = "0.16.1" -postgres.git = "https://github.com/Mooncake-Labs/rust-postgres.git" -postgres-native-tls.git = "https://github.com/Mooncake-Labs/rust-postgres.git" +# Pin to the same rev as ivy_moonlink/Cargo.toml so both workspaces resolve to +# one git source. Without the rev the source string differs ("...git" vs +# "...git?rev=..."), which makes `cargo vendor --sync` see two copies of the +# same package and refuse. Pinning also stops this dep from floating to branch +# head. +postgres = { git = "https://github.com/Mooncake-Labs/rust-postgres.git", rev = "14c8e599f5551a8caa96ff5023685a6f537a1455" } +postgres-native-tls = { git = "https://github.com/Mooncake-Labs/rust-postgres.git", rev = "14c8e599f5551a8caa96ff5023685a6f537a1455" } regex = "1" serde_json = "1" tokio = "1.48" diff --git a/Makefile b/Makefile index d317f20..c77e6bf 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,30 @@ PG_VERSION ?= pg18 export PG_CONFIG := $(shell cargo pgrx info pg-config $(PG_VERSION)) +# Select the pgrx pg feature from PG_VERSION instead of relying on the +# Cargo default. Keeps the build target and PG_CONFIG in sync. +PG_FEATURES := --no-default-features --features $(PG_VERSION),bgworker MAKEFLAGS += --no-print-directory -.PHONY: help clean ivy_duckdb_mooncake format install package ivy_duckdb run test +# Offline builds: `make OFFLINE=1 install` (etc). cargo-pgrx spawns its own +# cargo and exposes no --offline/--config flag, so we gate two ways cargo always +# honors: the CARGO_NET_OFFLINE env var, and auto-discovery of .cargo/config.toml. +# $(CARGO) temporarily swaps the vendored-source config in as .cargo/config.toml +# for the duration of the build and always restores the online one afterward +# (trap covers a failed/interrupted cargo; only SIGKILL can leave the .bak). +# Requires a pre-populated vendor/ (see .cargo/config.offline.toml); we fail +# loudly if it's missing rather than silently falling back to the network. +ifdef OFFLINE +export CARGO_NET_OFFLINE := true +CARGO := test -d vendor || { echo "OFFLINE=1 but vendor/ is missing. Run 'cargo vendor --locked --manifest-path Cargo.toml --sync ivy_moonlink/Cargo.toml vendor/' on a connected machine (or unpack the offline bundle) first." >&2; exit 1; } && \ + cp .cargo/config.toml .cargo/config.toml.online.bak && \ + cp .cargo/config.offline.toml .cargo/config.toml && \ + trap 'mv -f .cargo/config.toml.online.bak .cargo/config.toml' EXIT INT TERM && \ + cargo +else +CARGO := cargo +endif + +.PHONY: help clean ivy_duckdb_mooncake format install package ivy_duckdb run test offline-bundle install-duckdb-extensions help: @echo "Usage: make [OPTIONS]" @@ -14,10 +36,12 @@ help: @echo " package Build an installation package for release" @echo " format Format the codebase" @echo " test Run all tests" + @echo " offline-bundle Produce a self-contained tarball for air-gapped build (run online)" @echo " clean Remove build artifacts" @echo "" @echo "Options:" - @echo " PG_VERSION pg14, pg15, pg16, pg17, or pg18 (default)" + @echo " PG_VERSION pg14, pg15, pg16, pg17, or pg18 (default); selects pgrx feature + pg_config" + @echo " OFFLINE=1 Build Rust crates from vendor/ with no network (see .cargo/config.offline.toml)" clean: @cargo clean @@ -30,16 +54,40 @@ format: @cargo clippy install: - @cargo pgrx install --release + @$(CARGO) pgrx install --release -c $(PG_CONFIG) $(PG_FEATURES) +ifdef OFFLINE + @$(MAKE) install-duckdb-extensions +endif + +# Air-gapped runtime files: place the bundled mooncake.duckdb_extension under +# the Postgres sharedir so it is tied to the installation, not to any one data +# directory (survives re-initdb, no PGDATA needed at install time). Requires +# one line in postgresql.conf pointing DuckDB's extension cache there: +# duckdb.extension_directory = '/pg_duckdb/extensions' +# DuckDB appends // itself, which is exactly the layout the +# offline bundle ships. With the file pre-placed, the runtime +# `INSTALL mooncake FROM community` is a zero-network cache hit. +install-duckdb-extensions: + @test -d offline-deps/duckdb-extensions || { echo "offline-deps/duckdb-extensions missing. Run 'make offline-bundle' on a connected machine (or unpack the offline bundle) first." >&2; exit 1; } + @dest="$$($(PG_CONFIG) --sharedir)/pg_duckdb/extensions"; \ + mkdir -p "$$dest"; \ + cp -r offline-deps/duckdb-extensions/. "$$dest/"; \ + echo "duckdb extensions installed to $$dest"; \ + echo "add to postgresql.conf: duckdb.extension_directory = '$$dest'" package: - @cargo pgrx package + @$(CARGO) pgrx package -c $(PG_CONFIG) $(PG_FEATURES) ivy_duckdb: @$(MAKE) -C ivy_duckdb install -j$(shell nproc) run: ivy_duckdb - @cargo pgrx run + @$(CARGO) pgrx run $(PG_VERSION) $(PG_FEATURES) test: - @cargo pgrx regress --resetdb + @$(CARGO) pgrx regress $(PG_VERSION) --resetdb $(PG_FEATURES) + +# Run on a CONNECTED machine. Gathers submodules + vendored crates + the httpfs +# extension source into dist/ivy_mooncake-offline-*.tar.gz for air-gapped build. +offline-bundle: + @scripts/make-offline-bundle.sh diff --git a/ivy_duckdb b/ivy_duckdb index 7a6d35e..505507c 160000 --- a/ivy_duckdb +++ b/ivy_duckdb @@ -1 +1 @@ -Subproject commit 7a6d35e485bd258821dccee12f77dfde14159f18 +Subproject commit 505507c8de5dd175eaddf76ce9f490d1f1b57113 diff --git a/scripts/make-offline-bundle.sh b/scripts/make-offline-bundle.sh new file mode 100755 index 0000000..0a944b6 --- /dev/null +++ b/scripts/make-offline-bundle.sh @@ -0,0 +1,210 @@ +#!/usr/bin/env bash +# +# make-offline-bundle.sh -- produce a self-contained tarball for air-gapped +# ("完全内网" / no public internet) build + install of ivy_mooncake. +# +# RUN THIS ON A CONNECTED MACHINE. It gathers every build-time input that the +# normal build would otherwise fetch from the internet, so the air-gapped side +# can run `make OFFLINE=1 install` with zero external network: +# +# layer 1 (this script) source tree + recursive submodules, .git stripped and +# replaced with the marker stubs the Makefiles check +# layer 2 (cargo) vendor/ -- all Rust crates from Cargo.lock +# layer 3 (FetchContent) offline-deps/httpfs-src -- the one DuckDB extension +# built.-time git-cloned; pinned commit, recursive +# layer 5 (runtime) offline-deps/duckdb-extensions/... -- the signed +# mooncake.duckdb_extension from the community repo. +# At runtime pg_duckdb executes `INSTALL mooncake FROM +# community`; when the file already sits in the +# extension_directory that INSTALL is a no-network +# no-op (duckdb extension_install.cpp: file exists -> +# NOP), so pre-placing it makes first use fully +# offline. Air-gapped placement: see BUNDLE-INFO. +# +# NOT bundled (provision separately on the air-gapped host; see BUNDLE-INFO): +# - toolchain: rustc + cargo-pgrx (pinned in rust-toolchain.toml / Dockerfile) +# - system libs: OpenSSL + curl dev packages (httpfs links system OpenSSL) +# - IvorySQL itself (pg_config) +# +# Usage: +# scripts/make-offline-bundle.sh [OUTPUT_DIR] +# Env toggles (for iterating; leave unset for a real bundle): +# BUNDLE_SKIP_VENDOR=1 reuse an existing vendor/ instead of re-running it +# BUNDLE_SKIP_ARCHIVE=1 stage everything but don't create the final tarball +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO_ROOT" + +OUT_DIR="${1:-$REPO_ROOT/dist}" +HTTPFS_DEST="offline-deps/httpfs-src" +CMAKE_EXT="ivy_duckdb/third_party/pg_duckdb_extensions.cmake" + +log() { printf '\033[1;34m[bundle]\033[0m %s\n' "$*" >&2; } +die() { printf '\033[1;31m[bundle] ERROR:\033[0m %s\n' "$*" >&2; exit 1; } + +# --- preflight: this is the ONLINE step; fail loud if prerequisites missing --- +command -v git >/dev/null || die "git not found" +command -v cargo >/dev/null || die "cargo not found" +command -v tar >/dev/null || die "tar not found" +[ -f Cargo.toml ] && [ -f "$CMAKE_EXT" ] || die "run from the ivy_mooncake repo root" +if ! git ls-remote https://github.com/duckdb/duckdb-httpfs HEAD >/dev/null 2>&1; then + die "no network to github. This script must run on a CONNECTED machine." +fi + +# --- layer 1: complete the source tree (recursive submodules) --------------- +log "initializing submodules (recursive)..." +git submodule update --init --recursive + +# --- layer 2: vendor Rust crates -------------------------------------------- +if [ "${BUNDLE_SKIP_VENDOR:-}" = 1 ] && [ -d vendor ]; then + log "reusing existing vendor/ (BUNDLE_SKIP_VENDOR=1)" +else + log "vendoring Rust crates (both workspaces)..." + cargo vendor --locked --manifest-path Cargo.toml \ + --sync ivy_moonlink/Cargo.toml vendor/ >/dev/null +fi +[ -d vendor ] || die "vendor/ was not produced" + +# --- layer 3: fetch the httpfs extension source at its pinned commit --------- +# Parse url + commit from the extension config so they never drift from what +# the build actually loads. +HTTPFS_URL="$(awk '/duckdb_extension_load\(httpfs/{f=1} f&&/GIT_URL/{print $2; f=0}' "$CMAKE_EXT")" +HTTPFS_TAG="$(awk '/duckdb_extension_load\(httpfs/{f=1} f&&/GIT_TAG/{print $2; f=0}' "$CMAKE_EXT")" +[ -n "$HTTPFS_URL" ] && [ -n "$HTTPFS_TAG" ] || die "could not parse httpfs GIT_URL/GIT_TAG from $CMAKE_EXT" +log "fetching httpfs source $HTTPFS_TAG from $HTTPFS_URL ..." +rm -rf "$HTTPFS_DEST" +mkdir -p "$(dirname "$HTTPFS_DEST")" +git clone --quiet "$HTTPFS_URL" "$HTTPFS_DEST" +git -C "$HTTPFS_DEST" fetch --quiet origin "$HTTPFS_TAG" 2>/dev/null || git -C "$HTTPFS_DEST" fetch --quiet origin +git -C "$HTTPFS_DEST" checkout --quiet --detach "$HTTPFS_TAG" +# Deliberately NOT recursing httpfs's own submodules: they are just a full +# duckdb checkout + extension-ci-tools, used only to build httpfs standalone. +# Built in-tree (via EXTENSION_CONFIGS) it uses the parent duckdb, so pulling +# them would duplicate a whole duckdb tree into the bundle for nothing. +# It's pure source now; drop its git metadata (FETCHCONTENT_SOURCE_DIR does not +# need .git) so the bundle carries no dangling gitlinks. +find "$HTTPFS_DEST" -name .git -prune -exec rm -rf {} + + +# --- layer 5: the mooncake DuckDB extension (runtime, community-signed) ------ +# Version must match the DuckDB linked into pg_duckdb (ivy_duckdb/Makefile), +# platform must match the target host. The community file is signed and its +# origin is "community", so no allow_unsigned/repository changes are needed on +# the air-gapped host -- pre-placing the file turns the runtime INSTALL into a +# cache-hit no-op. +DUCKDB_VER="$(sed -n 's/^DUCKDB_VERSION *= *//p' ivy_duckdb/Makefile | head -1)" +[ -n "$DUCKDB_VER" ] || die "could not parse DUCKDB_VERSION from ivy_duckdb/Makefile" +# Which target platforms to bundle the (binary) extension for. Everything else +# in the bundle is source and builds natively on any architecture; this is the +# only per-platform artifact. Default covers the packaging machine; override +# for cross-platform bundles, e.g.: +# BUNDLE_PLATFORMS="linux_amd64 linux_arm64" make offline-bundle +# install-duckdb-extensions copies the whole tree, and DuckDB picks its own +# / subdirectory at runtime, so shipping several is harmless. +if [ -z "${BUNDLE_PLATFORMS:-}" ]; then + case "$(uname -m)" in + x86_64) BUNDLE_PLATFORMS=linux_amd64 ;; + aarch64) BUNDLE_PLATFORMS=linux_arm64 ;; + *) die "unsupported platform $(uname -m); set BUNDLE_PLATFORMS explicitly" ;; + esac +fi +for platform in $BUNDLE_PLATFORMS; do + EXT_DEST="offline-deps/duckdb-extensions/$DUCKDB_VER/$platform" + EXT_URL="http://community-extensions.duckdb.org/$DUCKDB_VER/$platform/mooncake.duckdb_extension.gz" + log "fetching mooncake.duckdb_extension ($DUCKDB_VER/$platform) from community repo..." + mkdir -p "$EXT_DEST" + if command -v curl >/dev/null; then + curl -fsSL "$EXT_URL" -o "$EXT_DEST/mooncake.duckdb_extension.gz" + else + wget -q "$EXT_URL" -O "$EXT_DEST/mooncake.duckdb_extension.gz" + fi + gunzip -f "$EXT_DEST/mooncake.duckdb_extension.gz" + [ -s "$EXT_DEST/mooncake.duckdb_extension" ] || die "mooncake.duckdb_extension download/unpack failed for $platform" +done +DUCKDB_PLATFORM="$BUNDLE_PLATFORMS" + +# --- overlay: .git-marker stubs so the air-gapped build needs no git --------- +# The ivy_duckdb Makefile depends on .git/modules/third_party/duckdb/HEAD whose +# recipe is `git submodule update`. Touching the marker satisfies the rule so +# the recipe never runs. Mirrors the Dockerfile's stub logic. +OVERLAY="$(mktemp -d)" +trap 'rm -rf "$OVERLAY"' EXIT +for sm in ivy_duckdb ivy_moonlink ivy_duckdb_mooncake; do + mkdir -p "$OVERLAY/$sm/.git/modules/third_party/duckdb" + : > "$OVERLAY/$sm/.git/modules/third_party/duckdb/HEAD" +done +mkdir -p "$OVERLAY/ivy_duckdb/third_party/duckdb/.git" +: > "$OVERLAY/ivy_duckdb/third_party/duckdb/.git/HEAD" + +# --- manifest --------------------------------------------------------------- +DESCRIBE="$(git describe --always --dirty 2>/dev/null || echo unknown)" +cat > "$OVERLAY/BUNDLE-INFO.txt" </dev/null || echo '(see rust-toolchain.toml)') +cargo-pgrx : 0.16.1 (must be pre-installed on the air-gapped host) + +Bundled build-time inputs (no network needed): + - full source tree + recursive submodules (.git stripped; marker stubs added) + - vendor/ : Rust crates for both workspaces (from Cargo.lock) + - offline-deps/httpfs-src : DuckDB httpfs extension source (pinned commit) + - offline-deps/duckdb-extensions/$DUCKDB_VER/{$DUCKDB_PLATFORM}/mooncake.duckdb_extension + : signed community build; pre-place it so the runtime + INSTALL becomes a zero-network cache hit + +Provision separately on the air-gapped host (NOT in this tarball): + - rustc $(sed -n 's/^channel *= *"\(.*\)"/\1/p' rust-toolchain.toml 2>/dev/null) and cargo-pgrx 0.16.1 + - system OpenSSL + curl dev packages (httpfs links system OpenSSL; do NOT enable vcpkg) + - IvorySQL (a working pg_config) + +Build on the air-gapped host: + tar xzf ivy_mooncake-offline-*.tar.gz && cd ivy_mooncake + make OFFLINE=1 ivy_duckdb PG_VERSION=pg14 # C++/DuckDB side (layer 3) + make OFFLINE=1 install PG_VERSION=pg14 # Rust extension (layer 2) + + # auto-installs the mooncake + # DuckDB extension into sharedir + +Runtime setup on the air-gapped host -- postgresql.conf, then restart: + shared_preload_libraries = 'pg_duckdb,pg_mooncake' + wal_level = logical + duckdb.allow_community_extensions = true + duckdb.extension_directory = '/pg_duckdb/extensions' + # exact path is printed by `make OFFLINE=1 install`; tying the + # extension cache to the installation instead of a data directory + # means re-initdb needs no re-copying. The pre-placed file turns the + # runtime INSTALL into a zero-network cache hit. +EOF + +if [ "${BUNDLE_SKIP_ARCHIVE:-}" = 1 ]; then + log "BUNDLE_SKIP_ARCHIVE=1: staged overlay at $OVERLAY (not archived)" + log "httpfs source at $HTTPFS_DEST, vendor/ ready" + trap - EXIT + echo "$OVERLAY" + exit 0 +fi + +# --- archive: working tree (minus VCS + build artifacts) + overlay ---------- +# Two passes into one uncompressed tar, then gzip: +# pass 1 the working tree, dropping every .git (real, huge) and build output +# pass 2 the overlay, whose members live UNDER .git/ (the marker stubs) -- +# so it must NOT carry the .git excludes, hence a separate append. +# A single invocation can't do this: --exclude is global and would strip the +# stubs too. (tar's `*` matches `/`, so `*/.git` catches .git at any depth.) +mkdir -p "$OUT_DIR" +TAR="$OUT_DIR/ivy_mooncake-offline-${DESCRIBE}.tar" +XFORM='s,^\./,ivy_mooncake/,' +log "archiving working tree ..." +tar cf "$TAR" \ + --exclude='*/.git' --exclude='./.git' \ + --exclude='./dist' \ + --exclude='./target' \ + --exclude='./ivy_duckdb/third_party/duckdb/build' \ + --transform="$XFORM" \ + -C "$REPO_ROOT" . +log "appending .git marker stubs ..." +tar rf "$TAR" --transform="$XFORM" -C "$OVERLAY" . +log "compressing ..." +gzip -f "$TAR" +log "done: ${TAR}.gz ($(du -h "${TAR}.gz" | cut -f1))" diff --git a/src/functions.rs b/src/functions.rs index a6fdf30..63625e8 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -31,15 +31,25 @@ fn create_table(dst: &str, src: &str, src_uri: Option<&str>, table_config: Optio let src_uri = src_uri.unwrap_or(&dst_uri).to_owned(); create_mooncake_table(&dst, &dst_uri, &src, &src_uri); let table_config = table_config.unwrap_or("{}").to_owned(); - block_on(moonlink_rpc::create_table( + // Bind the result before inspecting it so the moonlink stream guard is + // released first: panicking while the guard is alive poisons the mutex + // and breaks every later mooncake call in this backend. + let result = block_on(moonlink_rpc::create_table( &mut *get_stream(), DATABASE.clone(), - dst, + dst.clone(), src, src_uri, table_config, - )) - .expect("create_table failed"); + )); + if let Err(err) = result { + // Moonlink rejected the table (e.g. a column type it cannot store in + // Iceberg). Drop the mirror table just created over the loopback so a + // failed call doesn't leave behind a mooncake table that moonlink has + // no knowledge of. + drop_mooncake_table(&dst, &dst_uri); + panic!("create_table failed: {err:?}"); + } } #[pg_extern(sql = " @@ -195,11 +205,28 @@ fn uri_encode(input: &str) -> String { result } +/// IvorySQL: the loopback URI targets the PG-mode port, so the new session +/// defaults to `ivorysql.compatible_mode = pg` even on an Oracle-mode +/// cluster. Oracle types emitted by format_type (number, varchar2, clob) +/// then fail to resolve in the CREATE TABLE. Propagate the calling +/// session's mode so DDL built from Oracle-typed tables works. +fn propagate_compatible_mode(client: &mut Client) { + let mode = Spi::get_one::("SELECT current_setting('ivorysql.compatible_mode', true)"); + if let Ok(Some(mode)) = mode { + if mode == "oracle" { + client + .simple_query("SET ivorysql.compatible_mode TO oracle") + .expect("error setting ivorysql.compatible_mode"); + } + } +} + fn create_mooncake_table(dst: &str, dst_uri: &str, src: &str, src_uri: &str) { let tls_connector = TlsConnector::new().expect("error creating tls connector"); let make_tls_connector = MakeTlsConnector::new(tls_connector); let mut client = Client::connect(src_uri, make_tls_connector.clone()) .unwrap_or_else(|_| panic!("error connecting to server: {src_uri}")); + propagate_compatible_mode(&mut client); let get_columns_query = format!( "SELECT string_agg( @@ -223,6 +250,7 @@ fn create_mooncake_table(dst: &str, dst_uri: &str, src: &str, src_uri: &str) { if dst_uri != src_uri { client = Client::connect(dst_uri, make_tls_connector) .unwrap_or_else(|_| panic!("error connecting to server: {dst_uri}")); + propagate_compatible_mode(&mut client); } let create_table_query = format!("CREATE TABLE {dst} ({columns}) USING mooncake"); @@ -231,6 +259,20 @@ fn create_mooncake_table(dst: &str, dst_uri: &str, src: &str, src_uri: &str) { .unwrap_or_else(|_| panic!("error creating table: {dst}")); } +/// Best-effort cleanup of a mirror table whose moonlink registration failed. +/// Errors are swallowed: this runs on an error path already headed for panic, +/// and the original moonlink error is the one worth reporting. +fn drop_mooncake_table(dst: &str, dst_uri: &str) { + let Ok(tls_connector) = TlsConnector::new() else { + return; + }; + let make_tls_connector = MakeTlsConnector::new(tls_connector); + let Ok(mut client) = Client::connect(dst_uri, make_tls_connector) else { + return; + }; + let _ = client.simple_query(&format!("DROP TABLE IF EXISTS {dst}")); +} + /// Diagnostic helper for moonlink bgworker liveness. /// /// Returns a JSON object combining the GUC value with filesystem-level