Add OFFLINE mode for ivy_mooncake

.cargo/config.offline.toml

@@ -0,0 +1,37 @@
+# Offline build config for ivy_mooncake.
+#
+# NOT loaded automatically. Activated only by `make OFFLINE=1 ...`, which passes
+# it via `cargo --config .cargo/config.offline.toml` (see Makefile). Online
+# builds keep using the normal .cargo/config.toml and fetch from crates.io.
+#
+# It redirects every crate source -- crates.io plus the two pinned git deps
+# (rust-postgres, iceberg-rust) -- to the local vendor/ tree produced by:
+#
+#   cargo vendor --locked --manifest-path Cargo.toml \
+#       --sync ivy_moonlink/Cargo.toml vendor/
+#
+# Regenerate vendor/ (and re-copy the block below from cargo's output) whenever
+# Cargo.lock changes. vendor/ is git-ignored and shipped in the offline bundle,
+# not committed.
+#
+# Keep the macOS link flags from .cargo/config.toml in sync here: this file is
+# used INSTEAD of that one during offline builds, not merged with it.
+
+[target.'cfg(target_os="macos")']
+rustflags = ["-Clink-arg=-Wl,-undefined,dynamic_lookup"]
+
+[source.crates-io]
+replace-with = "vendored-sources"
+
+[source."git+https://github.com/Mooncake-Labs/rust-postgres.git?rev=14c8e599f5551a8caa96ff5023685a6f537a1455"]
+git = "https://github.com/Mooncake-Labs/rust-postgres.git"
+rev = "14c8e599f5551a8caa96ff5023685a6f537a1455"
+replace-with = "vendored-sources"
+
+[source."git+https://github.com/apache/iceberg-rust?rev=4a6ea15a7dd73e7c68b0a24441820b13bdd22a92"]
+git = "https://github.com/apache/iceberg-rust"
+rev = "4a6ea15a7dd73e7c68b0a24441820b13bdd22a92"
+replace-with = "vendored-sources"
+
+[source.vendored-sources]
+directory = "vendor/"

.gitignore

@@ -3,6 +3,16 @@
 /tests/pg_regress/regression.out
 /tests/pg_regress/results/
 
+# Vendored crate sources for offline builds: large (~1.3G), regenerated from
+# Cargo.lock via `cargo vendor`, and shipped in the offline bundle -- not git.
+/vendor/
+# Transient backup created while `make OFFLINE=1` swaps in config.offline.toml.
+/.cargo/config.toml.online.bak
+
+# Offline-bundle build inputs and output (regenerated by make offline-bundle).
+/offline-deps/
+/dist/
+
 # Local-only: developer notes, machine-specific helpers, alt-base image
 docs/
 Dockerfile.ivorysql-base

Cargo.toml

@@ -25,8 +25,13 @@ moonlink_rpc.path = "ivy_moonlink/src/moonlink_rpc"
 moonlink_service = { path = "ivy_moonlink/src/moonlink_service", optional = true }
 native-tls = "0.2"
 pgrx = "0.16.1"
-postgres.git = "https://github.com/Mooncake-Labs/rust-postgres.git"
-postgres-native-tls.git = "https://github.com/Mooncake-Labs/rust-postgres.git"
+# Pin to the same rev as ivy_moonlink/Cargo.toml so both workspaces resolve to
+# one git source. Without the rev the source string differs ("...git" vs
+# "...git?rev=..."), which makes `cargo vendor --sync` see two copies of the
+# same package and refuse. Pinning also stops this dep from floating to branch
+# head.
+postgres = { git = "https://github.com/Mooncake-Labs/rust-postgres.git", rev = "14c8e599f5551a8caa96ff5023685a6f537a1455" }
+postgres-native-tls = { git = "https://github.com/Mooncake-Labs/rust-postgres.git", rev = "14c8e599f5551a8caa96ff5023685a6f537a1455" }
 regex = "1"
 serde_json = "1"
 tokio = "1.48"

Makefile

@@ -1,8 +1,30 @@
 PG_VERSION ?= pg18
 export PG_CONFIG := $(shell cargo pgrx info pg-config $(PG_VERSION))
+# Select the pgrx pg feature from PG_VERSION instead of relying on the
+# Cargo default. Keeps the build target and PG_CONFIG in sync.
+PG_FEATURES := --no-default-features --features $(PG_VERSION),bgworker
 MAKEFLAGS += --no-print-directory
 
-.PHONY: help clean ivy_duckdb_mooncake format install package ivy_duckdb run test
+# Offline builds: `make OFFLINE=1 install` (etc). cargo-pgrx spawns its own
+# cargo and exposes no --offline/--config flag, so we gate two ways cargo always
+# honors: the CARGO_NET_OFFLINE env var, and auto-discovery of .cargo/config.toml.
+# $(CARGO) temporarily swaps the vendored-source config in as .cargo/config.toml
+# for the duration of the build and always restores the online one afterward
+# (trap covers a failed/interrupted cargo; only SIGKILL can leave the .bak).
+# Requires a pre-populated vendor/ (see .cargo/config.offline.toml); we fail
+# loudly if it's missing rather than silently falling back to the network.
+ifdef OFFLINE
+export CARGO_NET_OFFLINE := true
+CARGO := test -d vendor || { echo "OFFLINE=1 but vendor/ is missing. Run 'cargo vendor --locked --manifest-path Cargo.toml --sync ivy_moonlink/Cargo.toml vendor/' on a connected machine (or unpack the offline bundle) first." >&2; exit 1; } && \
+	cp .cargo/config.toml .cargo/config.toml.online.bak && \
+	cp .cargo/config.offline.toml .cargo/config.toml && \
+	trap 'mv -f .cargo/config.toml.online.bak .cargo/config.toml' EXIT INT TERM && \
+	cargo
+else
+CARGO := cargo
+endif
+
+.PHONY: help clean ivy_duckdb_mooncake format install package ivy_duckdb run test offline-bundle install-duckdb-extensions
 
 help:
 	@echo "Usage: make <COMMAND> [OPTIONS]"
@@ -14,10 +36,12 @@ help:
 	@echo "  package       Build an installation package for release"
 	@echo "  format        Format the codebase"
 	@echo "  test          Run all tests"
+	@echo "  offline-bundle  Produce a self-contained tarball for air-gapped build (run online)"
 	@echo "  clean         Remove build artifacts"
 	@echo ""
 	@echo "Options:"
-	@echo "  PG_VERSION    pg14, pg15, pg16, pg17, or pg18 (default)"
+	@echo "  PG_VERSION    pg14, pg15, pg16, pg17, or pg18 (default); selects pgrx feature + pg_config"
+	@echo "  OFFLINE=1     Build Rust crates from vendor/ with no network (see .cargo/config.offline.toml)"
 
 clean:
 	@cargo clean
@@ -30,16 +54,40 @@ format:
 	@cargo clippy
 
 install:
-	@cargo pgrx install --release
+	@$(CARGO) pgrx install --release -c $(PG_CONFIG) $(PG_FEATURES)
+ifdef OFFLINE
+	@$(MAKE) install-duckdb-extensions
+endif
+
+# Air-gapped runtime files: place the bundled mooncake.duckdb_extension under
+# the Postgres sharedir so it is tied to the installation, not to any one data
+# directory (survives re-initdb, no PGDATA needed at install time). Requires
+# one line in postgresql.conf pointing DuckDB's extension cache there:
+#   duckdb.extension_directory = '<sharedir>/pg_duckdb/extensions'
+# DuckDB appends <version>/<platform>/ itself, which is exactly the layout the
+# offline bundle ships. With the file pre-placed, the runtime
+# `INSTALL mooncake FROM community` is a zero-network cache hit.
+install-duckdb-extensions:
+	@test -d offline-deps/duckdb-extensions || { echo "offline-deps/duckdb-extensions missing. Run 'make offline-bundle' on a connected machine (or unpack the offline bundle) first." >&2; exit 1; }
+	@dest="$$($(PG_CONFIG) --sharedir)/pg_duckdb/extensions"; \
+	mkdir -p "$$dest"; \
+	cp -r offline-deps/duckdb-extensions/. "$$dest/"; \
+	echo "duckdb extensions installed to $$dest"; \
+	echo "add to postgresql.conf:  duckdb.extension_directory = '$$dest'"
 
 package:
-	@cargo pgrx package
+	@$(CARGO) pgrx package -c $(PG_CONFIG) $(PG_FEATURES)
 
 ivy_duckdb:
 	@$(MAKE) -C ivy_duckdb install -j$(shell nproc)
 
 run: ivy_duckdb
-	@cargo pgrx run
+	@$(CARGO) pgrx run $(PG_VERSION) $(PG_FEATURES)
 
 test:
-	@cargo pgrx regress --resetdb
+	@$(CARGO) pgrx regress $(PG_VERSION) --resetdb $(PG_FEATURES)
+
+# Run on a CONNECTED machine. Gathers submodules + vendored crates + the httpfs
+# extension source into dist/ivy_mooncake-offline-*.tar.gz for air-gapped build.
+offline-bundle:
+	@scripts/make-offline-bundle.sh