From 21b2ee52ccaca37f536c881324805cdfe4ad113b Mon Sep 17 00:00:00 2001 From: jnasbyupgrade Date: Thu, 16 Jul 2026 12:37:26 -0500 Subject: [PATCH 1/2] Squashed 'pgxntool/' changes from bed3604..eb84bc6 eb84bc6 Stamp 2.1.0 9fbe6f4 Fix results ordering, control file whitespace, ENABLE_* override, debug levels (#31) 88bb4f2 Add Claude Code GitHub Actions workflows (#41) af5bbbb ci: pass repo owners to run-tests.yml for fork-account matching (#40) c7928af Fix repo-root guard to work inside a git worktree (#39) b062fca ci: point reusable test workflow at @master b6cdbfd Add CI workflows and multi-session PR guard (#33) 1ba0987 Stamp 2.0.3 1931cbe Fix pgxntool-sync remote and make it runnable without make (#37) 8176304 Stamp 2.0.2 3e142ab Fix parse_control_file: remove comments before stripping quotes (#27) cacc301 Stamp 2.0.1 bf1db6b Fix bash 3.2 / Linux compatibility issues (#26) 62d0fcb Fix broken ifeq for --load-language=plpgsql on PG < 13 (#24) 121f0b3 Stamp 2.0.0 ad3ca7e Remove .source support; add test/install, test/build, and verify-results (#18) c010cf8 Fix bash 3.2 compatibility (#23) abeb9d3 Remove .source file support from pg_regress integration (#22) 08c1879 Stamp 1.1.2 6e0dad2 Fix double --dbname bug that defeated unique test database names 639756c Stamp 1.1.1 6ba3176 Fix pg_tle exception handler and empty upgrade files (#15) 3b8cb2a Stamp 1.1.0 550a901 Remove commit.md (maintained in pgxntool-test) d73ca93 Add unique test database names to prevent conflicts (#13) 9b344be Add update-setup-files.sh for 3-way merging after pgxntool-sync (#12) ab7f6e2 Stamp 1.0.0 3a571ba Add pg_tle support and modernize test infrastructure (#11) b96ea6d Add support for Claude code; build and doc improvements (#9) e9c24de Fix pg_regress on versions > 12 (#5) REVERT: bed3604 Fix pg_regress on versions > 12 (#5) (#6) git-subtree-dir: pgxntool git-subtree-split: eb84bc6e87e21f2ced11ff1b8ddb4028b7c67c8f --- .claude/CLAUDE.md | 28 + .claude/development.md | 47 + .gitattributes | 4 + .github/workflows/CLAUDE.md | 61 ++ .github/workflows/ci.yml | 294 ++++++ .github/workflows/claude-code-review.yml | 102 ++ .github/workflows/claude.yml | 46 + .github/workflows/protect-label.yml | 146 +++ .gitignore | 1 + CLAUDE.md | 284 ++++++ HISTORY.asc | 117 ++- LICENSE | 2 +- README.asc | 377 ++++++- README.html | 1172 +++++++++++++++++----- _.gitignore | 18 +- base.mk | 404 +++++++- build_meta.sh | 32 +- control.mk.sh | 90 ++ lib.sh | 110 ++ meta.mk.sh | 96 +- pgtle.sh | 858 ++++++++++++++++ pgtle_versions.md | 47 + pgxntool-sync.sh | 50 + run-test-build.sh | 47 + setup.sh | 49 +- update-setup-files.sh | 181 ++++ verify-results-pgtap.sh | 47 + 27 files changed, 4353 insertions(+), 357 deletions(-) create mode 100644 .claude/CLAUDE.md create mode 100644 .claude/development.md create mode 100644 .github/workflows/CLAUDE.md create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/claude-code-review.yml create mode 100644 .github/workflows/claude.yml create mode 100644 .github/workflows/protect-label.yml create mode 100644 CLAUDE.md create mode 100755 control.mk.sh create mode 100644 lib.sh create mode 100755 pgtle.sh create mode 100644 pgtle_versions.md create mode 100755 pgxntool-sync.sh create mode 100755 run-test-build.sh create mode 100755 update-setup-files.sh create mode 100755 verify-results-pgtap.sh diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md new file mode 100644 index 0000000..e2e74fe --- /dev/null +++ b/.claude/CLAUDE.md @@ -0,0 +1,28 @@ +# Claude Development Notes + +This file contains guidance for Claude Code when working in this repository. +It is excluded from distributions via `.gitattributes export-ignore`. + +## CI Monitoring After Every Push + +**REQUIRED**: After every `git push`, immediately start a background task to +monitor the CI run for that push. If you pushed to both pgxntool and +pgxntool-test, start a background task for each repo — do not monitor them +sequentially. + +Use `gh run watch` or poll with `gh run list` / `gh pr checks` in the +background task. Report failures to the user as soon as they are detected; +do not wait for all jobs to finish before reporting. + +## Multiple Concurrent Sessions + +It is common to have multiple Claude Code sessions open simultaneously across +pgxntool and pgxntool-test. To avoid cross-session interference: + +**If you are asked to do something on an existing PR that you did not open or +are not already working on in this session, immediately ask for confirmation +before proceeding.** For example: "I see PR #32 exists. Were you asking me to +work on that, or did you mean to send this to a different session?" + +This applies to: editing PR branches, pushing to them, closing/reopening them, +adding commits, modifying PR descriptions, or any other PR-level action. diff --git a/.claude/development.md b/.claude/development.md new file mode 100644 index 0000000..886484a --- /dev/null +++ b/.claude/development.md @@ -0,0 +1,47 @@ +# pgxntool Development Guidelines + +**THIS FILE IS FOR PGXNTOOL DEVELOPERS ONLY.** + +If you are an extension developer using pgxntool in your project, this file does not +apply to you. See the top-level `CLAUDE.md` instead. + +## Critical: Work from pgxntool-test, Not Here + +**NEVER make changes to pgxntool directly from this repository.** + +pgxntool development must be done from a checkout of **pgxntool-test**, which contains +the full test infrastructure. Working here directly means you cannot run tests, and +any changes you commit cannot be validated before merging. + +**Correct workflow:** +1. Clone or use an existing checkout of `pgxntool-test` +2. Work in a worktree: both `pgxntool/` and `pgxntool-test/` will be siblings +3. Make changes to `pgxntool/` from within that pgxntool-test context +4. Run the test suite via `make test` in pgxntool-test before committing + +**See:** https://github.com/Postgres-Extensions/pgxntool-test for the full development +workflow. + +--- + +## Makefile Variable Assignment Rules + +**RULE: Do not use `:=` (simply expanded) unless you have a specific need for immediate evaluation.** + +Use `=` (recursively expanded) for standard variable assignments. Reserve `:=` for cases where the right-hand side must be evaluated exactly once at assignment time — for example, when assigning the result of a `$(call ...)` function that references the variable being set (which would cause infinite recursion with `=`). + +When a variable must also override command-line values, combine `override` with `:=` — but only where `override` is genuinely needed. + +## Debug Level Rules (lib.sh `debug` function) + +`debug LEVEL "msg"` prints when `DEBUG >= LEVEL`. LEVEL encodes how noisy/esoteric a message is — how far you'd crank `DEBUG` before you'd want to see it — **not** code nesting depth. A top-level line can warrant a high level if it's esoteric, and loop-body detail is usually high precisely because it's noisy. Judge by signal-to-noise. + +The tiers are anchors, not strict multiples of 10 — any value in range is fine, leaving room to fine-tune between existing calls without renumbering: + +- **10**: Critical errors, important warnings +- **20**: Warnings, significant state changes +- **30**: General debugging, function entry/exit, array operations +- **40**: Verbose details, loop iterations +- **50+**: Maximum verbosity (per-iteration innards) + +Note: The BATS test helper `debug` function (in `tests/lib/helpers.bash` in pgxntool-test) uses a separate 1–5 scale controlled by `$TESTDEBUG`. The two systems are independent. diff --git a/.gitattributes b/.gitattributes index c602ea0..8dc1599 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,4 +1,8 @@ .gitattributes export-ignore +.claude/ export-ignore +.github/ export-ignore +*.md export-ignore +.DS_Store export-ignore *.asc export-ignore *.adoc export-ignore *.html export-ignore diff --git a/.github/workflows/CLAUDE.md b/.github/workflows/CLAUDE.md new file mode 100644 index 0000000..48c749c --- /dev/null +++ b/.github/workflows/CLAUDE.md @@ -0,0 +1,61 @@ +# .github/workflows — CI Architecture + +## Workflow files + +- **`ci.yml`** — main CI for pgxntool pull requests. Runs `check-test-pr` (verifies + the paired pgxntool-test PR's CI passed), then optionally runs `test` (only for the + commit-with-no-tests path — see below). +- **`protect-label.yml`** — enforces that only maintainers with write access can apply + or remove the `commit-with-no-tests` label. + +## Normal CI flow (paired test PR exists) + +When a pgxntool PR has a corresponding open PR in pgxntool-test with the same branch +name, the `check-test-pr` job polls (up to 20 minutes) for that test PR's CI to +complete and pass. If it passes, pgxntool CI passes — **no tests run here**. Tests run +exactly once, in pgxntool-test's own CI. + +## commit-with-no-tests path + +When a maintainer applies the `commit-with-no-tests` label (and no paired test PR +exists), the `test` job runs tests directly in pgxntool CI against pgxntool-test/master. +This is the rare exception, not the norm. + +## Cross-repo reusable workflow — tradeoffs and constraints + +The `test` job calls a reusable workflow from pgxntool-test: +```yaml +uses: Postgres-Extensions/pgxntool-test/.github/workflows/run-tests.yml@ +``` + +GitHub Actions requires the `uses:` ref to be a **static string** — expressions like +`${{ }}` are not supported in the repo/path portion or the `@ref` suffix in practice. + +### The @branch → @master ref + +While developing on a feature branch where pgxntool-test also has changes, this ref +is set to `@` so CI can find `run-tests.yml` before it lands on master. + +**IMPORTANT**: This ref must be updated to `@master` before pgxntool merges. The +correct merge order is: **pgxntool-test merges first**, then update this ref to +`@master`, then pgxntool merges. + +**For Claude**: Do NOT leave a `@` ref without explicit user approval. The +user merges directly from the PR page — there are no manual steps between merges. +See `.github/workflows/CLAUDE.md` in pgxntool-test for the full picture. + +### Changes to run-tests.yml + +`run-tests.yml` lives in pgxntool-test and is the single source of truth for all test +steps. If it changes, pgxntool's CI uses `@master` — so it won't see the new version +until pgxntool-test merges. This is acceptable because: +- Changes to `run-tests.yml` require a paired test PR (not commit-with-no-tests) +- When a paired test PR exists, pgxntool's `test` job is skipped anyway +- The two scenarios are mutually exclusive in practice + +## Label name + +The label `commit-with-no-tests` is defined as a const (`NO_TEST_LABEL`) in `ci.yml` +and as `LABEL` in `protect-label.yml`. The job-level `if:` condition in +`protect-label.yml` must also use the literal string (YAML can't reference JS consts) +— keep these in sync if the label name ever changes. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..9e3ce90 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,294 @@ +name: CI + +on: + pull_request: + # We use 'pull_request' (not 'pull_request_target') deliberately. + # 'pull_request_target' runs with write access to the base repo, which is + # a security risk for untrusted fork code. Since this workflow only reads + # from other public repos (no secrets needed), 'pull_request' is correct + # and safe even for fork PRs. + +permissions: + contents: read # required by actions/checkout in the reusable test workflow + pull-requests: read + checks: read + +concurrency: + group: ci-pr-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + check-test-pr: + name: Check for paired pgxntool-test PR + runs-on: ubuntu-latest + # This check polls until the paired pgxntool-test CI run completes + # (up to 20 minutes). The job timeout gives a few minutes of headroom. + timeout-minutes: 25 + outputs: + run-tests: ${{ steps.check.outputs.run_tests }} + test-ref: ${{ steps.check.outputs.test_ref }} + + steps: + - name: Find paired pgxntool-test PR or check commit-with-no-tests label + id: check + # Pinned to an immutable SHA (supply-chain hardening); comment tracks the tag. + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 + with: + # GITHUB_TOKEN is sufficient for reading public repos. If these repos + # are ever made private, replace with a PAT stored as a secret with + # 'repo' scope on both repos. Note: PAT expiration causes silent + # failures here — the API returns 401 and the job errors out instead + # of failing gracefully with a useful message. + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const branch = context.payload.pull_request.head.ref; + const prNumber = context.payload.pull_request.number; + // Single source of truth for the label name. Must also match the + // literal string in the protect-label.yml job-level `if:` condition + // (YAML expressions can't reference JS constants). + const NO_TEST_LABEL = 'commit-with-no-tests'; + + // master-to-master PRs have no paired test PR by convention. + // Run tests against pgxntool-test/master directly. + // + // If a fork PR's branch is named 'master', that's almost certainly + // a mistake (contributors should use a feature branch), but we + // don't block it — just warn visibly as an annotation on the run. + // Note: pull_request gives a read-only token for fork PRs, so we + // can't post a PR comment back to the upstream repo from here. + // Gate on the BASE branch too: this shortcut is only for + // master-to-master PRs. A PR from master into some other base must + // still go through the normal paired-test lookup below. + if (branch === 'master' && context.payload.pull_request.base.ref === 'master') { + const headRepo = context.payload.pull_request.head.repo; + const isBaseRepo = + headRepo?.owner?.login === context.repo.owner && + headRepo?.name === context.repo.repo; + if (!isBaseRepo) { + core.warning( + `PR head branch is named 'master' but comes from a fork ` + + `(${headRepo?.full_name ?? 'unknown'}). Contributors should ` + + `use a feature branch, not master. Proceeding with tests ` + + `against pgxntool-test/master.` + ); + } + core.setOutput('run_tests', 'true'); + core.setOutput('test_ref', 'master'); + return; + } + + // The owner of this PR's head repo — the contributor's fork owner + // for fork PRs, or the base repo owner for maintainer PRs. + // The paired pgxntool-test PR must come from the SAME owner. + // We never cross-match PRs across different contributors' forks. + const prOwner = context.payload.pull_request.head.repo?.owner?.login; + + // Look for open pgxntool-test PRs with the SAME branch name AND + // the same fork owner. Branch names must match exactly. + // + // The GitHub API's 'head' filter requires "owner:branch" format. + // We list all open PRs and filter locally — safe for repos with + // few open PRs, and avoids needing to know the fork repo name. + // paginate() fetches all pages automatically, so this is correct + // even if pgxntool-test ever exceeds 100 open PRs (the per_page cap). + const prs = await github.paginate(github.rest.pulls.list, { + owner: context.repo.owner, + repo: 'pgxntool-test', + state: 'open', + per_page: 100 + }); + + const matching = prs.filter(pr => + pr.head.ref === branch && + pr.head.repo?.owner?.login === prOwner + ); + if (matching.length > 1) { + core.setFailed( + `Multiple open pgxntool-test PRs from ${prOwner} match branch ` + + `'${branch}'. Cannot determine which one to use.\n\n` + + `Close all but one, then re-run this check.` + ); + return; + } + + const testPR = matching.length === 1 ? matching[0] : null; + + if (testPR) { + // Error if the no-test label is also set — that's contradictory. + // Re-fetch the PR live (not from payload) in case the label was + // added after this workflow was triggered. + const { data: currentPR } = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: prNumber + }); + if (currentPR.labels.some(l => l.name === NO_TEST_LABEL)) { + core.setFailed( + `PR has the '${NO_TEST_LABEL}' label, but a paired ` + + `pgxntool-test PR #${testPR.number} exists on branch '${branch}'.\n\n` + + `Remove the '${NO_TEST_LABEL}' label — it should only be used ` + + `when there is genuinely no paired test PR.` + ); + return; + } + + // A paired test PR exists. Verify its CI passed for the exact + // current HEAD SHA and that the run is recent enough to be valid. + const sha = testPR.head.sha; + const testPRUrl = + `https://github.com/${context.repo.owner}/pgxntool-test/pull/${testPR.number}`; + const recheckUrl = + `https://github.com/${context.repo.owner}/${context.repo.repo}/pull/${prNumber}/checks`; + + core.info(`Found pgxntool-test PR #${testPR.number} (${sha.slice(0, 7)})`); + + // Poll until all check runs for the exact HEAD SHA complete. + // Using 'ref: sha' (not branch name) ensures we only see runs for + // this commit — never stale runs from an older push on the same branch. + // + // We poll rather than fail immediately because both repos are often + // pushed close together. When that happens, pgxntool CI starts while + // pgxntool-test CI may not have queued yet. We wait up to 20 minutes. + const POLL_INTERVAL_MS = 30 * 1000; + const MAX_WAIT_MS = 20 * 60 * 1000; + const waitStart = Date.now(); + let runs; + + while (true) { + // per_page: 100 is intentional here — a single commit will + // not realistically have 100+ CI check runs, so pagination + // is unnecessary. (pulls.list uses paginate() above because + // an active repo could have many open PRs.) + const { data: checks } = await github.rest.checks.listForRef({ + owner: context.repo.owner, + repo: 'pgxntool-test', + ref: sha, + per_page: 100 + }); + runs = checks.check_runs; + + const incomplete = runs.filter(r => r.status !== 'completed'); + if (runs.length > 0 && incomplete.length === 0) break; + + const elapsed = Date.now() - waitStart; + if (elapsed >= MAX_WAIT_MS) { + const mins = Math.round(elapsed / 60000); + if (runs.length === 0) { + core.setFailed( + `pgxntool-test PR #${testPR.number} has no CI runs for ` + + `SHA ${sha.slice(0, 7)} after waiting ${mins} min.\n\n` + + `Push a commit (or manually re-run CI) on the test PR:\n` + + ` Test PR: ${testPRUrl}\n` + + ` Re-run this check: ${recheckUrl}` + ); + } else { + const names = incomplete.map(r => r.name).join(', '); + core.setFailed( + `pgxntool-test PR #${testPR.number} CI did not finish within ` + + `${mins} min for SHA ${sha.slice(0, 7)}: ${names}\n\n` + + ` Test PR: ${testPRUrl}\n` + + ` Re-run this check: ${recheckUrl}` + ); + } + return; + } + + if (runs.length === 0) { + core.info(`No CI runs yet for pgxntool-test PR #${testPR.number} (${sha.slice(0, 7)}); waiting 30s...`); + } else { + const names = incomplete.map(r => r.name).join(', '); + core.info(`pgxntool-test CI still running (${names}); waiting 30s...`); + } + await new Promise(resolve => setTimeout(resolve, POLL_INTERVAL_MS)); + } + + // All checks complete — look for failures. + // 'success', 'skipped', 'neutral' are non-blocking. + const failed = runs.filter( + r => !['success', 'skipped', 'neutral'].includes(r.conclusion) + ); + if (failed.length > 0) { + const names = failed.map(r => `${r.name} (${r.conclusion})`).join(', '); + core.setFailed( + `pgxntool-test PR #${testPR.number} CI failed for ` + + `SHA ${sha.slice(0, 7)}: ${names}\n\n` + + `Fix the test PR CI, then re-run this check:\n` + + ` Test PR: ${testPRUrl}\n` + + ` Re-run this check: ${recheckUrl}` + ); + return; + } + + core.info( + `pgxntool-test PR #${testPR.number} CI passed for ` + + `SHA ${sha.slice(0, 7)} — tests run there, not here.` + ); + core.setOutput('run_tests', 'false'); + core.setOutput('test_ref', sha); + return; + } + + // No paired test PR found. Check for the NO_TEST_LABEL label, + // which a maintainer can apply when a pgxntool change genuinely + // needs no test changes (unusual). + // + // We make a live API call rather than reading from the event + // payload. The payload is a snapshot from when this workflow was + // triggered — a maintainer may have added the label after that. + const { data: pr } = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: prNumber + }); + + if (pr.labels.some(l => l.name === NO_TEST_LABEL)) { + core.info( + `'${NO_TEST_LABEL}' label is present; running tests ` + + "against pgxntool-test/master. The protect-label workflow " + + "ensures only maintainers can apply this label." + ); + core.setOutput('run_tests', 'true'); + core.setOutput('test_ref', 'master'); + return; + } + + // Neither a paired test PR nor the override label was found. + // Fail with a clear, actionable message. + core.setFailed( + `No paired pgxntool-test PR found for branch '${branch}', ` + + `and no '${NO_TEST_LABEL}' label on this PR.\n\n` + + `pgxntool changes should always be paired with matching test\n` + + `changes in pgxntool-test. This check enforces that pairing.\n\n` + + `To resolve:\n` + + ` 1. Open a PR in pgxntool-test from the SAME account (${prOwner}),\n` + + ` on a branch ALSO named '${branch}'. Both the branch name and\n` + + ` the head owner must match exactly for the pairing to work.\n\n` + + ` 2. If this pgxntool change truly needs no test updates (unusual),\n` + + ` ask a maintainer to apply the '${NO_TEST_LABEL}' label.\n` + + ` Only maintainers can apply this label. It is not a normal\n` + + ` shortcut — most pgxntool changes require test updates.\n\n` + + `See: https://github.com/Postgres-Extensions/pgxntool-test#ci-and-contributing` + ); + + test: + needs: check-test-pr + if: needs.check-test-pr.outputs.run-tests == 'true' + # ----------------------------------------------------------------------- + # CROSS-REPO REUSABLE WORKFLOW — READ BEFORE CHANGING THIS REF + # See: .github/workflows/CLAUDE.md for full architecture notes. + # + # The ref must be a static string — GitHub Actions does not support + # expressions in uses:. It points at pgxntool-test's run-tests.yml on + # master. (During feature-branch development this is temporarily set to + # @ so CI can find run-tests.yml before it lands on master, and + # flipped back to @master once pgxntool-test/ has merged.) + # ----------------------------------------------------------------------- + uses: Postgres-Extensions/pgxntool-test/.github/workflows/run-tests.yml@master + with: + # pgxntool: this PR's own branch, on its own account (a fork for fork PRs). + pgxntool-owner: ${{ github.event.pull_request.head.repo.owner.login }} + pgxntool-branch: ${{ github.event.pull_request.head.ref }} + # pgxntool-test: no paired test PR in this path, so use canonical master + # from Postgres-Extensions only (never a fork's master). + pgxntool-test-owner: Postgres-Extensions + pgxntool-test-ref: master diff --git a/.github/workflows/claude-code-review.yml b/.github/workflows/claude-code-review.yml new file mode 100644 index 0000000..fc7e26c --- /dev/null +++ b/.github/workflows/claude-code-review.yml @@ -0,0 +1,102 @@ +name: Claude Code Review + +# Runs on PRs INTO this repo. We use pull_request_target (not pull_request) so +# that PRs from a fork can access CLAUDE_CODE_OAUTH_TOKEN — GitHub withholds +# secrets from `pull_request` runs triggered by forks, which is why the plain +# `pull_request` version never worked for fork PRs. +# +# SECURITY: pull_request_target runs in the BASE repo with secrets and a +# write-capable token. The job is gated to PRs from the trusted `jnasbyupgrade` +# fork only — an arbitrary external fork can never trigger this secret-bearing +# job. The workflow file always comes from the base branch (master), so a PR +# cannot modify the reviewer that runs on it. We check out the PR head only for +# read context (persist-credentials: false) and never build or execute PR code. +on: + pull_request_target: + types: [opened, synchronize, reopened, ready_for_review] + +concurrency: + group: claude-review-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + claude-review: + # Trusted fork only, and skip drafts (don't spend API/CI on unfinished PRs). + # To add more trusted owners, extend the head-owner check. + if: >- + github.event.pull_request.draft == false && + github.event.pull_request.head.repo.owner.login == 'jnasbyupgrade' + runs-on: ubuntu-latest + timeout-minutes: 60 + permissions: + contents: read + pull-requests: write # post the review comments + checks: read # read sibling check-runs for the cost gate + steps: + # COST GATE: the paid Claude review is the last thing to run. Wait for the + # PR head's OTHER check-runs to finish and only proceed if they are clean. + # If any sibling check failed we skip the review to avoid spending money + # reviewing a PR that is already known-broken. Uniform across all repos: + # it discovers sibling checks dynamically (no per-repo workflow names). + # - decision=run : all sibling checks completed with a good conclusion, + # OR no sibling checks exist after a short grace window + # (nothing to gate on), OR the poll timed out is treated + # as skip (see below). + # - decision=skip : at least one sibling check failed/cancelled/etc, or + # we timed out waiting for still-pending checks. + # We exclude this workflow's own check-run (job name `claude-review`) so the + # gate never waits on or fails because of itself. + - name: Wait for CI; skip the paid review if any check failed + id: gate + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + SHA: ${{ github.event.pull_request.head.sha }} + run: | + decision=skip + for i in $(seq 1 72); do # ~24 min max + json=$(gh api "repos/$REPO/commits/$SHA/check-runs" --paginate \ + --jq '[.check_runs[] | select(.name != "claude-review")]' 2>/dev/null) || json='' + [ -z "$json" ] && { sleep 20; continue; } + total=$(jq 'length' <<<"$json") + if [ "$total" -eq 0 ]; then + [ "$i" -ge 9 ] && { decision=run; break; } # ~3 min grace: nothing to gate on + sleep 20; continue + fi + pending=$(jq '[.[]|select(.status!="completed")]|length' <<<"$json") + if [ "$pending" -eq 0 ]; then + bad=$(jq '[.[]|select((.conclusion//"")|test("^(failure|cancelled|timed_out|action_required|stale)$"))]|length' <<<"$json") + [ "$bad" -eq 0 ] && decision=run || decision=skip + break + fi + sleep 20 + done + echo "decision=$decision" >> "$GITHUB_OUTPUT" + echo "gate decision: $decision" + + - name: Check out PR head (read-only context) + if: steps.gate.outputs.decision == 'run' + # Intentionally tracks the major-version tag (not a pinned SHA) so + # upstream fixes are picked up automatically. + uses: actions/checkout@v4 + with: + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 1 + persist-credentials: false + + - name: Run Claude Code Review + if: steps.gate.outputs.decision == 'run' + uses: anthropics/claude-code-action@v1 + with: + claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + # Provide github_token so the action uses it directly for GitHub API + # calls instead of the OIDC->GitHub-App-token exchange, which 401s under + # pull_request_target. GITHUB_TOKEN is repo/workflow-scoped (independent + # of the actor's role) and has pull-requests: write here. + github_token: ${{ secrets.GITHUB_TOKEN }} + # NOTE: plugin_marketplaces can't be pinned — it tracks the + # marketplace repo's default branch (upstream anthropics/claude-code). + plugin_marketplaces: 'https://github.com/anthropics/claude-code.git' + plugins: 'code-review@claude-code-plugins' + prompt: '/code-review:code-review ${{ github.repository }}/pull/${{ github.event.pull_request.number }}' diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml new file mode 100644 index 0000000..c85ec00 --- /dev/null +++ b/.github/workflows/claude.yml @@ -0,0 +1,46 @@ +name: Claude Code + +on: + issue_comment: + types: [created] + pull_request_review_comment: + types: [created] + issues: + types: [opened, assigned] + pull_request_review: + types: [submitted] + +# No concurrency limit: @claude mentions are independent, read-only requests; +# serializing would only delay responses and cancelling would drop them. +jobs: + claude: + if: | + (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) || + (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) || + (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) || + (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude'))) + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + pull-requests: read + issues: read + id-token: write + actions: read # Required for Claude to read CI results on PRs + steps: + - name: Checkout repository + # Intentionally tracks the major-version tag (not a pinned SHA) so + # upstream fixes are picked up automatically. + uses: actions/checkout@v4 + with: + fetch-depth: 1 + persist-credentials: false + + - name: Run Claude Code + id: claude + uses: anthropics/claude-code-action@v1 + with: + claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + # Allows Claude to read CI results on PRs + additional_permissions: | + actions: read diff --git a/.github/workflows/protect-label.yml b/.github/workflows/protect-label.yml new file mode 100644 index 0000000..de71ea3 --- /dev/null +++ b/.github/workflows/protect-label.yml @@ -0,0 +1,146 @@ +name: Protect 'commit-with-no-tests' label + +on: + # IMPORTANT: Must use pull_request_target, NOT pull_request. + # + # 'pull_request' from a fork runs with a read-only GITHUB_TOKEN scoped to + # the fork. It cannot add or remove labels on the upstream repo (write + # operation), and cannot call getCollaboratorPermissionLevel (requires write + # permission to the target repo). + # + # 'pull_request_target' runs in the base repo's context with a token that + # has write access — exactly what we need here. + # + # Security: because pull_request_target has write access, never check out + # or execute code from the PR head in this workflow. This workflow only calls + # the GitHub API via actions/github-script and is safe. + pull_request_target: + types: [labeled, unlabeled] + +jobs: + protect: + # Only fire for the label we care about. All other label changes are + # unaffected by this workflow. + # Note: this literal must match the LABEL const defined in the script below. + if: github.event.label.name == 'commit-with-no-tests' + runs-on: ubuntu-latest + permissions: + pull-requests: write # To add/remove labels + issues: write # GitHub label API goes through the issues endpoint + + steps: + - name: Enforce write-access-only on 'commit-with-no-tests' label + # Pinned to an immutable SHA: this workflow runs as pull_request_target + # with write access, so a moved upstream tag must not change what runs. + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 + with: + script: | + const actor = context.actor; + const prNumber = context.payload.pull_request.number; + const action = context.payload.action; // 'labeled' or 'unlabeled' + // Single source of truth for the label name within this script. + // Must also match the literal in the job-level `if:` condition above + // (YAML expressions can't reference JS constants). + const LABEL = 'commit-with-no-tests'; + + // When this workflow re-adds or removes the label itself, that fires + // this event again with actor = 'github-actions[bot]'. Without this + // guard the job loops forever. We match any '[bot]' suffix to also + // cover other automation (Dependabot, Renovate, etc.). + if (actor.endsWith('[bot]')) { + core.info(`Actor is a bot (${actor}); skipping permission check`); + return; + } + + // Check the actor's effective permission level in this repo. + // + // EDGE CASE — 404 for non-collaborators: This API returns 404 when + // the user is not an explicit collaborator. This is the normal case + // for contributors who forked and opened a PR. If we don't catch + // this error, the job crashes with an unhandled exception and the + // label stays in whatever state the contributor put it in — + // defeating the entire protection. + // + // EDGE CASE — org team members: Users with write access via org + // team membership (not a direct collaborator invite) correctly show + // as 'write' here because the API returns effective permission. + // Exception: if the org has "private member visibility" set and the + // token can't enumerate team membership, they may get a 404 instead. + // If that becomes an issue, add a fallback to + // github.rest.orgs.getMembershipForUser(). + // + // EDGE CASE — other errors: Network blips, API outages, and rate + // limiting all throw here. We fail safe by treating any unexpected + // error as "no write access" and logging for debugging. + let hasWrite = false; + try { + const { data: perm } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: context.repo.owner, + repo: context.repo.repo, + username: actor + }); + hasWrite = ['admin', 'write'].includes(perm.permission); + } catch (e) { + if (e.status === 404) { + // Not a collaborator — no write access. Expected and normal. + hasWrite = false; + } else { + core.warning( + `Unexpected error checking permissions for ${actor} ` + + `(HTTP ${e.status}): ${e.message}. Treating as no write access.` + ); + hasWrite = false; + } + } + + if (action === 'labeled' && !hasWrite) { + core.info(`${actor} lacks write access; removing '${LABEL}' label`); + await github.rest.issues.removeLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + name: LABEL + }); + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + body: + `@${actor} The \`commit-with-no-tests\` label can only be applied by ` + + `maintainers with write access to this repository.\n\n` + + `If you believe no test changes are needed for this PR, please ask a ` + + `maintainer to apply the label after reviewing. Note that most pgxntool ` + + `changes do require paired test updates — this label should be used sparingly.` + }); + + } else if (action === 'unlabeled' && !hasWrite) { + // Non-writer removed the label. Put it back. + // + // EDGE CASE — brief label-absent window: There is a short window + // between removal and this workflow re-adding the label. During + // that window the label genuinely does not exist. This is harmless + // in practice: the ci.yml workflow reads labels via a live API + // call (not from its cached payload), so a re-run after the label + // is restored will pick it up correctly. + core.info(`${actor} lacks write access; re-adding '${LABEL}' label`); + await github.rest.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + labels: [LABEL] + }); + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + body: + `@${actor} The \`commit-with-no-tests\` label can only be removed by ` + + `maintainers with write access to this repository.\n\n` + + `Contact a maintainer if you believe this label was applied in error.` + }); + + } else if (hasWrite) { + core.info( + `${actor} has write access; '${action}' on '${LABEL}' label is approved` + ); + } diff --git a/.gitignore b/.gitignore index a01ee28..5ffb236 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ .*.swp +.claude/*.local.json diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..75c9d16 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,284 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## CI Monitoring After Every Push + +**REQUIRED**: After every `git push`, immediately start a background task to +monitor the CI run for that push. If you pushed to both pgxntool and +pgxntool-test, start a background task for each repo — do not monitor them +sequentially. + +The CI monitor lives in the pgxntool-test checkout: run +`bash ../pgxntool-test/.claude/skills/ci/scripts/monitor-ci.sh` (the `/ci` +skill). It monitors both repos and derives the owner from the current repo. +Pass the exact push SHA when available — `gh run list --branch` has a race +condition: if two pushes land close together on the same branch, `--branch` +may pick up the wrong run. `--commit SHA` targets the exact push and avoids it. + +## Scope of This File + +**CLAUDE.md is for people USING pgxntool** — extension developers who have embedded +pgxntool into their project via `git subtree`. It documents the build system, available +commands, and how pgxntool works. + +**If you are making changes to pgxntool itself**, stop — you are in the wrong place. +See `.claude/` in this directory for developer guidelines. More importantly, pgxntool +development must be done from the **pgxntool-test** repository, not from here. See the +`Development Workflow` section below. + +Any agent working in an extension project should always defer to that project's own +CLAUDE.md and instructions over anything stated here. + +## Git Commit Guidelines + +**IMPORTANT**: When creating commit messages, do not attribute commits to yourself (Claude). Commit messages should reflect the work being done without AI attribution in the message body. The standard Co-Authored-By trailer is acceptable. + +## Critical: What This Repo Actually Is + +**pgxntool is NOT a standalone project.** It is a meta-framework that exists ONLY to be embedded into PostgreSQL extension projects via `git subtree`. This repo cannot be built, tested, or run directly. + +**Think of it like this**: pgxntool is to PostgreSQL extensions what a Makefile template library is to C projects - it's infrastructure code that gets copied into other projects, not a project itself. + +## Critical: Directory Purity - NO Temporary Files + +**This directory contains ONLY files that get embedded into extension projects.** When extension developers run `git subtree add`, they pull the entire pgxntool directory into their project. + +**ABSOLUTE RULE**: NO temporary files, scratch work, or development tools may be added to this directory. + +**Examples of what NEVER belongs here:** +- Temporary files (scratch notes, test output, debugging artifacts) +- Development scripts or tools (these go in pgxntool-test/) +- Planning documents (PLAN-*.md files go in pgxntool-test/) +- Any file you wouldn't want in every extension project that uses pgxntool + +**CLAUDE.md exception**: CLAUDE.md exists here for AI assistant guidance, but is excluded from distributions via `.gitattributes export-ignore`. Same with `.claude/` directory. + +**Why this matters**: Any file you add here will be pulled into hundreds of extension projects via git subtree. Keep this directory lean and clean. + +## Development Workflow: Work from pgxntool-test + +**CRITICAL**: All development work on pgxntool should be done from the pgxntool-test repository, NOT from this repository. + +**For complete development workflow documentation, see:** +https://github.com/Postgres-Extensions/pgxntool-test + +## Two-Repository Development Pattern + +This codebase uses a two-repository pattern: + +1. **pgxntool/** (this repo) - The framework code that gets embedded into extension projects +2. **pgxntool-test** - The test harness that validates pgxntool functionality + +**For development and testing workflow, see:** +https://github.com/Postgres-Extensions/pgxntool-test + +## How Extension Developers Use pgxntool + +Extension projects include pgxntool via git subtree: + +```bash +git subtree add -P pgxntool --squash git@github.com:decibel/pgxntool.git release +pgxntool/setup.sh +``` + +After setup, their Makefile typically contains just: +```makefile +include pgxntool/base.mk +``` + +## Architecture: Two-Phase Build System + +### Phase 1: Meta Generation (`build_meta.sh`) +- Processes `META.in.json` (template with placeholders/empty values) +- Strips out X_comment fields and empty values +- Produces clean `META.json` + +### Phase 2: Variable Extraction (`meta.mk.sh`) +- Parses `META.json` using `JSON.sh` (a bash-based JSON parser) +- Generates `meta.mk` with Make variables: + - `PGXN` - distribution name + - `PGXNVERSION` - version number + - `EXTENSIONS` - list of extensions provided + - `EXTENSION_*_VERSION` - per-extension versions + - `EXTENSION_VERSION_FILES` - auto-generated versioned SQL files +- `base.mk` includes `meta.mk` via `-include` + +### The Magic of base.mk + +`base.mk` provides a complete PGXS-based build system: +- Auto-detects extension SQL files in `sql/` +- Auto-detects C modules in `src/*.c` +- Auto-detects tests in `test/sql/*.sql` +- Auto-generates versioned extension files (`extension--version.sql`) +- Handles Asciidoc → HTML conversion +- Integrates with PGXN distribution format +- Manages git tagging and release packaging + +## File Structure for Consumer Projects + +Projects using pgxntool follow this layout: +``` +project/ +├── Makefile # include pgxntool/base.mk +├── META.in.json # Template metadata (customize for your extension) +├── META.json # Auto-generated from META.in.json +├── extension.control # Standard PostgreSQL control file +├── pgxntool/ # This repo, embedded via git subtree +├── sql/ +│ └── extension.sql # Base extension SQL +├── src/ # Optional C code (*.c files) +├── test/ +│ ├── deps.sql # Load extension and test dependencies +│ ├── sql/*.sql # Test SQL files +│ └── expected/*.out # Expected test outputs +└── doc/ # Optional docs (*.adoc, *.asciidoc) +``` + +## Commands for Extension Developers (End Users) + +These are the commands extension developers use (documented for context): + +```bash +make # Build extension (generates versioned SQL, docs) +make test # Full test: testdeps → install → installcheck → show diffs +make results # Run tests and update expected output files +make html # Generate HTML from Asciidoc sources +make tag # Create git branch for current META.json version +make dist # Create PGXN .zip (auto-tags, places in ../) +make pgtle # Generate pg_tle registration SQL (see pg_tle Support below) +make check-pgtle # Check pg_tle installation and report version +make install-pgtle # Install pg_tle registration SQL files into database +make pgxntool-sync # Update to latest pgxntool via git subtree pull +``` + +## Testing with pgxntool + +### Critical Testing Rules + +**NEVER use `make installcheck` directly**. Always use `make test` instead. The `make test` target ensures: +- Clean builds before testing +- Proper test isolation +- Correct test dependency installation +- Proper cleanup and result comparison + +**Database Connection Requirement**: PostgreSQL must be running before executing `make test`. If you get connection errors (e.g., "could not connect to server"), stop and ask the user to start PostgreSQL. + +**Claude Code MUST NEVER run `make results`**. This target updates test expected output files and requires manual human verification of test changes before execution. + +**Claude Code MUST NEVER modify files in `test/expected/`**. These are expected test outputs that define correct behavior and must only be updated through the `make results` workflow. + +The workflow is: +1. Human runs `make test` and examines diffs +2. Human manually verifies changes are correct +3. Human manually runs `make results` to update expected files + +### Test Output Mechanics + +pgxntool uses PostgreSQL's pg_regress test framework: +- **Actual test output**: Written to `test/results/` directory +- **Expected output**: Stored in `test/expected/` directory +- **Test comparison**: pg_regress compares actual vs expected and generates diffs; `make test` displays them +- **Updating expectations**: `make results` copies `test/results/` → `test/expected/` + +When tests fail, examine the diff output carefully. The actual test output in `test/results/` shows what your code produced, while `test/expected/` shows what was expected. + +## Key Implementation Details + +### PostgreSQL Version Handling +- `MAJORVER` = version × 10 (e.g., 9.6 → 96, 13 → 130) +- Tests use `--load-language=plpgsql` for versions < 13 +- Version detection via `pg_config --version` + +### Test System (pg_regress based) +- Tests in `test/sql/*.sql`, outputs compared to `test/expected/*.out` +- Setup via `test/pgxntool/setup.sql` (loads pgTap and deps.sql) +- `.IGNORE: installcheck` allows `make test` to handle errors (show diffs, then exit with error status) +- `make results` updates expected outputs after test runs + +### Document Generation +- Auto-detects `asciidoctor` or `asciidoc` +- Generates HTML from `*.adoc` and `*.asciidoc` in `$(DOC_DIRS)` +- HTML required for `make dist`, optional for `make install` +- Template-based rules via `ASCIIDOC_template` + +### Distribution Packaging +- `make dist` creates `../PGXN-VERSION.zip` +- Always creates git branch tag matching version +- Uses `git archive` to package +- Validates repo is clean before tagging + +### Subtree Sync Support +- `make pgxntool-sync` pulls the latest release (the `release` tag) from the canonical repo +- `pgxntool/pgxntool-sync.sh [ []]` does the work and can be run without make +- `make pgxntool-sync-` pulls from the `pgxntool-sync-` variable (` `) +- Uses `git subtree pull --squash`, then `update-setup-files.sh` for a 3-way merge of copied files +- Requires clean repo (no uncommitted changes) + +### pg_tle Support + +pgxntool can generate pg_tle (Trusted Language Extensions) registration SQL for deploying extensions in AWS RDS/Aurora without filesystem access. + +**Usage:** `make pgtle` or `make pgtle PGTLE_VERSION=1.5.0+` + +**Output:** `pg_tle/{version_range}/{extension}.sql` + +**For version range details and API compatibility boundaries, see:** `pgtle_versions.md` + +**Installation targets:** + +- `make check-pgtle` - Checks if pg_tle is installed and reports the version. Reports version from `pg_extension` if extension has been created, or newest available version from `pg_available_extension_versions` if available but not created. Errors if pg_tle not available in cluster. Assumes `PG*` environment variables are configured. + +- `make install-pgtle` - Auto-detects pg_tle version and installs appropriate registration SQL files. Updates or creates pg_tle extension as needed. Determines which version range files to install based on detected version. Runs all generated SQL files via `psql` to register extensions with pg_tle. Assumes `PG*` environment variables are configured. + +**Version notation:** +- `X.Y.Z+` means >= X.Y.Z +- `X.Y.Z-A.B.C` means >= X.Y.Z and < A.B.C (note boundary) + +**Key implementation details:** +- Script: `pgxntool/pgtle-wrap.sh` (bash) +- Parses `.control` files for metadata (NOT META.json) +- Fixed delimiter: `$_pgtle_wrap_delimiter_$` (validated not in source) +- Each output file contains ALL versions and ALL upgrade paths +- Multi-extension support (multiple .control files) +- Output directory `pg_tle/` excluded from git +- Depends on `make all` to ensure versioned SQL files exist first +- Only processes versioned files (`sql/{ext}--{version}.sql`), not base files + +**SQL file handling:** +- **Version files** (`sql/{ext}--{version}.sql`): Generated automatically by `make all` from base `sql/{ext}.sql` file +- **Upgrade scripts** (`sql/{ext}--{v1}--{v2}.sql`): Created manually by users when adding new extension versions +- The script ensures the default_version file exists if the base file exists (creates it from base file if missing) +- All version files and upgrade scripts are discovered and included in the generated pg_tle registration SQL + +**Dependencies:** +Generated files depend on: +- Control file (metadata source) +- All SQL files (sql/{ext}--*.sql) - must run `make all` first +- Generator script itself + +**Limitations:** +- No C code support (pg_tle requires trusted languages only) +- PostgreSQL 14.5+ required (pg_tle not available on earlier versions) + +## Critical Gotchas + +1. **Empty Variables**: If `DOCS` or `MODULES` is empty, base.mk sets to empty to prevent PGXS errors +2. **testdeps Pattern**: Never add recipes to `testdeps` - create separate target and make it a prerequisite +3. **META.json is Generated**: Always edit `META.in.json`, never `META.json` directly +4. **Control File Versions**: No automatic validation that `.control` matches `META.json` version +5. **PGXNTOOL_NO_PGXS_INCLUDE**: Setting this skips PGXS inclusion (for special scenarios) +6. **Distribution Placement**: `.zip` files go in parent directory (`../`) to avoid repo clutter + +## Scripts + +- **setup.sh** - Initializes pgxntool in a new extension project (copies templates, creates directories) +- **build_meta.sh** - Strips empty fields from META.in.json to create META.json +- **meta.mk.sh** - Parses META.json via JSON.sh and generates meta.mk with Make variables +- **JSON.sh** - Third-party bash JSON parser (MIT licensed) +- **safesed** - Utility for safe sed operations + +## Related Repositories + +- **pgxntool-test** - Test harness for validating pgxntool functionality: https://github.com/Postgres-Extensions/pgxntool-test +- Never produce any kind of metrics or estimates unless you have data to back them up. If you do have data you MUST reference it. \ No newline at end of file diff --git a/HISTORY.asc b/HISTORY.asc index 9cb793b..b0894da 100644 --- a/HISTORY.asc +++ b/HISTORY.asc @@ -1,5 +1,112 @@ -STABLE ------- +2.1.0 +----- +== Fix setup.sh / pgxntool-sync.sh / update-setup-files.sh inside a git worktree +These scripts guarded the project root with `[ -d .git ]`. In a linked worktree +`.git` is a file, not a directory, so the check failed: setup.sh would wrongly +re-run `git init`, and the sync scripts aborted with "Not in a git repository." +They now detect the repo with `git rev-parse --git-dir`, which works in both a +normal clone and a worktree. + +== Fix `verify-results` checking stale results in `make results` +`make results` ran `verify-results` before `make test`, so it checked stale +`regression.diffs` from a prior run. Reordered so `verify-results` always +checks the fresh results. + +== Fix `PGXNTOOL_ENABLE_TEST_BUILD`/`PGXNTOOL_ENABLE_TEST_INSTALL` ignoring command-line values +Without `override`, the `pgxntool_validate_yesno` normalization was silently +skipped when these variables were set on the command line. + +2.0.3 +----- +== Fix pgxntool-sync remote, and make it runnable without make +`make pgxntool-sync` pointed at the old SSH URL, which fails without +GitHub SSH keys; it now uses `https://github.com/Postgres-Extensions/pgxntool.git` +over HTTPS (still the `release` tag). The subtree pull + `update-setup-files.sh` +logic moved into `pgxntool/pgxntool-sync.sh`, which the make targets now wrap, so +you can also sync without `make` by running it directly. + +2.0.2 +----- +== Fix parse_control_file corrupting values with trailing comments +Control file values like `default_version = '1.0.0' # comment` were parsed +incorrectly — the trailing quote was left in the value due to comment removal +happening after quote stripping. Fixed by removing comments first. + +2.0.1 +----- +== Improve bash compatibility (specifically for Mac OS) +Mac OS uses the (very old) bash 3.2 as the default shell; fix a few compatibility bugs. + +2.0.0 +----- +== Remove .source file support +PostgreSQL removed `.source` file processing from `pg_regress` in PG15. +The `input/*.source` → `sql/*.sql` and `output/*.source` → +`expected/*.out` conversion mechanism no longer functions. All related +variables (`TEST__SOURCE__*`), the `make_results.sh` helper script, and +the special-case logic in `make results` have been removed. Extensions +that used `.source` files should convert them to regular `test/sql/*.sql` +and `test/expected/*.out` files using relative paths. + +== Add test-build for pre-test validation +When CREATE EXTENSION fails due to a SQL syntax error, PostgreSQL reports only a cryptic error with limited context. test-build runs your extension SQL directly through pg_regress first, so syntax errors show the exact file, line, and position — cutting debugging time significantly. Place SQL files in `test/build/` to enable; auto-detects based on file presence. + +== Add test/install for one-time test setup +Extensions that install dependencies or run expensive setup in every test file pay that cost once per test. test/install runs setup SQL once before the entire test suite, and all regular tests share the resulting database state. This can dramatically speed up test suites that install extensions or load fixtures. Place SQL files in `test/install/` to enable; auto-detects based on file presence. + +== Add verify-results safeguard for make results +`make results` now refuses to run when tests are failing (detected via `regression.diffs`). Prevents accidentally blessing incorrect output as the new expected results. Enabled by default; disable with `PGXNTOOL_ENABLE_VERIFY_RESULTS=no`. + +== Fix bash 3.2 compatibility and shebang portability +`${#ARRAY[@]:-0}` is a syntax error in bash 3.2; replaced with `${#ARRAY[@]}`. +Shell scripts now use `#!/usr/bin/env bash`. + +1.1.2 +----- +== Fix double --dbname bug that defeated unique test database names +The unique database naming introduced in 1.1.0 was ineffective because +base.mk added --dbname=$(REGRESS_DBNAME) to REGRESS_OPTS while PGXS +also appends --dbname=$(CONTRIB_TESTDB). The second --dbname caused +pg_regress to create a contrib_regression database that collided across +projects. Fixed by overriding CONTRIB_TESTDB after include $(PGXS) instead. + +1.1.1 +----- +== Fix pg_tle exception handler and empty upgrade files +The exception handler for `uninstall_extension()` now correctly catches +`no_data_found` (P0002) instead of `undefined_object` (42704). Empty upgrade +files are now treated as valid no-op upgrades for version bumps. Added +`ON_ERROR_STOP=1` to `run_pgtle_sql()` so psql errors propagate correctly. + +1.1.0 +----- +== Use unique database names for tests +Tests now use a unique database name based on the project name and a hash of the +current directory. This prevents test conflicts when running tests for multiple +projects in parallel. + +== Add 3-way merge support for setup files after pgxntool-sync +New `update-setup-files.sh` script handles merging changes to files initially +copied by `setup.sh` (`.gitignore`, `test/deps.sql`). After running `make +pgxntool-sync`, the script performs a 3-way merge if both you and pgxntool have +modified the same file, using git's native conflict markers for resolution. + +1.0.0 +----- +== Fix broken multi-extension support +Prior to this fix, distributions with multiple extensions or extensions with +versions different from the PGXN distribution version were completely broken. +Extension versions are now correctly read from each `.control` file's +`default_version` instead of using META.json's distribution version. + +== Add pg_tle support +New `make pgtle` target generates pg_tle registration SQL for extensions. +Supports pg_tle version ranges (1.0.0-1.4.0, 1.4.0-1.5.0, 1.5.0+) with +appropriate API calls for each range. See README for usage. + +== Use git tags for distribution versioning +The `tag` and `rmtag` targets now create/delete git tags instead of branches. + == Support 13+ The `--load-language` option was removed from `pg_regress` in 13. @@ -7,7 +114,9 @@ The `--load-language` option was removed from `pg_regress` in 13. As part of this change, you will want to review the changes to test/deps.sql. === Support asciidoc documentation targets -By default, if asciidoctor or asciidoc exists on the system, any files in doc/ that end in .adoc or .asciidoc will be processed to html. +By default, if asciidoctor or asciidoc exists on the system, any files in doc/ +that end in .adoc or .asciidoc will be processed to html. + See the README for full details. === Support 9.2 @@ -35,7 +144,7 @@ VERSION is defined by PGXS itself, so trying to use it causes problems. Old code didn't deal with the lack of a . that can appear in a 10+ version. 0.1.10 ------- +----- ### Remove invalid `git subtree pull` options 0.1.9 diff --git a/LICENSE b/LICENSE index 5a20925..4a507f7 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2015, Jim Nasby, Blue Treble Solutions +Copyright (c) 2015-2026, Jim Nasby, Blue Treble Solutions All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/README.asc b/README.asc index c2c6683..db6162f 100644 --- a/README.asc +++ b/README.asc @@ -23,6 +23,12 @@ pgxntool/setup.sh TODO: Create a nice script that will init a new project for you. +== Development + +If you want to contribute to pgxntool development, work from the https://github.com/Postgres-Extensions/pgxntool-test[pgxntool-test] repository, not from this repository. That repository contains the test infrastructure and development tools needed to validate changes to pgxntool. This repository contains only the framework files that get embedded into extension projects via `git subtree`. + +Changes are normally paired across both repos: a `pgxntool` change should come with a matching branch (same name, on the same account) and PR in `pgxntool-test`, and CI enforces this pairing. See the https://github.com/Postgres-Extensions/pgxntool-test#ci-and-contributing[CI and Contributing] section in pgxntool-test for the full workflow. + == Usage Typically, you can just create a simple Makefile that does nothing but include base.mk: @@ -41,6 +47,122 @@ This will build any .html files that can be created. See <<_Document_Handling>>. === test Runs unit tests via the PGXS `installcheck` target. Unlike a simple `make installcheck` though, the `test` rule has the following prerequisites: clean testdeps install installcheck. All of those are PGXS rules, except for `testdeps`. +NOTE: While you can still run `make installcheck` or any other valid PGXS make target directly, it's recommended to use `make test` when using pgxntool. The `test` target ensures clean builds, proper test isolation, and correct dependency installation. + +=== test-build +Validates that extension SQL files are syntactically correct before running the full test suite. This feature runs SQL files from `test/build/` through `pg_regress`, providing better error messages than `CREATE EXTENSION` failures when there are syntax errors in your extension code. + +**How it works:** + +1. Place SQL files in `test/build/*.sql` +2. Place expected output in `test/build/expected/*.out` +3. These files run through `pg_regress` before `make test` runs the main test suite +4. If any build test fails, the test run stops immediately with clear error messages + +**Directory structure:** + +---- +test/build/ +├── *.sql # SQL test files (checked in) +├── expected/ # Expected output files (checked in) +│ └── *.out +└── sql/ # GENERATED - do not edit or check in + └── *.sql # Synced from *.sql above +---- + +The `sql/` subdirectory is generated automatically by `make test-build`. It is listed in `.gitignore` and removed by `make clean`. Do not place files directly in `test/build/sql/`. + +**Configuration:** + +The feature auto-detects based on whether `test/build/*.sql` files exist: + +- Files present → feature enabled automatically +- No files → feature disabled (no impact on existing projects) + +You can override auto-detection by setting `PGXNTOOL_ENABLE_TEST_BUILD`: +---- +# In your Makefile +PGXNTOOL_ENABLE_TEST_BUILD = yes # or no +---- + +**Example: Validate extension SQL compiles** + +Create `test/build/build.sql` to run your extension's SQL directly: + +---- +\set ECHO none +-- Sets ON_ERROR_STOP, VERBOSITY verbose, and ON_ERROR_ROLLBACK +\i test/pgxntool/psql.sql +-- Suppress column headers and row counts for cleaner expected output +\t + +BEGIN; +SET client_min_messages = WARNING; + +-- Install dependencies your extension requires +CREATE EXTENSION IF NOT EXISTS pgtap CASCADE; + +-- Clean slate +DROP EXTENSION IF EXISTS myext; +DROP SCHEMA IF EXISTS myext; +CREATE SCHEMA myext; + +-- Run the actual extension SQL (not CREATE EXTENSION) +-- psql.sql above ensures errors abort immediately with clear messages +\i sql/myext.sql + +-- If we get here, the build succeeded; ON_ERROR_STOP would have aborted on any error above +\echo # BUILD TEST SUCCEEDED +ROLLBACK; +---- + +This approach catches SQL syntax errors *before* running `CREATE EXTENSION`, giving clearer error messages with line numbers. The `ROLLBACK` ensures nothing persists—this is purely validation. + +**Why use `\i` instead of `CREATE EXTENSION`?** + +When `CREATE EXTENSION` fails, PostgreSQL shows only "syntax error" with limited context. Running the SQL directly via `\i` shows the exact line and position of errors, making debugging much faster. + +=== test/install +Runs setup files before the main test suite within the same `pg_regress` invocation. This allows expensive one-time operations (like extension installation) to set up state that persists into the regular test files. + +**How it works:** + +1. Place SQL files in `test/install/*.sql` +2. Place expected output alongside as `test/install/*.out` +3. A schedule file is auto-generated that lists install files with `../install/` relative paths +4. `pg_regress` processes the install schedule first, then runs regular test files — all in one invocation, so database state persists + +**Directory structure:** + +---- +test/install/ +├── *.sql # SQL setup files (checked in) +├── *.out # Expected output (checked in, alongside .sql) +├── .gitignore # Ignores pg_regress artifacts (*.out.diff) +└── schedule # GENERATED - auto-created by make +---- + +The `schedule` file is generated automatically and listed in `.gitignore`. Do not edit it. + +**Configuration:** + +The feature auto-detects based on whether `test/install/*.sql` files exist: + +- Files present → feature enabled automatically +- No files → feature disabled (no impact on existing projects) + +You can override auto-detection by setting `PGXNTOOL_ENABLE_TEST_INSTALL`: +---- +# In your Makefile +PGXNTOOL_ENABLE_TEST_INSTALL = yes # or no +---- + +**Why this is useful:** + +Without `test/install`, each test file typically needs to run `CREATE EXTENSION` in its setup, which adds overhead and doesn't allow validating the installation step separately. With `test/install`, setup runs once before all tests, and any state it creates (tables, extensions, etc.) is available to every subsequent test file. + +**Key detail:** Install files and regular tests run in a single `pg_regress` invocation. This means the database is NOT dropped between install and test phases — state created by install files persists into the main test suite. + === testdeps This rule allows you to ensure certain actions have taken place before running tests. By default it has a single prerequisite, `pgtap`, which will attempt to install http://pgtap.org[pgtap] from PGXN. This depneds on having the pgxn client installed. @@ -60,10 +182,36 @@ If you want to over-ride the default dependency on `pgtap` you should be able to WARNING: It will probably cause problems if you try to create a `testdeps` rule that has a recipe. Instead of doing that, put the recipe in a separate rule and make that rule a prerequisite of `testdeps` as show in the example. === results -Because `make test` ultimately runs `installcheck`, it's using the Postgres test suite. Unfortunately, that suite is based on running `diff` between a raw output file and expected results. I *STRONGLY* recommend you use http://pgtap.org[pgTap] instead! The extra effort of learning pgTap will quickly pay for itself. https://github.com/decibel/trunklet-format/blob/master/test/sql/base.sql[This example] might help get you started. +Because `make test` ultimately runs `installcheck`, it's using the Postgres test suite. Unfortunately, that suite is based on running `diff` between a raw output file and expected results. I *STRONGLY* recommend you use http://pgtap.org[pgTap] instead! With pgTap, it's MUCH easier to determine whether a test is passing or not - tests explicitly pass or fail rather than requiring you to examine diff output. The extra effort of learning pgTap will quickly pay for itself. https://github.com/decibel/trunklet-format/blob/master/test/sql/base.sql[This example] might help get you started. No matter what method you use, once you know that all your tests are passing correctly, you need to create or update the test output expected files. `make results` does that for you. +IMPORTANT: *`make results` requires manual verification first*. The correct workflow is: + +1. Run `make test` and examine the diff output +2. Manually verify that the differences are correct and expected +3. Only then run `make results` to update the expected output files in `test/expected/` + +Never run `make results` without first verifying the test changes are correct. The `results` target copies files from `test/results/` to `test/expected/`, so running it blindly will make incorrect output become the new expected behavior. + +==== verify-results safeguard +By default, `make results` will refuse to run if `test/results/regression.diffs` exists (indicating failing tests). This prevents accidentally updating expected files with incorrect output. + +If tests are failing, you'll see: +---- +ERROR: Tests are failing. Cannot run 'make results'. +Fix test failures first, then run 'make results'. +---- + +To disable this safeguard (not recommended): +---- +# In your Makefile +PGXNTOOL_ENABLE_VERIFY_RESULTS = no + +# Or on the command line +make PGXNTOOL_ENABLE_VERIFY_RESULTS=no results +---- + === tag `make tag` will create a git branch for the current version of your extension, as determined by the META.json file. The reason to do this is so you can always refer to the exact code that went into a released version. @@ -77,11 +225,126 @@ WARNING: You will be very unhappy if you forget to update the .control file for NOTE: Part of the `clean` recipe is cleaning up these .zip files. If you accidentally clean before uploading, just run `make dist-only`. === pgxntool-sync -This rule will pull down the latest released version of PGXNtool via `git subtree pull`. +This rule will pull down the latest released version of PGXNtool via `git subtree pull` and then reconcile the files `setup.sh` copied into your project (`.gitignore`, `test/deps.sql`) with a 3-way merge. NOTE: Your repository must be clean (no modified files) in order to run this. Running this command will produce a git commit of the merge. -TIP: There is also a `pgxntool-sync-%` rule if you need to do more advanced things. +TIP: The actual work is done by `pgxntool/pgxntool-sync.sh`, so you can run it directly (`pgxntool/pgxntool-sync.sh`) if you'd rather not go through `make`. It optionally takes `` and `` arguments to pull from somewhere other than the default. + +TIP: There is also a `pgxntool-sync-%` rule if you need to do more advanced things. `make pgxntool-sync-` pulls from the ` ` defined by the `pgxntool-sync-` make variable. + +=== distclean +`make distclean` removes generated configuration files (`META.json`, `meta.mk`, `control.mk`) that survive a normal `make clean`. + +NOTE: PGXS doesn't provide any special support for `distclean` — its built-in `distclean` target simply depends on `clean`. PGXNtool uses its own `PGXNTOOL_distclean` variable to track files that should only be removed by `distclean`, not `clean`. + +If your extension generates additional files that should be removed by `distclean` but not `clean`, you can add them: +---- +PGXNTOOL_distclean += my_generated_config.mk +---- + +=== pgtle +Generates pg_tle (Trusted Language Extensions) registration SQL files for deploying extensions in managed environments like AWS RDS/Aurora. See <<_pg_tle_Support>> for complete documentation. + +`make pgtle` generates SQL files in `pg_tle/` subdirectories organized by pg_tle version ranges. For version range details, see `pgtle_versions.md`. + +=== check-pgtle +Checks if pg_tle is installed and reports the version. This target: +- Reports the version from `pg_extension` if `CREATE EXTENSION pg_tle` has been run in the database +- Errors if pg_tle is not available in the cluster + +This target assumes `PG*` environment variables are configured for `psql` connectivity. + +---- +make check-pgtle +---- + +=== run-pgtle +Registers all extensions with pg_tle by executing the generated pg_tle registration SQL files in a PostgreSQL database. This target: +- Requires pg_tle extension to be installed (checked via `check-pgtle`) +- Uses `pgtle.sh` to determine which version range directory to use based on the installed pg_tle version +- Runs all generated SQL files via `psql` to register your extensions with pg_tle + +This target assumes that running `psql` without any arguments will connect to the desired database. You can control this by setting the various PG* environment variables (and possibly using the `.pgpassword` file). See the PostgreSQL documentation for more details. + +NOTE: The `pgtle` target is a dependency, so `make run-pgtle` will automatically generate the SQL files if needed. + +---- +make run-pgtle +---- + +After running `make run-pgtle`, you can create your extension in the database: +---- +CREATE EXTENSION "your-extension-name"; +---- + +== Version-Specific SQL Files + +PGXNtool automatically generates version-specific SQL files from your base SQL file. These files follow the pattern `sql/{extension}--{version}.sql` and are used by PostgreSQL's extension system to install specific versions of your extension. + +=== How Version Files Are Generated + +When you run `make` (or `make all`), PGXNtool: + +1. Reads your `META.json` file to determine the extension version from `provides.{extension}.version` +2. Generates a Makefile rule that copies your base SQL file (`sql/{extension}.sql`) to the version-specific file (`sql/{extension}--{version}.sql`) +3. Executes this rule, creating the version-specific file with a header comment indicating it's auto-generated + +For example, if your `META.json` contains: +---- +"provides": { + "myext": { + "version": "1.2.3", + ... + } +} +---- + +Running `make` will create `sql/myext--1.2.3.sql` by copying `sql/myext.sql`. + +=== What Controls the Version Number + +The version number comes from `META.json` → `provides.{extension}.version`, *not* from your `.control` file's `default_version` field. The `.control` file's `default_version` is used by PostgreSQL to determine which version to install by default, but the actual version-specific file that gets generated is determined by what's in `META.json`. + +To change the version of your extension: +1. Update `provides.{extension}.version` in `META.json` +2. Run `make` to regenerate the version-specific file +3. Update `default_version` in your `.control` file to match (if needed) + +=== Committing Version Files + +Version-specific SQL files are now treated as permanent files that should be committed to your repository. This makes it much easier to test updates to extensions, as you can see exactly what SQL was included in each version. + +IMPORTANT: These files are auto-generated and include a header comment warning not to edit them. Any manual changes will be overwritten the next time you run `make`. To modify the extension, edit the base SQL file (`sql/{extension}.sql`) instead. + +=== Alternative: Ignoring Version Files + +If you prefer not to commit version-specific SQL files, you must add them to your `.gitignore` to prevent `make dist` from failing due to untracked files. Add the following to your `.gitignore`: + +---- +# Auto-generated version-specific SQL files (if not committing them) +sql/*--*.sql +!sql/*--*--*.sql +---- + +The second line (`!sql/*--*--*.sql`) ensures that upgrade scripts (which contain two version numbers and should be manually written) are still tracked. + +WARNING: If you ignore version files instead of committing them, they will NOT be included in your PGXN distribution (`make dist` uses `git archive`, which only includes tracked files). This means users installing your extension from PGXN will need `make` and PGXS available to build the extension - they cannot simply copy the SQL files into their PostgreSQL installation. For maximum compatibility, we recommend committing version files. + +=== Distribution Inclusion + +Version-specific files are included in distributions created by `make dist` only if they are committed to git. Since `make dist` uses `git archive`, only tracked files are included in the distribution archive. + +=== Multiple Versions + +If you need to support multiple versions of your extension: + +1. Create additional version-specific files manually (e.g., `sql/myext--1.0.0.sql`, `sql/myext--1.1.0.sql`) +2. Create upgrade scripts for version transitions (e.g., `sql/myext--1.0.0--1.1.0.sql`) +3. Update `META.json` to reflect the current version you're working on +4. Commit all version files and upgrade scripts to your repository + +The version file for the current version (specified in `META.json`) will be automatically regenerated when you run `make`, but other version files you create manually will be preserved. == Document Handling PGXNtool supports generation and installation of document files. There are several variables and rules that control this behavior. @@ -106,7 +369,7 @@ Location of `asciidoc` or equivalent executable. If not set PGXNtool will search for first `asciidoctor`, then `asciidoc`. ASCIIDOC_EXTS:: File extensions to consider as Asciidoc. -Defined as `+= adoc asciidoc`. +Defined as `+= adoc asciidoc asc`. ASCIIDOC_FILES:: Asciidoc input files. PGXNtool searches each `$(DOC_DIRS)` directory, looking for files with any `$(ASCIIDOC_EXTS)` extension. @@ -158,7 +421,111 @@ Because of this, `base.mk` will forcibly define it to be NULL if it's empty. PGXNtool appends *all* files found in all `$(DOC_DIRS)` to `DOCS`. +== pg_tle Support +[[_pg_tle_Support]] +pgxntool can generate link:https://github.com/aws/pg_tle[pg_tle (Trusted Language Extensions)] registration SQL for deploying PostgreSQL extensions in managed environments like AWS RDS and Aurora where filesystem access is not available. + +For make targets, see: <<_pgtle>>, <<_check_pgtle>>, <<_run_pgtle>>. + +=== What is pg_tle? + +pg_tle is an AWS open-source framework that enables developers to create and deploy PostgreSQL extensions without filesystem access. Traditional PostgreSQL extensions require `.control` and `.sql` files on the filesystem, which isn't possible in managed services like RDS and Aurora. + +pg_tle solves this by: +- Storing extension metadata and SQL in database tables +- Using the `pgtle_admin` role for administrative operations +- Enabling `CREATE EXTENSION` to work in managed environments + +=== Quick Start + +Generate pg_tle registration SQL for your extension: + +---- +make pgtle +---- + +This creates files in `pg_tle/` subdirectories organized by pg_tle version ranges. See `pgtle_versions.md` for complete version range details and API compatibility boundaries. + +=== Version Groupings + +pgxntool creates different sets of files for different pg_tle versions to handle backward-incompatible API changes. Each version boundary represents a change to pg_tle's API functions that we use. + +For details on version boundaries and API changes, see `pgtle_versions.md`. + +=== Installation Example + +IMPORTANT: This is only a basic example. Always refer to the link:https://github.com/aws/pg_tle[main pg_tle documentation] for complete installation instructions and best practices. + +Basic installation steps: + +. Ensure pg_tle is installed and grant the `pgtle_admin` role to your user +. Generate and run the pg_tle registration SQL files: ++ +---- +make run-pgtle +---- ++ +This automatically detects your pg_tle version and runs the appropriate SQL files. See `pgtle_versions.md` for version range details. +. Create your extension: `CREATE EXTENSION myextension;` + +=== Advanced Usage + +==== Multi-Extension Projects + +If your project has multiple extensions (multiple `.control` files), `make pgtle` generates files for all of them: + +---- +myproject/ +├── ext1.control +├── ext2.control +└── pg_tle/ + ├── 1.0.0-1.5.0/ + │ ├── ext1.sql + │ └── ext2.sql + └── 1.5.0+/ + ├── ext1.sql + └── ext2.sql +---- + +=== How It Works +`make pgtle` does the following: + +. Parses control file(s): Extracts `comment`, `default_version`, `requires`, and `schema` fields +. Discovers SQL files: Finds all versioned files (`sql/{ext}--{version}.sql`) and upgrade scripts (`sql/{ext}--{ver1}--{ver2}.sql`) +. Wraps SQL content: Uses a fixed dollar-quote delimiter (`$_pgtle_wrap_delimiter_$`) to wrap SQL for pg_tle functions +. Generates registration SQL: Creates `pgtle.install_extension()` calls for each version, `pgtle.install_update_path()` for upgrades, and `pgtle.set_default_version()` for the default +. Version-specific output: Generates separate files for different pg_tle capability levels + +Each generated SQL file is wrapped in a transaction (`BEGIN;` ... `COMMIT;`) to ensure atomic installation. + +=== Troubleshooting + +==== "No versioned SQL files found" + +*Problem*: The script can't find `sql/{ext}--{version}.sql` files. + +*Solution*: Run `make` first to generate versioned files from your base `sql/{ext}.sql` file. + +==== "Control file not found" + +*Problem*: The script can't find `{ext}.control` in the current directory. + +*Solution*: Run `make pgtle` from your extension's root directory (where the `.control` file is). + +==== "SQL file contains reserved pg_tle delimiter" + +*Problem*: Your SQL files contain the string `$_pgtle_wrap_delimiter_$` (extremely unlikely). + +*Solution*: Don't use that dollar-quote delimiter in your code. + +==== Extension uses C code + +*Problem*: Your control file has `module_pathname`, indicating C code. + +*Solution*: pg_tle only supports trusted languages. You cannot use C extensions with pg_tle. The script will warn you but still generate files (which won't work). + +NOTE: there are several untrusted languages (such as plpython), and the only tests for C. == Copyright -Copyright (c) 2015 Jim Nasby +Copyright (c) 2026 Jim Nasby PGXNtool is released under a https://github.com/decibel/pgxntool/blob/master/LICENCE[BSD license]. Note that it includes https://github.com/dominictarr/JSON.sh[JSON.sh], which is released under a https://github.com/decibel/pgxntool/blob/master/JSON.sh.LICENCE[MIT license]. diff --git a/README.html b/README.html index ae4a597..431358f 100644 --- a/README.html +++ b/README.html @@ -2,31 +2,26 @@ - + - + PGXNtool @@ -428,26 +445,53 @@

PGXNtool

Table of Contents
@@ -466,7 +510,7 @@

PGXNtool

-

1. Install

+

1. Install

This assumes that you’ve already initialized your extension in git.

@@ -495,7 +539,18 @@

1. Install

-

2. Usage

+

2. Development

+
+
+

If you want to contribute to pgxntool development, work from the pgxntool-test repository, not from this repository. That repository contains the test infrastructure and development tools needed to validate changes to pgxntool. This repository contains only the framework files that get embedded into extension projects via git subtree.

+
+
+

Changes are normally paired across both repos: a pgxntool change should come with a matching branch (same name, on the same account) and PR in pgxntool-test, and CI enforces this pairing. See the CI and Contributing section in pgxntool-test for the full workflow.

+
+
+
+
+

3. Usage

Typically, you can just create a simple Makefile that does nothing but include base.mk:

@@ -508,7 +563,7 @@

2. Usage

-

3. make targets

+

4. make targets

These are the make targets that are provided by base.mk

@@ -526,19 +581,214 @@

3. make targe

-

3.1. html

+

4.1. html

This will build any .html files that can be created. See [_Document_Handling].

-

3.2. test

+

4.2. test

Runs unit tests via the PGXS installcheck target. Unlike a simple make installcheck though, the test rule has the following prerequisites: clean testdeps install installcheck. All of those are PGXS rules, except for testdeps.

+
+ + + + + +
+
Note
+		 +While you can still run make installcheck or any other valid PGXS make target directly, it’s recommended to use make test when using pgxntool. The test target ensures clean builds, proper test isolation, and correct dependency installation. +
+
+
+
+

4.3. test-build

+
+

Validates that extension SQL files are syntactically correct before running the full test suite. This feature runs SQL files from test/build/ through pg_regress, providing better error messages than CREATE EXTENSION failures when there are syntax errors in your extension code.

+
+
+

How it works:

+
+
+
    +
  1. +

    Place SQL files in test/build/*.sql

    +
    2. +
  2. +

    Place expected output in test/build/expected/*.out

    +
    3. +
  3. +

    These files run through pg_regress before make test runs the main test suite

    +
    4. +
  4. +

    If any build test fails, the test run stops immediately with clear error messages

    +
    5. +
+
+
+

Directory structure:

+
+
+
+
test/build/
+├── *.sql              # SQL test files (checked in)
+├── expected/          # Expected output files (checked in)
+│   └── *.out
+└── sql/               # GENERATED - do not edit or check in
+    └── *.sql          # Synced from *.sql above
+
+
+
+

The sql/ subdirectory is generated automatically by make test-build. It is listed in .gitignore and removed by make clean. Do not place files directly in test/build/sql/.

+
+
+

Configuration:

+
+
+

The feature auto-detects based on whether test/build/*.sql files exist:

+
+
+
    +
  • +

    Files present → feature enabled automatically

    +
    • +
  • +

    No files → feature disabled (no impact on existing projects)

    +
    • +
+
+
+

You can override auto-detection by setting PGXNTOOL_ENABLE_TEST_BUILD:

+
+
+
+
# In your Makefile
+PGXNTOOL_ENABLE_TEST_BUILD = yes  # or no
+
+
+
+

Example: Validate extension SQL compiles

+
+
+

Create test/build/build.sql to run your extension’s SQL directly:

+
+
+
+
\set ECHO none
+-- Sets ON_ERROR_STOP, VERBOSITY verbose, and ON_ERROR_ROLLBACK
+\i test/pgxntool/psql.sql
+-- Suppress column headers and row counts for cleaner expected output
+\t
+
+BEGIN;
+SET client_min_messages = WARNING;
+
+-- Install dependencies your extension requires
+CREATE EXTENSION IF NOT EXISTS pgtap CASCADE;
+
+-- Clean slate
+DROP EXTENSION IF EXISTS myext;
+DROP SCHEMA IF EXISTS myext;
+CREATE SCHEMA myext;
+
+-- Run the actual extension SQL (not CREATE EXTENSION)
+-- psql.sql above ensures errors abort immediately with clear messages
+\i sql/myext.sql
+
+-- If we get here, the build succeeded; ON_ERROR_STOP would have aborted on any error above
+\echo # BUILD TEST SUCCEEDED
+ROLLBACK;
+
+
+
+

This approach catches SQL syntax errors before running CREATE EXTENSION, giving clearer error messages with line numbers. The ROLLBACK ensures nothing persists—this is purely validation.

+
+
+

Why use \i instead of CREATE EXTENSION?

+
+
+

When CREATE EXTENSION fails, PostgreSQL shows only "syntax error" with limited context. Running the SQL directly via \i shows the exact line and position of errors, making debugging much faster.

+
-

3.3. testdeps

+

4.4. test/install

+
+

Runs setup files before the main test suite within the same pg_regress invocation. This allows expensive one-time operations (like extension installation) to set up state that persists into the regular test files.

+
+
+

How it works:

+
+
+
    +
  1. +

    Place SQL files in test/install/*.sql

    +
    2. +
  2. +

    Place expected output alongside as test/install/*.out

    +
    3. +
  3. +

    A schedule file is auto-generated that lists install files with ../install/ relative paths

    +
    4. +
  4. +

    pg_regress processes the install schedule first, then runs regular test files — all in one invocation, so database state persists

    +
    5. +
+
+
+

Directory structure:

+
+
+
+
test/install/
+├── *.sql              # SQL setup files (checked in)
+├── *.out              # Expected output (checked in, alongside .sql)
+├── .gitignore         # Ignores pg_regress artifacts (*.out.diff)
+└── schedule           # GENERATED - auto-created by make
+
+
+
+

The schedule file is generated automatically and listed in .gitignore. Do not edit it.

+
+
+

Configuration:

+
+
+

The feature auto-detects based on whether test/install/*.sql files exist:

+
+
+
    +
  • +

    Files present → feature enabled automatically

    +
    • +
  • +

    No files → feature disabled (no impact on existing projects)

    +
    • +
+
+
+

You can override auto-detection by setting PGXNTOOL_ENABLE_TEST_INSTALL:

+
+
+
+
# In your Makefile
+PGXNTOOL_ENABLE_TEST_INSTALL = yes  # or no
+
+
+
+

Why this is useful:

+
+
+

Without test/install, each test file typically needs to run CREATE EXTENSION in its setup, which adds overhead and doesn’t allow validating the installation step separately. With test/install, setup runs once before all tests, and any state it creates (tables, extensions, etc.) is available to every subsequent test file.

+
+
+

Key detail: Install files and regular tests run in a single pg_regress invocation. This means the database is NOT dropped between install and test phases — state created by install files persists into the main test suite.

+
+
+
+

4.5. testdeps

This rule allows you to ensure certain actions have taken place before running tests. By default it has a single prerequisite, pgtap, which will attempt to install pgtap from PGXN. This depneds on having the pgxn client installed.

@@ -574,16 +824,71 @@

3.3. testdeps

-

3.4. results

+

4.6. results

-

Because make test ultimately runs installcheck, it’s using the Postgres test suite. Unfortunately, that suite is based on running diff between a raw output file and expected results. I STRONGLY recommend you use pgTap instead! The extra effort of learning pgTap will quickly pay for itself. This example might help get you started.

+

Because make test ultimately runs installcheck, it’s using the Postgres test suite. Unfortunately, that suite is based on running diff between a raw output file and expected results. I STRONGLY recommend you use pgTap instead! With pgTap, it’s MUCH easier to determine whether a test is passing or not - tests explicitly pass or fail rather than requiring you to examine diff output. The extra effort of learning pgTap will quickly pay for itself. This example might help get you started.

No matter what method you use, once you know that all your tests are passing correctly, you need to create or update the test output expected files. make results does that for you.

+
+ + + + + +
+
Important
+		 +make results requires manual verification first. The correct workflow is: +
+
+
+
    +
  1. +

    Run make test and examine the diff output

    +
    2. +
  2. +

    Manually verify that the differences are correct and expected

    +
    3. +
  3. +

    Only then run make results to update the expected output files in test/expected/

    +
    4. +
+
+
+

Never run make results without first verifying the test changes are correct. The results target copies files from test/results/ to test/expected/, so running it blindly will make incorrect output become the new expected behavior.

+
+
+

4.6.1. verify-results safeguard

+
+

By default, make results will refuse to run if test/results/regression.diffs exists (indicating failing tests). This prevents accidentally updating expected files with incorrect output.

+
+
+

If tests are failing, you’ll see:

+
+
+
+
ERROR: Tests are failing. Cannot run 'make results'.
+Fix test failures first, then run 'make results'.
+
+
+
+

To disable this safeguard (not recommended):

+
+
+
+
# In your Makefile
+PGXNTOOL_ENABLE_VERIFY_RESULTS = no
+
+# Or on the command line
+make PGXNTOOL_ENABLE_VERIFY_RESULTS=no results
+
+
+
-

3.5. tag

+

4.7. tag

make tag will create a git branch for the current version of your extension, as determined by the META.json file. The reason to do this is so you can always refer to the exact code that went into a released version.

@@ -604,7 +909,7 @@

3.5. tag

-

3.6. dist

+

4.8. dist

make dist will create a .zip file for your current version that you can upload to PGXN. The file is named after the PGXN name and version (the top-level "name" and "version" attributes in META.json). The .zip file is placed in the parent directory so as not to clutter up your git repo.

@@ -622,9 +927,9 @@

3.6. dist

-

3.7. pgxntool-sync

+

4.9. pgxntool-sync

-

This rule will pull down the latest released version of PGXNtool via git subtree pull.

+

This rule will pull down the latest released version of PGXNtool via git subtree pull and then reconcile the files setup.sh copied into your project (.gitignore, test/deps.sql) with a 3-way merge.

@@ -645,16 +950,248 @@

3.7. pgxnto
Tip

+ +
-There is also a pgxntool-sync-% rule if you need to do more advanced things. +The actual work is done by pgxntool/pgxntool-sync.sh, so you can run it directly (pgxntool/pgxntool-sync.sh) if you’d rather not go through make. It optionally takes <repo> and <ref> arguments to pull from somewhere other than the default. +
+
+
+ + + + + +
+
Tip
+		 +There is also a pgxntool-sync-% rule if you need to do more advanced things. make pgxntool-sync-<name> pulls from the <repo> <ref> defined by the pgxntool-sync-<name> make variable. +
+
+
+
+

4.10. distclean

+
+

make distclean removes generated configuration files (META.json, meta.mk, control.mk) that survive a normal make clean.

+
+
+ + + + + +
+
Note
+		 +PGXS doesn’t provide any special support for distclean — its built-in distclean target simply depends on clean. PGXNtool uses its own PGXNTOOL_distclean variable to track files that should only be removed by distclean, not clean. +
+
+
+

If your extension generates additional files that should be removed by distclean but not clean, you can add them:

+
+
+
+
PGXNTOOL_distclean += my_generated_config.mk
+
+
+
+
+

4.11. pgtle

+
+

Generates pg_tle (Trusted Language Extensions) registration SQL files for deploying extensions in managed environments like AWS RDS/Aurora. See [_pg_tle_Support] for complete documentation.

+
+
+

make pgtle generates SQL files in pg_tle/ subdirectories organized by pg_tle version ranges. For version range details, see pgtle_versions.md.

+
+
+
+

4.12. check-pgtle

+
+

Checks if pg_tle is installed and reports the version. This target: +- Reports the version from pg_extension if CREATE EXTENSION pg_tle has been run in the database +- Errors if pg_tle is not available in the cluster

+
+
+

This target assumes PG* environment variables are configured for psql connectivity.

+
+
+
+
make check-pgtle
+
+
+
+
+

4.13. run-pgtle

+
+

Registers all extensions with pg_tle by executing the generated pg_tle registration SQL files in a PostgreSQL database. This target: +- Requires pg_tle extension to be installed (checked via check-pgtle) +- Uses pgtle.sh to determine which version range directory to use based on the installed pg_tle version +- Runs all generated SQL files via psql to register your extensions with pg_tle

+
+
+

This target assumes that running psql without any arguments will connect to the desired database. You can control this by setting the various PG* environment variables (and possibly using the .pgpassword file). See the PostgreSQL documentation for more details.

+
+
+ + + +
+
Note
+		 +The pgtle target is a dependency, so make run-pgtle will automatically generate the SQL files if needed.
+
+
+
make run-pgtle
+
+
+
+

After running make run-pgtle, you can create your extension in the database:

+
+
+
+
CREATE EXTENSION "your-extension-name";
+
+
-

4. Document Handling

+

5. Version-Specific SQL Files

+
+
+

PGXNtool automatically generates version-specific SQL files from your base SQL file. These files follow the pattern sql/{extension}--{version}.sql and are used by PostgreSQL’s extension system to install specific versions of your extension.

+
+
+

5.1. How Version Files Are Generated

+
+

When you run make (or make all), PGXNtool:

+
+
+
    +
  1. +

    Reads your META.json file to determine the extension version from provides.{extension}.version

    +
    2. +
  2. +

    Generates a Makefile rule that copies your base SQL file (sql/{extension}.sql) to the version-specific file (sql/{extension}--{version}.sql)

    +
    3. +
  3. +

    Executes this rule, creating the version-specific file with a header comment indicating it’s auto-generated

    +
    4. +
+
+
+

For example, if your META.json contains:

+
+
+
+
"provides": {
+  "myext": {
+    "version": "1.2.3",
+    ...
+  }
+}
+
+
+
+

Running make will create sql/myext—​1.2.3.sql by copying sql/myext.sql.

+
+
+
+

5.2. What Controls the Version Number

+
+

The version number comes from META.json → provides.{extension}.version, not from your .control file’s default_version field. The .control file’s default_version is used by PostgreSQL to determine which version to install by default, but the actual version-specific file that gets generated is determined by what’s in META.json.

+
+
+

To change the version of your extension: +1. Update provides.{extension}.version in META.json +2. Run make to regenerate the version-specific file +3. Update default_version in your .control file to match (if needed)

+
+
+
+

5.3. Committing Version Files

+
+

Version-specific SQL files are now treated as permanent files that should be committed to your repository. This makes it much easier to test updates to extensions, as you can see exactly what SQL was included in each version.

+
+
+ + + + + +
+
Important
+		 +These files are auto-generated and include a header comment warning not to edit them. Any manual changes will be overwritten the next time you run make. To modify the extension, edit the base SQL file (sql/{extension}.sql) instead. +
+
+
+
+

5.4. Alternative: Ignoring Version Files

+
+

If you prefer not to commit version-specific SQL files, you must add them to your .gitignore to prevent make dist from failing due to untracked files. Add the following to your .gitignore:

+
+
+
+
# Auto-generated version-specific SQL files (if not committing them)
+sql/*--*.sql
+!sql/*--*--*.sql
+
+
+
+

The second line (!sql/----*.sql) ensures that upgrade scripts (which contain two version numbers and should be manually written) are still tracked.

+
+
+ + + + + +
+
Warning
+		 +If you ignore version files instead of committing them, they will NOT be included in your PGXN distribution (make dist uses git archive, which only includes tracked files). This means users installing your extension from PGXN will need make and PGXS available to build the extension - they cannot simply copy the SQL files into their PostgreSQL installation. For maximum compatibility, we recommend committing version files. +
+
+
+
+

5.5. Distribution Inclusion

+
+

Version-specific files are included in distributions created by make dist only if they are committed to git. Since make dist uses git archive, only tracked files are included in the distribution archive.

+
+
+
+

5.6. Multiple Versions

+
+

If you need to support multiple versions of your extension:

+
+
+
    +
  1. +

    Create additional version-specific files manually (e.g., sql/myext—​1.0.0.sql, sql/myext—​1.1.0.sql)

    +
    2. +
  2. +

    Create upgrade scripts for version transitions (e.g., sql/myext—​1.0.0—​1.1.0.sql)

    +
    3. +
  3. +

    Update META.json to reflect the current version you’re working on

    +
    4. +
  4. +

    Commit all version files and upgrade scripts to your repository

    +
    5. +
+
+
+

The version file for the current version (specified in META.json) will be automatically regenerated when you run make, but other version files you create manually will be preserved.

+
+
+
+
+
+

6. Document Handling

PGXNtool supports generation and installation of document files. There are several variables and rules that control this behavior.

@@ -665,7 +1202,7 @@

4. If any generated files are missing (or out-of-date) during installation, PGXNtool will build them if Asciidoc is present on the system.

-

4.1. Document Variables

+

6.1. Document Variables

DOC_DIRS
@@ -691,7 +1228,7 @@

4
ASCIIDOC_EXTS

File extensions to consider as Asciidoc. -Defined as += adoc asciidoc.

+Defined as += adoc asciidoc asc.

ASCIIDOC_FILES
@@ -712,7 +1249,7 @@

4

-

4.2. Document Rules

+

6.2. Document Rules

If Asciidoc is found (or $(ASCIIDOC) is set), the html rule will be added as a prerequisite to the install and installchec rules. That will ensure that docs are generated for install and test, but only if Asciidoc is available. @@ -730,7 +1267,7 @@

4.2. Docu
ASCIIDOC_template
define ASCIIDOC_template
-%.html: %.$(1) (1)
+%.html: %.$(1) # (1)
 ifndef ASCIIDOC
 	$$(warning Could not find "asciidoc" or "asciidoctor". Add one of them to your PATH,)
 	$$(warning or set ASCIIDOC to the correct location.)
@@ -754,7 +1291,7 @@ 
4.2. Docu

-

4.3. The DOCS variable

+

6.3. The DOCS variable

This variable has special meaning to PGXS. See the Postgres documentation for full details.

@@ -782,10 +1319,189 @@

4.3

-

5. Copyright

+

7. pg_tle Support

+
+

pgxntool can generate pg_tle (Trusted Language Extensions) registration SQL for deploying PostgreSQL extensions in managed environments like AWS RDS and Aurora where filesystem access is not available.

+
+
+

For make targets, see: pgtle, check-pgtle, run-pgtle.

+
+
+

7.1. What is pg_tle?

+
+

pg_tle is an AWS open-source framework that enables developers to create and deploy PostgreSQL extensions without filesystem access. Traditional PostgreSQL extensions require .control and .sql files on the filesystem, which isn’t possible in managed services like RDS and Aurora.

+
+
+

pg_tle solves this by: +- Storing extension metadata and SQL in database tables +- Using the pgtle_admin role for administrative operations +- Enabling CREATE EXTENSION to work in managed environments

+
+
+
+

7.2. Quick Start

+
+

Generate pg_tle registration SQL for your extension:

+
+
+
+
make pgtle
+
+
+
+

This creates files in pg_tle/ subdirectories organized by pg_tle version ranges. See pgtle_versions.md for complete version range details and API compatibility boundaries.

+
+
+
+

7.3. Version Groupings

+
+

pgxntool creates different sets of files for different pg_tle versions to handle backward-incompatible API changes. Each version boundary represents a change to pg_tle’s API functions that we use.

+
+
+

For details on version boundaries and API changes, see pgtle_versions.md.

+
+
+
+

7.4. Installation Example

+
+ + + + + +
+
Important
+		 +This is only a basic example. Always refer to the main pg_tle documentation for complete installation instructions and best practices. +
+
+
+

Basic installation steps:

+
+
+
    +
  1. +

    Ensure pg_tle is installed and grant the pgtle_admin role to your user

    +
    2. +
  2. +

    Generate and run the pg_tle registration SQL files:

    +
    +
    +
    make run-pgtle
    +
    +
    -

    Copyright (c) 2015 Jim Nasby <Jim.Nasby@BlueTreble.com>

    +

    This automatically detects your pg_tle version and runs the appropriate SQL files. See pgtle_versions.md for version range details.

    +
    +
    3. +
  3. +

    Create your extension: CREATE EXTENSION myextension;

    +
    4. +
+
+
+
+

7.5. Advanced Usage

+
+

7.5.1. Multi-Extension Projects

+
+

If your project has multiple extensions (multiple .control files), make pgtle generates files for all of them:

+
+
+
+
myproject/
+├── ext1.control
+├── ext2.control
+└── pg_tle/
+    ├── 1.0.0-1.5.0/
+    │   ├── ext1.sql
+    │   └── ext2.sql
+    └── 1.5.0+/
+        ├── ext1.sql
+        └── ext2.sql
+
+
+
+
+
+

7.6. How It Works

+
+

make pgtle does the following:

+
+
+
    +
  1. +

    Parses control file(s): Extracts comment, default_version, requires, and schema fields

    +
    2. +
  2. +

    Discovers SQL files: Finds all versioned files (sql/{ext}--{version}.sql) and upgrade scripts (sql/{ext}--{ver1}--{ver2}.sql)

    +
    3. +
  3. +

    Wraps SQL content: Uses a fixed dollar-quote delimiter ($pgtle_wrap_delimiter$) to wrap SQL for pg_tle functions

    +
    4. +
  4. +

    Generates registration SQL: Creates pgtle.install_extension() calls for each version, pgtle.install_update_path() for upgrades, and pgtle.set_default_version() for the default

    +
    5. +
  5. +

    Version-specific output: Generates separate files for different pg_tle capability levels

    +
    6. +
+
+
+

Each generated SQL file is wrapped in a transaction (BEGIN; …​ COMMIT;) to ensure atomic installation.

+
+
+
+

7.7. Troubleshooting

+
+

7.7.1. "No versioned SQL files found"

+
+

Problem: The script can’t find sql/{ext}--{version}.sql files.

+
+
+

Solution: Run make first to generate versioned files from your base sql/{ext}.sql file.

+
+
+
+

7.7.2. "Control file not found"

+
+

Problem: The script can’t find {ext}.control in the current directory.

+
+
+

Solution: Run make pgtle from your extension’s root directory (where the .control file is).

+
+
+
+

7.7.3. "SQL file contains reserved pg_tle delimiter"

+
+

Problem: Your SQL files contain the string $pgtle_wrap_delimiter$ (extremely unlikely).

+
+
+

Solution: Don’t use that dollar-quote delimiter in your code.

+
+
+
+

7.7.4. Extension uses C code

+
+

Problem: Your control file has module_pathname, indicating C code.

+
+
+

Solution: pg_tle only supports trusted languages. You cannot use C extensions with pg_tle. The script will warn you but still generate files (which won’t work).

+
+
+ + + + + +
+
Note
+		 +there are several untrusted languages (such as plpython), and the only tests for C. +== Copyright +Copyright (c) 2026 Jim Nasby <Jim.Nasby@gmail.com> +

PGXNtool is released under a BSD license. Note that it includes JSON.sh, which is released under a MIT license.

@@ -793,9 +1509,11 @@

5. Copyright

+
+
diff --git a/_.gitignore b/_.gitignore index 3eb345a..1873c2c 100644 --- a/_.gitignore +++ b/_.gitignore @@ -1,11 +1,15 @@ # Editor files .*.swp +# Claude Code local settings +.claude/*.local.json + # Explicitly exclude META.json! !/META.json # Generated make files meta.mk +control.mk # Compiler output *.o @@ -13,14 +17,24 @@ meta.mk .deps/ # built targets -/sql/*--* -!/sql/*--*--*.sql +# Note: Version-specific files (sql/*--*.sql) are now tracked in git and should be committed # Test artifacts results/ regression.diffs regression.out +# Generated sql/ directory for test/build +# Created by make test-build. See README.asc for details. +test/build/sql/ + +# Auto-generated schedule file for test/install +# Created by make when test/install/*.sql files exist. +test/install/schedule + # Misc tmp/ .DS_Store + +# pg_tle generated files +/pg_tle/ diff --git a/base.mk b/base.mk index a976ebb..a7ab93c 100644 --- a/base.mk +++ b/base.mk @@ -1,5 +1,8 @@ PGXNTOOL_DIR := pgxntool +# Ensure 'all' is the default target (not META.json which happens to be first) +.DEFAULT_GOAL := all + # # META.json # @@ -10,13 +13,30 @@ META.json: META.in.json $(PGXNTOOL_DIR)/build_meta.sh # # meta.mk # -# Buind meta.mk, which contains info from META.json, and include it +# Build meta.mk, which contains PGXN distribution info from META.json PGXNTOOL_distclean += meta.mk meta.mk: META.json Makefile $(PGXNTOOL_DIR)/base.mk $(PGXNTOOL_DIR)/meta.mk.sh @$(PGXNTOOL_DIR)/meta.mk.sh $< >$@ -include meta.mk +# +# control.mk +# +# Build control.mk, which contains extension info from .control files +# This is separate from meta.mk because: +# - META.json specifies PGXN distribution metadata +# - .control files specify what PostgreSQL actually uses (e.g., default_version) +# These can differ, and PostgreSQL cares about the control file version. +# +# Find all control files first (needed for dependencies) +PGXNTOOL_CONTROL_FILES := $(wildcard *.control) +PGXNTOOL_distclean += control.mk +control.mk: $(PGXNTOOL_CONTROL_FILES) Makefile $(PGXNTOOL_DIR)/base.mk $(PGXNTOOL_DIR)/control.mk.sh + @$(PGXNTOOL_DIR)/control.mk.sh $(PGXNTOOL_CONTROL_FILES) >$@ + +-include control.mk + DATA = $(EXTENSION_VERSION_FILES) $(wildcard sql/*--*--*.sql) DOC_DIRS += doc # NOTE: if this is empty it gets forcibly defined to NUL before including PGXS @@ -24,25 +44,155 @@ DOCS += $(foreach dir,$(DOC_DIRS),$(wildcard $(dir)/*)) # Find all asciidoc targets ASCIIDOC ?= $(shell which asciidoctor 2>/dev/null || which asciidoc 2>/dev/null) -ASCIIDOC_EXTS += adoc asciidoc +ASCIIDOC_EXTS += adoc asciidoc asc ASCIIDOC_FILES += $(foreach dir,$(DOC_DIRS),$(foreach ext,$(ASCIIDOC_EXTS),$(wildcard $(dir)/*.$(ext)))) PG_CONFIG ?= pg_config TESTDIR ?= test TESTOUT ?= $(TESTDIR) -TEST_SOURCE_FILES += $(wildcard $(TESTDIR)/input/*.source) -TEST_OUT_FILES = $(subst input,output,$(TEST_SOURCE_FILES)) TEST_SQL_FILES += $(wildcard $(TESTDIR)/sql/*.sql) TEST_RESULT_FILES = $(patsubst $(TESTDIR)/sql/%.sql,$(TESTDIR)/expected/%.out,$(TEST_SQL_FILES)) -TEST_FILES = $(TEST_SOURCE_FILES) $(TEST_SQL_FILES) -REGRESS = $(sort $(notdir $(subst .source,,$(TEST_FILES:.sql=)))) # Sort is to get unique list +TEST_FILES = $(TEST_SQL_FILES) +REGRESS = $(sort $(notdir $(TEST_FILES:.sql=))) REGRESS_OPTS = --inputdir=$(TESTDIR) --outputdir=$(TESTOUT) # See additional setup below + +# +# OPTIONAL TEST FEATURES +# +# These sections configure optional test features. Each feature can be enabled/disabled +# via a makefile variable. If not explicitly set, features auto-detect based on +# directory existence or default behavior. The actual feature implementation is +# located later in this file (see test-build target, schedule file generation, etc.). +# + +# Helper function: normalize a yes/no variable to lowercase and validate. +# Usage: $(call pgxntool_validate_yesno,VALUE,VARIABLE_NAME) +# Returns the lowercase value ("yes" or "no"), or errors if invalid. +pgxntool_validate_yesno = $(strip \ + $(if $(filter yes no,$(shell echo "$(1)" | tr '[:upper:]' '[:lower:]')),\ + $(shell echo "$(1)" | tr '[:upper:]' '[:lower:]'),\ + $(error $(2) must be "yes" or "no", got "$(1)"))) + +# ------------------------------------------------------------------------------ +# test-build: Sanity check extension files before running full test suite +# ------------------------------------------------------------------------------ +# Purpose: Validates that extension SQL files are syntactically correct by running +# files from test/build/ through pg_regress. This provides better error +# messages than CREATE EXTENSION failures. +# +# Variable: PGXNTOOL_ENABLE_TEST_BUILD +# - Can be set manually in Makefile or command line +# - Allowed values: "yes" or "no" (case-insensitive) +# - If not set: Auto-detects based on existence of test/build/*.sql files +# - Set to "yes" explicitly to get an error if test/build/ has no SQL files +# (useful to catch accidental deletion of test/build/ contents) +# - Set to "no" explicitly to disable even when test/build/ has SQL files +# +# Implementation: See test-build target definition (search for "test-build:" in this file) +# +TEST_BUILD_SQL_FILES = $(wildcard $(TESTDIR)/build/*.sql) +TEST_BUILD_FILES = $(TEST_BUILD_SQL_FILES) +ifdef PGXNTOOL_ENABLE_TEST_BUILD + # override needed so command-line values (make VAR=YES) are normalized, not silently ignored. + # := needed for immediate evaluation of the function call (avoids infinite recursion with =). + override PGXNTOOL_ENABLE_TEST_BUILD := $(call pgxntool_validate_yesno,$(PGXNTOOL_ENABLE_TEST_BUILD),PGXNTOOL_ENABLE_TEST_BUILD) +else + # Auto-detect: enable if test/build/ directory has SQL files + ifneq ($(strip $(TEST_BUILD_FILES)),) + PGXNTOOL_ENABLE_TEST_BUILD = yes + else + PGXNTOOL_ENABLE_TEST_BUILD = no + endif +endif + +# ------------------------------------------------------------------------------ +# test/install: Run setup files before all tests in the same pg_regress session +# ------------------------------------------------------------------------------ +# Purpose: Runs files from test/install/ before all test/sql/ files within a +# SINGLE pg_regress invocation via schedule files. This ensures that +# state created by install files (tables, extensions, etc.) persists +# into the main test suite. +# +# Variable: PGXNTOOL_ENABLE_TEST_INSTALL +# - Can be set manually in Makefile or command line +# - Allowed values: "yes" or "no" (case-insensitive) +# - If not set: Auto-detects based on existence of test/install/*.sql files +# - Set to "yes" explicitly to get an error if test/install/ has no SQL files +# (useful to catch accidental deletion of test/install/ contents) +# - Set to "no" explicitly to disable even when test/install/ has SQL files +# +# Directory layout (follows ~/code/extensions/archive/ pattern): +# test/install/*.sql - Install SQL files +# test/install/*.out - Expected output (lives alongside .sql files) +# test/install/schedule - Auto-generated schedule file +# test/sql/schedule - Auto-generated schedule file for regular tests +# +# The schedule files use relative paths (../install/testname) so pg_regress +# resolves install files from their original location without copying. +# +# NOTE: The variable normalization pattern below (ifdef/NORM/error/override) is +# identical to test-build and verify-results. Refactoring options: +# 1. A $(call normalize_bool_var,VAR,DEFAULT) Make function +# 2. A small include fragment (e.g. pgxntool/mk/bool-var.mk) +# Either approach would eliminate the ~10-line block repeated for each feature. +TEST_INSTALL_SQL_FILES = $(wildcard $(TESTDIR)/install/*.sql) +ifdef PGXNTOOL_ENABLE_TEST_INSTALL + # override needed so command-line values (make VAR=YES) are normalized, not silently ignored. + # := needed for immediate evaluation of the function call (avoids infinite recursion with =). + override PGXNTOOL_ENABLE_TEST_INSTALL := $(call pgxntool_validate_yesno,$(PGXNTOOL_ENABLE_TEST_INSTALL),PGXNTOOL_ENABLE_TEST_INSTALL) +else + # Auto-detect: enable if test/install/ directory has SQL files + ifneq ($(strip $(TEST_INSTALL_SQL_FILES)),) + PGXNTOOL_ENABLE_TEST_INSTALL = yes + else + PGXNTOOL_ENABLE_TEST_INSTALL = no + endif +endif + +# ------------------------------------------------------------------------------ +# verify-results: Safeguard for make results +# ------------------------------------------------------------------------------ +# Purpose: Prevents accidentally running 'make results' when tests are failing. +# +# Variable: PGXNTOOL_ENABLE_VERIFY_RESULTS +# - Can be set manually in Makefile or command line +# - Allowed values: "yes" or "no" (case-insensitive) +# - Setting to empty on the command line (e.g. PGXNTOOL_ENABLE_VERIFY_RESULTS=) also disables the feature +# - If not set: Defaults to "yes" (enabled by default for all pgxntool projects) +# - Usage: Controls whether verify-results target exists and blocks make results +# +# Variable: PGXNTOOL_VERIFY_RESULTS_MODE +# - Controls how verify-results detects test failures +# - "pgtap" (default): scans test/results/*.out for "not ok" lines and plan +# mismatches (TAP failures). Also checks regression.diffs as a fallback. +# Use this mode when your test suite uses pgTap. +# - "diffs": checks only for regression.diffs existence (classic pg_regress behavior) +# Use this mode when your tests use plain SQL expected-output comparison only. +# +# Implementation: See verify-results target definition and results target modification +# (search for "verify-results" and "results:" in this file) +# +ifdef PGXNTOOL_ENABLE_VERIFY_RESULTS + override PGXNTOOL_ENABLE_VERIFY_RESULTS := $(call pgxntool_validate_yesno,$(PGXNTOOL_ENABLE_VERIFY_RESULTS),PGXNTOOL_ENABLE_VERIFY_RESULTS) +else + # Default to yes (enabled by default for all pgxntool projects) + PGXNTOOL_ENABLE_VERIFY_RESULTS = yes +endif + +# Default mode: pgtap (scans results/*.out for TAP failures) +PGXNTOOL_VERIFY_RESULTS_MODE ?= pgtap + +# Generate unique database name for tests to prevent conflicts across projects +# Uses project name + first 5 chars of md5 hash of current directory +# This prevents multiple test runs in different directories from clobbering each other +REGRESS_DBHASH := $(shell echo $(CURDIR) | (md5 2>/dev/null || md5sum) | cut -c1-5) +REGRESS_DBNAME := $(or $(PGXN),regression)_$(REGRESS_DBHASH) MODULES = $(patsubst %.c,%,$(wildcard src/*.c)) ifeq ($(strip $(MODULES)),) MODULES =# Set to NUL so PGXS doesn't puke endif -EXTRA_CLEAN = $(wildcard ../$(PGXN)-*.zip) $(EXTENSION_VERSION_FILES) +EXTRA_CLEAN = $(wildcard ../$(PGXN)-*.zip) pg_tle/ # Get Postgres version, as well as major (9.4, etc) version. # NOTE! In at least some versions, PGXS defines VERSION, so we intentionally don't use that variable @@ -59,8 +209,37 @@ ifeq ($(GE91),yes) all: $(EXTENSION_VERSION_FILES) endif -ifeq ($($call test, $(MAJORVER), -lt 13), yes) - REGRESS_OPTS += --load-language=plpgsql +ifeq ($(call test, $(MAJORVER), -lt, 130), yes) +REGRESS_OPTS += --load-language=plpgsql +endif + +# +# test/install: Schedule-based approach +# +# When enabled, generates a schedule file listing install files, and adds it +# to REGRESS_OPTS. pg_regress processes --schedule tests before command-line +# test names, so install files run first in the SAME pg_regress invocation. +# This ensures state created by install files persists into the main test suite. +# +# The schedule uses relative paths (../install/testname) so pg_regress finds +# install files in their original location without copying. +# +ifeq ($(PGXNTOOL_ENABLE_TEST_INSTALL),yes) +PGXNTOOL_INSTALL_SCHEDULE = $(TESTDIR)/install/schedule +EXTRA_CLEAN += $(PGXNTOOL_INSTALL_SCHEDULE) + +# Add install schedule; REGRESS stays as-is (regular tests run after schedule) +REGRESS_OPTS += --schedule=$(PGXNTOOL_INSTALL_SCHEDULE) + +# Always regenerate schedule file to catch added/removed files +.PHONY: $(PGXNTOOL_INSTALL_SCHEDULE) +$(PGXNTOOL_INSTALL_SCHEDULE): + @echo "# Auto-generated - DO NOT EDIT" > $@ + @for f in $(notdir $(basename $(TEST_INSTALL_SQL_FILES))); do \ + echo "test: ../install/$$f" >> $@; \ + done + +installcheck: $(PGXNTOOL_INSTALL_SCHEDULE) endif PGXS := $(shell $(PG_CONFIG) --pgxs) @@ -70,7 +249,7 @@ DATA += $(wildcard *.control) # Don't have installcheck bomb on error .IGNORE: installcheck -installcheck: $(TEST_RESULT_FILES) $(TEST_OUT_FILES) $(TEST_SQL_FILES) $(TEST_SOURCE_FILES) +installcheck: $(TEST_RESULT_FILES) $(TEST_SQL_FILES) | $(TESTDIR)/sql/ $(TESTDIR)/expected/ $(TESTOUT)/results/ # # TEST SUPPORT @@ -84,30 +263,135 @@ installcheck: $(TEST_RESULT_FILES) $(TEST_OUT_FILES) $(TEST_SQL_FILES) $(TEST_SO # watch-make if you're generating intermediate files. If tests end up needing # clean it's an indication of a missing dependency anyway. .PHONY: test -test: testdeps install installcheck +# Build test dependencies list based on enabled features +TEST_DEPS = testdeps +ifeq ($(PGXNTOOL_ENABLE_TEST_BUILD),yes) +TEST_DEPS += test-build +endif +TEST_DEPS += install installcheck +test: $(TEST_DEPS) @if [ -r $(TESTOUT)/regression.diffs ]; then cat $(TESTOUT)/regression.diffs; fi -# make results: runs `make test` and copy all result files to expected +# +# verify-results: Safeguard for make results +# +# Checks if tests are passing before allowing make results to proceed +ifeq ($(PGXNTOOL_ENABLE_VERIFY_RESULTS),yes) +.PHONY: verify-results +ifeq ($(PGXNTOOL_VERIFY_RESULTS_MODE),pgtap) +verify-results: + @$(PGXNTOOL_DIR)/verify-results-pgtap.sh $(TESTOUT) +else +verify-results: + @if [ -r $(TESTOUT)/regression.diffs ]; then \ + echo "ERROR: Tests are failing. Cannot run 'make results'."; \ + echo "Fix test failures first, then run 'make results'."; \ + echo ""; \ + echo "See $(TESTOUT)/regression.diffs for details:"; \ + cat $(TESTOUT)/regression.diffs; \ + exit 1; \ + fi +endif +endif + +# make results: runs `make test` and copies all result files to expected. # DO NOT RUN THIS UNLESS YOU'RE CERTAIN ALL YOUR TESTS ARE PASSING! +# +# Dependency chain (verify-results: test) guarantees test completes before verify-results +# checks regression.diffs, even under make -j. Listing both as independent prerequisites +# of results would allow them to run concurrently, letting verify-results see stale state. .PHONY: results +ifeq ($(PGXNTOOL_ENABLE_VERIFY_RESULTS),yes) +verify-results: test +results: verify-results +else results: test - rsync -rlpgovP $(TESTOUT)/results/ $(TESTDIR)/expected +endif + @mkdir -p $(TESTDIR)/expected + @for f in $(TESTOUT)/results/*.out; do \ + [ -f "$$f" ] || continue; \ + cp "$$f" $(TESTDIR)/expected/$$(basename "$$f"); \ + done # testdeps is a generic dependency target that you can add targets to .PHONY: testdeps testdeps: pgtap +# +# pg_tle support - Generate pg_tle registration SQL +# + +# PGXNTOOL_CONTROL_FILES is defined above (for control.mk dependencies) +PGXNTOOL_EXTENSIONS = $(basename $(PGXNTOOL_CONTROL_FILES)) + +# Main target +# Depend on 'all' to ensure versioned SQL files are generated first +# Depend on control.mk (which defines EXTENSION_VERSION_FILES) +# Depend on control files explicitly so changes trigger rebuilds +# Generates all supported pg_tle versions for each extension +.PHONY: pgtle +pgtle: all control.mk $(PGXNTOOL_CONTROL_FILES) + @$(foreach ext,$(PGXNTOOL_EXTENSIONS),\ + $(PGXNTOOL_DIR)/pgtle.sh --extension $(ext);) + +# +# pg_tle installation support +# + +# Check if pg_tle is installed and report version +# Only reports version if CREATE EXTENSION pg_tle has been run +# Errors if pg_tle extension is not installed +# Uses pgtle.sh to get version (avoids code duplication) +.PHONY: check-pgtle +check-pgtle: + @echo "Checking pg_tle installation..." + @PGTLE_VERSION=$$($(PGXNTOOL_DIR)/pgtle.sh --get-version 2>/dev/null); \ + if [ -n "$$PGTLE_VERSION" ]; then \ + echo "pg_tle extension version: $$PGTLE_VERSION"; \ + exit 0; \ + fi; \ + echo "ERROR: pg_tle extension is not installed" >&2; \ + echo " Run 'CREATE EXTENSION pg_tle;' first" >&2; \ + exit 1 + +# Run pg_tle registration SQL files +# Requires pg_tle extension to be installed (checked via check-pgtle) +# Uses pgtle.sh to determine which version range directory to use +# Assumes PG* environment variables are configured +.PHONY: run-pgtle +run-pgtle: pgtle + @$(PGXNTOOL_DIR)/pgtle.sh --run + # These targets ensure all the relevant directories exist -$(TESTDIR)/sql: - @mkdir -p $@ -$(TESTDIR)/expected/: +$(TESTDIR)/sql $(TESTDIR)/expected/ $(TESTOUT)/results/: @mkdir -p $@ +# pg_regress aborts with "could not open file" if an expected output file is +# missing, so create empty placeholders for any test that lacks one. $(TEST_RESULT_FILES): | $(TESTDIR)/expected/ + @# Create empty expected file so pg_regress doesn't abort with "file not found". + @# pg_regress requires an expected/*.out file to exist for each test; without it + @# it stops immediately rather than running the test and showing the diff. @touch $@ -$(TESTDIR)/output/: - @mkdir -p $@ -$(TEST_OUT_FILES): | $(TESTDIR)/output/ $(TESTDIR)/expected/ $(TESTDIR)/sql/ - @touch $@ + +# +# test-build: Sanity check extension files in test/build/ +# +# The sql/ subdirectory is generated - files are synced from test/build/*.sql. +# This directory should be in .gitignore and is cleaned by make clean. +# +ifeq ($(PGXNTOOL_ENABLE_TEST_BUILD),yes) +TEST_BUILD_SQL_DIR = $(TESTDIR)/build/sql +TEST_BUILD_REGRESS = $(sort $(notdir $(basename $(TEST_BUILD_SQL_FILES)))) +.PHONY: test-build +test-build: install + @$(PGXNTOOL_DIR)/run-test-build.sh $(TESTDIR) + $(MAKE) -C . REGRESS="$(TEST_BUILD_REGRESS)" REGRESS_OPTS="--inputdir=$(TESTDIR)/build --outputdir=$(TESTDIR)/build" installcheck + @if [ -r $(TESTDIR)/build/regression.diffs ]; then \ + echo "test-build failed - see $(TESTDIR)/build/regression.diffs"; \ + cat $(TESTDIR)/build/regression.diffs; \ + exit 1; \ + fi +endif # @@ -137,8 +421,9 @@ dist: html # But don't add it as an install or test dependency unless we do have asciidoc ifneq (,$(strip $(ASCIIDOC))) -# Need to do this so install & co will pick up ALL targets. Unfortunately this can result in some duplication. -DOCS += $(ASCIIDOC_HTML) +# Add HTML to DOCS for install, deduplicating against any HTML already picked +# up by the wildcard (e.g. pre-built HTML committed to the repo). +DOCS := $(sort $(filter-out $(ASCIIDOC_HTML),$(DOCS)) $(ASCIIDOC_HTML)) # Also need to add html as a dep to all (which will get picked up by install & installcheck all: html @@ -155,14 +440,23 @@ docclean: # rmtag: git fetch origin # Update our remotes - @test -z "$$(git branch --list $(PGXNVERSION))" || git branch -d $(PGXNVERSION) - @test -z "$$(git branch --list -r origin/$(PGXNVERSION))" || git push --delete origin $(PGXNVERSION) + @test -z "$$(git tag --list $(PGXNVERSION))" || git tag -d $(PGXNVERSION) + @test -z "$$(git ls-remote --tags origin $(PGXNVERSION) | grep -v '{}')" || git push --delete origin $(PGXNVERSION) -# TODO: Don't puke if tag already exists *and is the same* tag: @test -z "$$(git status --porcelain)" || (echo 'Untracked changes!'; echo; git status; exit 1) - git branch $(PGXNVERSION) - git push --set-upstream origin $(PGXNVERSION) + @# Skip if tag already exists and points to HEAD + @if git rev-parse $(PGXNVERSION) >/dev/null 2>&1; then \ + if [ "$$(git rev-parse $(PGXNVERSION))" = "$$(git rev-parse HEAD)" ]; then \ + echo "Tag $(PGXNVERSION) already exists at HEAD, skipping"; \ + else \ + echo "ERROR: Tag $(PGXNVERSION) exists but points to different commit" >&2; \ + exit 1; \ + fi; \ + else \ + git tag $(PGXNVERSION); \ + fi + git push origin $(PGXNVERSION) .PHONY: forcetag forcetag: rmtag tag @@ -171,6 +465,13 @@ forcetag: rmtag tag dist: tag dist-only dist-only: + @# Check if .gitattributes exists but isn't committed + @if [ -f .gitattributes ] && ! git ls-files --error-unmatch .gitattributes >/dev/null 2>&1; then \ + echo "ERROR: .gitattributes exists but is not committed to git." >&2; \ + echo " git archive only respects export-ignore for committed files." >&2; \ + echo " Please commit .gitattributes for export-ignore to take effect." >&2; \ + exit 1; \ + fi git archive --prefix=$(PGXN)-$(PGXNVERSION)/ -o ../$(PGXN)-$(PGXNVERSION).zip $(PGXNVERSION) .PHONY: forcedist @@ -190,20 +491,33 @@ print-% : ; $(info $* is $(flavor $*) variable set to "$($*)") @true # # subtree sync support # -# This is setup to allow any number of pull targets by defining special -# variables. pgxntool-sync-release is an example of this. -.PHONY: pgxn-sync-% +# All the real work (git subtree pull + update-setup-files.sh) lives in +# pgxntool/pgxntool-sync.sh so it can be run directly, without make. These +# targets are thin wrappers around that script. +# +# `make pgxntool-sync` pulls the latest released version from the canonical +# repository (the script's built-in default). +# +# `make pgxntool-sync-` pulls from the " " defined by the +# pgxntool-sync- variable, allowing any number of custom pull sources. +.PHONY: pgxntool-sync pgxntool-sync-% +pgxntool-sync: + @pgxntool/pgxntool-sync.sh pgxntool-sync-%: - git subtree pull -P pgxntool --squash -m "Pull pgxntool from $($@)" $($@) -pgxntool-sync: pgxntool-sync-release + @pgxntool/pgxntool-sync.sh $($@) # DANGER! Use these with caution. They may add extra crap to your history and # could make resolving merges difficult! -pgxntool-sync-release := git@github.com:decibel/pgxntool.git release -pgxntool-sync-stable := git@github.com:decibel/pgxntool.git stable -pgxntool-sync-local := ../pgxntool release # Not the same as PGXNTOOL_DIR! -pgxntool-sync-local-stable := ../pgxntool stable # Not the same as PGXNTOOL_DIR! - +# `pgxntool-sync` (no suffix) already pulls the canonical release; these are the +# alternatives. `-master` pulls the bleeding edge; `-local*` pull from a sibling +# ../pgxntool checkout (not the same as PGXNTOOL_DIR!). +pgxntool-sync-master := https://github.com/Postgres-Extensions/pgxntool.git master +pgxntool-sync-local := ../pgxntool release +pgxntool-sync-local-master := ../pgxntool master + +# PGXS doesn't provide any special support for distclean (it just depends on +# clean), so we roll our own. Files that should only be removed by distclean +# (not clean) are added to PGXNTOOL_distclean near their build rules above. distclean: rm -f $(PGXNTOOL_distclean) @@ -214,6 +528,22 @@ DOCS =# Set to NUL so PGXS doesn't puke endif include $(PGXS) + +# Override CONTRIB_TESTDB (set unconditionally by PGXS) with our unique database +# name. This must be after include $(PGXS) because PGXS uses = (not ?=). +# PGXS appends --dbname=$(CONTRIB_TESTDB) to REGRESS_OPTS, so overriding +# CONTRIB_TESTDB is the correct way to control the database name — adding our +# own --dbname would result in two --dbname flags passed to pg_regress. +CONTRIB_TESTDB = $(REGRESS_DBNAME) + +# Clean generated sql/ directory for test-build +ifeq ($(PGXNTOOL_ENABLE_TEST_BUILD),yes) +.PHONY: clean-test-build +clean-test-build: + rm -rf $(TEST_BUILD_SQL_DIR) +clean: clean-test-build +endif + # # pgtap # diff --git a/build_meta.sh b/build_meta.sh index 70d2273..d4ea9fa 100755 --- a/build_meta.sh +++ b/build_meta.sh @@ -1,16 +1,28 @@ -#!/bin/bash +#!/usr/bin/env bash + +# Build META.json from META.in.json template +# +# WHY META.in.json EXISTS: +# META.in.json serves as a template that: +# 1. Shows all possible PGXN metadata fields (both required and optional) with comments +# 2. Can have empty placeholder fields like "key": "" or "key": [ "", "" ] +# 3. Users edit this to fill in their extension's metadata +# +# WHY WE GENERATE META.json: +# The reason we generate META.json from a template is to eliminate empty fields that +# are optional; PGXN.org gets upset about them. In the future it's possible we'll do +# more here (for example, if we added more info to the template we could use it to +# generate control files). +# +# WHY WE COMMIT META.json: +# PGXN.org requires META.json to be present in submitted distributions. We choose +# to commit it to git instead of manually adding it to distributions for simplicity +# (and since it generally only changes once for each new version). set -e -error () { - echo $@ >&2 -} -die () { - return=$1 - shift - error $@ - exit $return -} +BASEDIR=$(dirname "$0") +source "$BASEDIR/lib.sh" [ $# -eq 2 ] || die 2 Invalid number of arguments $# diff --git a/control.mk.sh b/control.mk.sh new file mode 100755 index 0000000..cc63cea --- /dev/null +++ b/control.mk.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# +# control.mk.sh - Generate Makefile rules from PostgreSQL extension control files +# +# This script parses .control files to extract extension metadata (particularly +# default_version) and generates Make variables and rules for building versioned +# SQL files. +# +# Usage: control.mk.sh [ ...] +# +# Output (to stdout, meant to be redirected to control.mk): +# EXTENSIONS += +# EXTENSION_SQL_FILES += sql/.sql +# EXTENSION__VERSION := +# EXTENSION__VERSION_FILE = sql/--.sql +# EXTENSION_VERSION_FILES += $(EXTENSION__VERSION_FILE) +# +# +# Why control files instead of META.json? +# META.json's "provides" section specifies versions for PGXN distribution metadata. +# But PostgreSQL uses the control file's default_version to determine which +# versioned SQL file to load. These can differ (e.g., PGXN distribution version +# might be updated independently of extension version). Using the control file +# ensures the generated SQL files match what PostgreSQL expects. + +set -o errexit -o errtrace -o pipefail + +BASEDIR=$(dirname "$0") +source "$BASEDIR/lib.sh" + +# Extract default_version from a PostgreSQL extension control file +# Usage: get_control_default_version +# Errors if: +# - Control file doesn't exist +# - default_version is not specified (pgxntool requires it) +# - Multiple default_version lines exist +get_control_default_version() { + local control_file="$1" + + if [ ! -f "$control_file" ]; then + die 2 "Control file '$control_file' not found" + fi + + # Count default_version lines + local count + count=$(grep -cE "^[[:space:]]*default_version[[:space:]]*=" "$control_file" 2>/dev/null) || count=0 + + if [ "$count" -eq 0 ]; then + die 2 "default_version not specified in '$control_file'. PostgreSQL allows extensions without a default_version, but pgxntool requires it to generate versioned SQL files." + fi + + if [ "$count" -gt 1 ]; then + die 2 "Multiple default_version lines found in '$control_file'. Control files must have exactly one default_version." + fi + + # Extract the version value + # Handles: default_version = '1.0', default_version = "1.0", trailing comments + local version=$(grep -E "^[[:space:]]*default_version[[:space:]]*=" "$control_file" | \ + sed -e "s/^[^=]*=[[:space:]]*//" \ + -e "s/[[:space:]]*#.*//" \ + -e "s/^['\"]//;s/['\"]$//" ) + + if [ -z "$version" ]; then + die 2 "Could not parse default_version value from '$control_file'" + fi + + echo "$version" +} + +# Main: process each control file passed as argument +if [ $# -eq 0 ]; then + die 1 "Usage: control.mk.sh [ ...]" +fi + +for control_file in "$@"; do + ext=$(basename "$control_file" .control) + version=$(get_control_default_version "$control_file") + + echo "EXTENSIONS += $ext" + echo "EXTENSION_SQL_FILES += sql/${ext}.sql" + echo "EXTENSION_${ext}_VERSION := ${version}" + echo "EXTENSION_${ext}_VERSION_FILE = sql/${ext}--\$(EXTENSION_${ext}_VERSION).sql" + echo "EXTENSION_VERSION_FILES += \$(EXTENSION_${ext}_VERSION_FILE)" + echo "\$(EXTENSION_${ext}_VERSION_FILE): sql/${ext}.sql ${control_file}" + echo " @echo '/* DO NOT EDIT - AUTO-GENERATED FILE */' > \$(EXTENSION_${ext}_VERSION_FILE)" + echo " @cat sql/${ext}.sql >> \$(EXTENSION_${ext}_VERSION_FILE)" + echo +done + +# vi: expandtab ts=2 sw=2 diff --git a/lib.sh b/lib.sh new file mode 100644 index 0000000..3cbeb0e --- /dev/null +++ b/lib.sh @@ -0,0 +1,110 @@ +# lib.sh - Common utility functions for pgxntool scripts +# +# This file is meant to be sourced by other scripts, not executed directly. +# Usage: source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +# ============================================================================= +# SETUP FILES CONFIGURATION +# ============================================================================= +# Files copied by setup.sh and tracked by update-setup-files.sh for sync updates. +# Format: "source_in_pgxntool:destination_in_project" +# ============================================================================= +SETUP_FILES=( + "_.gitignore:.gitignore" + "test/deps.sql:test/deps.sql" +) + +# Symlinks created by setup.sh and verified by update-setup-files.sh +# Format: "destination:target" +SETUP_SYMLINKS=( + "test/pgxntool:../pgxntool/test/pgxntool" +) + +# Error function - outputs to stderr but doesn't exit +# Usage: error "message" +error() { + echo "ERROR: $*" >&2 +} + +# Die function - outputs error message and exits with specified code +# Usage: die EXIT_CODE "message" +die() { + local exit_code=$1 + shift + error "$@" + exit $exit_code +} + +# Returns true if an array isn't empty. +# +# array_not_empty "${#errors[@]}" +# +# BUT WHY ON EARTH DO THIS?? +# +# This wraps a one-liner intentionally. The function forces any reader +# (human or AI agent) to navigate here and read this comment before +# "simplifying" the call site. Without it, the natural next step is to +# inline the expression — and the natural inline form breaks bash 3.2. +# +# On bash 3.2 (Mac OS default), when using `set -u`, expanding "${arr[@]}" +# on an empty array triggers "unbound variable" even when the array was +# explicitly initialized with arr=(). +# +# The comment inside the function body exists to catch any agent or human who +# navigates to the function without reading this comment first. +array_not_empty() { + # DO NOT EDIT THIS FUNCTION! DO NOT REMOVE THIS COMMENT! (see main function comment) + [ "${1:-0}" -gt 0 ] +} + +# Debug function +# Usage: debug LEVEL "message" +# Outputs message to stderr if DEBUG >= LEVEL +# +# LEVEL encodes how noisy/esoteric a message is -- roughly, how far you'd crank +# DEBUG before you'd actually want to see it. Higher = noisier, more rarely +# useful. This is signal-to-noise, NOT code nesting depth: a top-level line can +# warrant a high level if it's esoteric, and loop-body detail is usually high +# precisely because it's noisy. +# +# The tiers below are anchors, not strict multiples -- pick any value in range +# to fine-tune between existing calls without renumbering: +# - 10: Critical errors, important warnings +# - 20: Warnings, significant state changes +# - 30: General debugging, function entry/exit, array operations +# - 40: Verbose details, loop iterations +# - 50+: Maximum verbosity (per-iteration innards) +# +# Enable with: DEBUG=30 scriptname.sh +debug() { + local level=$1 + shift + local message="$*" + + if [ "${DEBUG:-0}" -ge "$level" ]; then + echo "DEBUG[$level]: $message" >&2 + fi +} + +# Remove pgxntool's own dev-only directories from a consuming project. +# +# `git subtree` copies the ENTIRE pgxntool tree into the consumer, including +# dev-only dirs like .github/ (pgxntool's CI) and .claude/. Those are +# export-ignored from `make dist` and don't belong in a project that merely +# embeds pgxntool. (GitHub only runs workflows at the repo root, so a consumer's +# pgxntool/.github never executes anyway — but it's still clutter.) git subtree +# doesn't honor export-ignore, so we prune them here after a sync. +# +# Must be run from the project root (the dir containing pgxntool/). Safe to call +# repeatedly; a no-op once the dirs are gone. +prune_pgxntool_dev_dirs() { + local d + for d in .github .claude; do + [ -e "pgxntool/$d" ] || continue + echo " pgxntool/$d: pruning (pgxntool dev-only, not for embedding projects)" + # Stage the removal if tracked; rm -rf guarantees it's gone even if not. + # || : keeps this best-effort under `set -e` (rm -rf is the real cleanup). + git rm -rq --ignore-unmatch "pgxntool/$d" >/dev/null 2>&1 || : + rm -rf "pgxntool/$d" + done +} diff --git a/meta.mk.sh b/meta.mk.sh index a5da2ec..e6cecc5 100755 --- a/meta.mk.sh +++ b/meta.mk.sh @@ -1,25 +1,38 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash +# +# meta.mk.sh - Generate Makefile variables from META.json +# +# This script parses META.json (PGXN distribution metadata) and generates +# Make variables for the distribution name and version. +# +# Usage: meta.mk.sh +# +# Output (to stdout, meant to be redirected to meta.mk): +# PGXN := +# PGXNVERSION := +# +# Note: Extension-specific variables (like EXTENSION_*_VERSION) are generated +# by control.mk.sh from .control files, not from META.json. This is because +# META.json specifies PGXN distribution metadata, while .control files specify +# what PostgreSQL actually uses. set -o errexit -o errtrace -o pipefail -trap 'echo "Error on line ${LINENO}" >&2' ERR -META=$1 -BASEDIR=`dirname $0` +BASEDIR=$(dirname "$0") +source "$BASEDIR/lib.sh" + JSON_SH=$BASEDIR/JSON.sh -error () { - echo $@ >&2 -} trap 'error "Error on line ${LINENO}"' ERR -die () { - local retval=$1 - shift - error $@ - exit $retval -} +META=$1 +if [ -z "$META" ]; then + die 1 "Usage: meta.mk.sh " +fi -REQUIRED='abstract maintainer license provides name version' +if [ ! -f "$META" ]; then + die 2 "META.json file '$META' not found" +fi #function to get value of specified key #returns empty string if not found @@ -27,7 +40,7 @@ REQUIRED='abstract maintainer license provides name version' #usage: VAR=$(getkey foo.bar) #get value of "bar" contained within "foo" # VAR=$(getkey foo[4].bar) #get value of "bar" contained in the array "foo" on position 4 # VAR=$(getkey [4].foo) #get value of "foo" contained in the root unnamed array on position 4 -function _getkey { +_getkey() { #reformat key string (parameter) to what JSON.sh uses KEYSTRING=$(sed -e 's/\[/\"\,/g' -e 's/^\"\,/\[/g' -e 's/\]\./\,\"/g' -e 's/\./\"\,\"/g' -e '/^\[/! s/^/\[\"/g' -e '/\]$/! s/$/\"\]/g' <<< "$@") #extract the key value @@ -37,60 +50,21 @@ function _getkey { FOUT="${FOUT%\"*}" echo "$FOUT" } -function getkeys { - KEYSTRING=$(sed -e 's/\[/\"\,/g' -e 's/^\"\,/\[/g' -e 's/\]\./\,\"/g' -e 's/\./\"\,\"/g' -e '/^\[/! s/^/\[\"/g' -e '/\",\"$/! s/$/\",\"/g' <<< "$@") - #extract the key value - FOUT=$(grep -F "$KEYSTRING" <<< "$JSON_PARSED") - FOUT="${FOUT%$'\t'*}" - echo "$FOUT" -} - -#function returning length of array -#returns zero if key in parameter does not exist or is not an array -#usage: VAR=$(getarrlen foo.bar) #get length of array "bar" contained within "foo" -# VAR=$(getarrlen) #get length of the root unnamed array -# VAR=$(getarrlen [2].foo.bar) #get length of array "bar" contained within "foo", which is stored in the root unnamed array on position 2 -function getarrlen { - #reformat key string (parameter) to what JSON.sh uses - KEYSTRING=$(gsed -e '/^\[/! s/\[/\"\,/g' -e 's/\]\./\,\"/g' -e 's/\./\"\,\"/g' -e '/^$/! {/^\[/! s/^/\[\"/g}' -e '/^$/! s/$/\"\,/g' -e 's/\[/\\\[/g' -e 's/\]/\\\]/g' -e 's/\,/\\\,/g' -e '/^$/ s/^/\\\[/g' <<< "$@") - #extract the key array length - get last index - LEN=$(grep -o "${KEYSTRING}[0-9]*" <<< "$JSON_PARSED" | tail -n -1 | grep -o "[0-9]*$") - #increment to get length, if empty => zero - if [ -n "$LEN" ]; then - LEN=$(($LEN+1)) - else - LEN="0" - fi - echo "$LEN" -} -JSON_PARSED=$(cat $META | $JSON_SH -l) - -function getkey { +getkey() { out=$(_getkey "$@") [ -n "$out" ] || die 2 "key $@ not found in $META" echo $out } -# Handle meta-spec specially -spec_version=`getkey meta-spec.version` -[ "$spec_version" == "1.0.0" ] || die 2 "Unknown meta-spec/version: $PGXN_meta-spec_version" +JSON_PARSED=$(cat "$META" | $JSON_SH -l) + +# Validate meta-spec version +spec_version=$(getkey meta-spec.version) +[ "$spec_version" == "1.0.0" ] || die 2 "Unknown meta-spec/version: $spec_version" +# Output distribution name and version echo "PGXN := $(getkey name)" echo "PGXNVERSION := $(getkey version)" -echo - -provides=$(getkeys provides | sed -e 's/\["provides","//' -e 's/",".*//' | uniq) -for ext in $provides; do - version=$(getkey provides.${ext}.version) - [ -n "$version" ] || die 2 "provides/${ext} does not specify a version number" - echo "EXTENSIONS += $ext" - echo "EXTENSION_SQL_FILES += sql/${ext}.sql" - echo "EXTENSION_${ext}_VERSION := ${version}" - echo "EXTENSION_${ext}_VERSION_FILE = sql/${ext}--\$(EXTENSION_${ext}_VERSION).sql" - echo "EXTENSION_VERSION_FILES += \$(EXTENSION_${ext}_VERSION_FILE)" - echo "\$(EXTENSION_${ext}_VERSION_FILE): sql/${ext}.sql META.json meta.mk" - echo ' cp $< $@' -done # vi: expandtab ts=2 sw=2 diff --git a/pgtle.sh b/pgtle.sh new file mode 100755 index 0000000..124ca84 --- /dev/null +++ b/pgtle.sh @@ -0,0 +1,858 @@ +#!/usr/bin/env bash +# +# pgtle.sh - Generate pg_tle registration SQL for PostgreSQL extensions +# +# Part of pgxntool: https://github.com/decibel/pgxntool +# +# SYNOPSIS +# pgtle.sh --extension EXTNAME [--pgtle-version VERSION] +# pgtle.sh --get-dir VERSION +# pgtle.sh --get-version +# pgtle.sh --run +# +# DESCRIPTION +# Generates pg_tle (Trusted Language Extensions) registration SQL from +# a pgxntool-based PostgreSQL extension. Reads the extension's .control +# file and SQL files, wrapping them for pg_tle deployment in managed +# environments like AWS RDS and Aurora. +# +# pg_tle enables extension installation without filesystem access by +# storing extension code in database tables. This script converts +# traditional PostgreSQL extensions into pg_tle-compatible SQL. +# +# OPTIONS +# --extension NAME +# Extension name (required). Must match a .control file basename +# in the current directory. +# +# --pgtle-version VERSION +# Generate for specific pg_tle version only (optional). +# Format: 1.0.0-1.4.0, 1.4.0-1.5.0, or 1.5.0+ +# Default: Generate all supported versions +# +# --get-dir VERSION +# Returns the directory path for the given pg_tle version. +# Format: VERSION is a version string like "1.5.2" +# Output: Directory path like "pg_tle/1.5.0+", "pg_tle/1.4.0-1.5.0", or "pg_tle/1.0.0-1.4.0" +# This option is used by make to determine which directory to use +# +# --get-version +# Returns the installed pg_tle version from the database. +# Output: Version string like "1.5.2" or empty if not installed +# Exit status: 0 if pg_tle is installed, 1 if not installed +# +# --run +# Runs the generated pg_tle registration SQL files. This option: +# - Detects the installed pg_tle version from the database +# - Determines the appropriate directory using --get-dir logic +# - Executes all SQL files in that directory via psql +# - Assumes PG* environment variables are configured for psql +# +# VERSION NOTATION +# X.Y.Z+ Works on pg_tle >= X.Y.Z +# X.Y.Z-A.B.C Works on pg_tle >= X.Y.Z and < A.B.C +# +# Note the boundary conditions: +# 1.5.0+ means >= 1.5.0 (includes 1.5.0) +# 1.4.0-1.5.0 means >= 1.4.0 and < 1.5.0 (excludes 1.5.0) +# 1.0.0-1.4.0 means >= 1.0.0 and < 1.4.0 (excludes 1.4.0) +# +# SUPPORTED VERSIONS +# 1.0.0-1.4.0 pg_tle 1.0.0 through 1.3.x (no uninstall function, no schema parameter) +# 1.4.0-1.5.0 pg_tle 1.4.0 through 1.4.x (has uninstall function, no schema parameter) +# 1.5.0+ pg_tle 1.5.0 and later (has uninstall function, schema parameter support) +# +# EXAMPLES +# # Generate all versions (default) +# pgtle.sh --extension myext +# +# # Generate only for pg_tle 1.5+ +# pgtle.sh --extension myext --pgtle-version 1.5.0+ +# +# # Get directory for a specific pg_tle version +# pgtle.sh --get-dir 1.5.2 +# # Output: pg_tle/1.5.0+ +# +# pgtle.sh --get-dir 1.4.2 +# # Output: pg_tle/1.4.0-1.5.0 +# +# # Get installed pg_tle version from database +# pgtle.sh --get-version +# # Output: 1.5.2 (or empty if not installed) +# +# # Run generated pg_tle registration SQL files +# pgtle.sh --run +# +# OUTPUT +# Creates files in version-specific subdirectories: +# pg_tle/1.0.0-1.4.0/{extension}.sql +# pg_tle/1.4.0-1.5.0/{extension}.sql +# pg_tle/1.5.0+/{extension}.sql +# +# Each file contains: +# - All versions of the extension +# - All upgrade paths between versions +# - Default version configuration +# - Complete installation instructions +# +# For --get-dir: Outputs the directory path to stdout. +# +# For --get-version: Outputs the installed pg_tle version to stdout, or empty if not installed. +# +# For --run: Executes SQL files and outputs progress messages to stderr. +# +# REQUIREMENTS +# - Must run from extension directory (where .control files are) +# - Extension must use only trusted languages (PL/pgSQL, SQL, PL/Perl, etc.) +# - No C code (module_pathname not supported by pg_tle) +# - Versioned SQL files must exist: sql/{ext}--{version}.sql +# +# EXIT STATUS +# 0 Success +# 1 Error (missing files, validation failure, C code detected, etc.) +# +# SEE ALSO +# pgxntool/README-pgtle.md - Complete user guide +# https://github.com/aws/pg_tle - pg_tle documentation +# + +set -eo pipefail + +# Source common library functions (error, die, debug) +PGXNTOOL_DIR="$(dirname "${BASH_SOURCE[0]}")" +source "$PGXNTOOL_DIR/lib.sh" + +# Constants +PGTLE_DELIMITER='$_pgtle_wrap_delimiter_$' +PGTLE_VERSIONS=("1.0.0-1.4.0" "1.4.0-1.5.0" "1.5.0+") + +# Supported pg_tle version ranges and their capabilities +# Use a function instead of associative array for compatibility with bash < 4.0 +get_pgtle_capability() { + local version="$1" + case "$version" in + "1.0.0-1.4.0") + echo "no_uninstall_no_schema" + ;; + "1.4.0-1.5.0") + echo "has_uninstall_no_schema" + ;; + "1.5.0+") + echo "has_uninstall_has_schema" + ;; + *) + echo "unknown" + ;; + esac +} + +# Global variables (populated from control file) +EXTENSION="" +DEFAULT_VERSION="" +COMMENT="" +REQUIRES="" +SCHEMA="" +MODULE_PATHNAME="" +VERSION_FILES=() +UPGRADE_FILES=() + +debug 30 "Global arrays initialized: VERSION_FILES=${#VERSION_FILES[@]}, UPGRADE_FILES=${#UPGRADE_FILES[@]}" +PGTLE_VERSION="" # Empty = generate all +GET_DIR_VERSION="" # For --get-dir option + +# Arrays (populated from SQL discovery) +VERSION_FILES=() +UPGRADE_FILES=() + +# Parse and validate a version string +# Extracts numeric version (major.minor.patch) from version strings +# Handles versions with suffixes like "1.5.0alpha1", "2.0beta", "1.2.3dev" +# Returns: numeric version string (e.g., "1.5.0") or exits with error +parse_version() { + local version="$1" + + if [ -z "$version" ]; then + die 1 "Version string is empty" + fi + + # Extract numeric version part (major.minor.patch) + # Matches: 1.5.0, 1.5, 10.2.1alpha, 2.0beta1, etc. + # Pattern: start of string, then digits, dot, digits, optionally (dot digits), then anything + local numeric_version + if [[ "$version" =~ ^([0-9]+\.[0-9]+(\.[0-9]+)?) ]]; then + numeric_version="${BASH_REMATCH[1]}" + else + die 1 "Cannot parse version string: '$version' + Expected format: major.minor[.patch][suffix] + Examples: 1.5.0, 1.5, 2.0alpha1, 10.2.3dev" + fi + + # Ensure we have at least major.minor (add .0 if needed) + if [[ ! "$numeric_version" =~ \. ]]; then + die 1 "Invalid version format: '$version' (need at least major.minor)" + fi + + # If we only have major.minor, add .0 for patch + if [[ ! "$numeric_version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + numeric_version="${numeric_version}.0" + fi + + echo "$numeric_version" +} + +# Convert version string to comparable integer +# Takes a numeric version string (major.minor.patch) and converts to integer +# Example: "1.5.0" -> 1005000 +# Encoding scheme: major * 1000000 + minor * 1000 + patch +# This limits each component to 0-999 to prevent overflow +version_to_number() { + local version="$1" + + # Parse major.minor.patch + local major minor patch + if [[ "$version" =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then + major="${BASH_REMATCH[1]}" + minor="${BASH_REMATCH[2]}" + patch="${BASH_REMATCH[3]}" + else + die 1 "version_to_number: Invalid numeric version format: '$version'" + fi + + # Check for overflow in encoding scheme + # Each component must be < 1000 to fit in the allocated space + if [ "$major" -ge 1000 ]; then + die 1 "version_to_number: Major version too large: $major (max 999) + Version: $version" + fi + if [ "$minor" -ge 1000 ]; then + die 1 "version_to_number: Minor version too large: $minor (max 999) + Version: $version" + fi + if [ "$patch" -ge 1000 ]; then + die 1 "version_to_number: Patch version too large: $patch (max 999) + Version: $version" + fi + + # Convert to comparable number: major * 1000000 + minor * 1000 + patch + echo $(( major * 1000000 + minor * 1000 + patch )) +} + +# Get directory for a given pg_tle version +# Takes a version string like "1.5.2" and returns the directory path +# Handles versions with suffixes (e.g., "1.5.0alpha1") +# Returns: "pg_tle/1.0.0-1.4.0", "pg_tle/1.4.0-1.5.0", or "pg_tle/1.5.0+" +get_version_dir() { + local version="$1" + + if [ -z "$version" ]; then + die 1 "Version required for --get-dir (got empty string)" + fi + + # Parse and validate version + local numeric_version + numeric_version=$(parse_version "$version") + + # Check if the original version has a pre-release suffix + # Pre-release versions (alpha, beta, rc, dev) are considered BEFORE the release + # Example: 1.4.0alpha1 comes BEFORE 1.4.0, so it should use the 1.0.0-1.4.0 range + local has_prerelease=0 + if [[ "$version" =~ (alpha|beta|rc|dev) ]]; then + has_prerelease=1 + fi + + # Convert versions to comparable numbers + local version_num + local threshold_1_4_num + local threshold_1_5_num + version_num=$(version_to_number "$numeric_version") + threshold_1_4_num=$(version_to_number "1.4.0") + threshold_1_5_num=$(version_to_number "1.5.0") + + # Compare and return appropriate directory: + # < 1.4.0 -> 1.0.0-1.4.0 + # >= 1.4.0 and < 1.5.0 -> 1.4.0-1.5.0 + # >= 1.5.0 -> 1.5.0+ + # + # Special handling for pre-release versions: + # If version equals a threshold but has a pre-release suffix, treat it as less than that threshold + # Example: 1.4.0alpha1 is treated as < 1.4.0, so it uses 1.0.0-1.4.0 + if [ "$version_num" -lt "$threshold_1_4_num" ]; then + echo "pg_tle/1.0.0-1.4.0" + elif [ "$version_num" -eq "$threshold_1_4_num" ] && [ "$has_prerelease" -eq 1 ]; then + # Pre-release of 1.4.0 is considered < 1.4.0 + echo "pg_tle/1.0.0-1.4.0" + elif [ "$version_num" -lt "$threshold_1_5_num" ]; then + echo "pg_tle/1.4.0-1.5.0" + elif [ "$version_num" -eq "$threshold_1_5_num" ] && [ "$has_prerelease" -eq 1 ]; then + # Pre-release of 1.5.0 is considered < 1.5.0 + echo "pg_tle/1.4.0-1.5.0" + else + echo "pg_tle/1.5.0+" + fi +} + +# Get pg_tle version from installed extension +# Returns version string or empty if not installed +get_pgtle_version() { + psql --no-psqlrc --tuples-only --no-align --command "SELECT extversion FROM pg_extension WHERE extname = 'pg_tle';" 2>/dev/null | tr -d '[:space:]' || echo "" +} + +# Run pg_tle registration SQL files +# Detects installed pg_tle version and runs appropriate SQL files +run_pgtle_sql() { + echo "Running pg_tle registration SQL files..." >&2 + + # Get version from installed extension + local pgtle_version=$(get_pgtle_version) + if [ -z "$pgtle_version" ]; then + die 1 "pg_tle extension is not installed + Run 'CREATE EXTENSION pg_tle;' first, or use 'make check-pgtle' to verify" + fi + + # Get directory for this version + local pgtle_dir=$(get_version_dir "$pgtle_version") + if [ -z "$pgtle_dir" ]; then + die 1 "Failed to determine pg_tle directory for version $pgtle_version" + fi + + echo "Using pg_tle files for version $pgtle_version (directory: $pgtle_dir)" >&2 + + # Check if directory exists + if [ ! -d "$pgtle_dir" ]; then + die 1 "pg_tle directory $pgtle_dir does not exist + Run 'make pgtle' first to generate files" + fi + + # Run all SQL files in the directory + local sql_file + local found=0 + for sql_file in "$pgtle_dir"/*.sql; do + if [ -f "$sql_file" ]; then + found=1 + echo "Running $sql_file..." >&2 + psql --no-psqlrc -v ON_ERROR_STOP=1 --file="$sql_file" || exit 1 + fi + done + + if [ "$found" -eq 0 ]; then + die 1 "No SQL files found in $pgtle_dir + Run 'make pgtle' first to generate files" + fi + + echo "pg_tle registration complete" >&2 +} + +# Main logic +main() { + # Handle --get-dir, --get-version, --test-function, and --run options first (early exit, before other validation) + local args=("$@") + local i=0 + while [ $i -lt ${#args[@]} ]; do + if [ "${args[$i]}" = "--get-dir" ] && [ $((i+1)) -lt ${#args[@]} ]; then + get_version_dir "${args[$((i+1))]}" + exit 0 + elif [ "${args[$i]}" = "--get-version" ]; then + local version=$(get_pgtle_version) + if [ -n "$version" ]; then + echo "$version" + exit 0 + else + exit 1 + fi + elif [ "${args[$i]}" = "--test-function" ] && [ $((i+1)) -lt ${#args[@]} ]; then + # Hidden option for testing internal functions + # NOT a supported public interface - used only by the test suite + # Usage: pgtle.sh --test-function FUNC_NAME [ARGS...] + local func_name="${args[$((i+1))]}" + shift $((i+2)) # Remove script name and --test-function and func_name + + # Check if function exists + if ! declare -f "$func_name" >/dev/null 2>&1; then + die 1 "Function '$func_name' does not exist" + fi + + # Call the function with remaining arguments + "$func_name" "${args[@]:$((i+2))}" + exit $? + elif [ "${args[$i]}" = "--run" ]; then + run_pgtle_sql + exit 0 + fi + i=$((i+1)) + done + + # Parse other arguments + parse_args "$@" + + validate_environment + parse_control_file + discover_sql_files + + if [ -z "$PGTLE_VERSION" ]; then + # Generate all versions + for version in "${PGTLE_VERSIONS[@]}"; do + generate_pgtle_sql "$version" + done + else + # Generate specific version + generate_pgtle_sql "$PGTLE_VERSION" + fi +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case $1 in + --extension) + EXTENSION="$2" + shift 2 + ;; + --pgtle-version) + PGTLE_VERSION="$2" + shift 2 + ;; + --get-dir) # This case should ideally not be hit due to early exit + GET_DIR_VERSION="$2" + shift 2 + ;; + --get-version) # This case should ideally not be hit due to early exit + shift + ;; + --test-function) # Hidden option for testing - not documented, not supported + shift 2 # Skip function name and --test-function + ;; + --run) # This case should ideally not be hit due to early exit + shift + ;; + *) + echo "Unknown option: $1" >&2 + exit 1 + ;; + esac + done + + if [ -z "$EXTENSION" ] && [ -z "$GET_DIR_VERSION" ]; then + die 1 "--extension is required (unless using --get-dir, --get-version, --test-function, or --run)" + fi +} + +validate_environment() { + # Check if control file exists + if [ ! -f "${EXTENSION}.control" ]; then + die 1 "Control file not found: ${EXTENSION}.control + Must run from extension directory" + fi +} + +parse_control_file() { + local control_file="${EXTENSION}.control" + + echo "Parsing control file: $control_file" >&2 + + # Parse key = value or key = 'value' format + while IFS= read -r line; do + # Skip comments and empty lines + [[ "$line" =~ ^[[:space:]]*# ]] && continue + [[ "$line" =~ ^[[:space:]]*$ ]] && continue + + # Extract key = value + if [[ "$line" =~ ^[[:space:]]*([a-z_]+)[[:space:]]*=[[:space:]]*(.*)[[:space:]]*$ ]]; then + local key="${BASH_REMATCH[1]}" + local value="${BASH_REMATCH[2]}" + + # Trim trailing comments and whitespace FIRST, then strip quotes. + # Order matters: stripping quotes before removing comments leaves a rogue + # trailing quote for values like 'version' # note. See issue #25. + value="${value%%#*}" # Remove trailing comments + # Trim all trailing whitespace (spaces and tabs). The prior %% pattern with + # a literal space only removed one character; multiple spaces or a tab before + # the comment (e.g., 'value' # note or 'value'$'\t'# note) would leave + # stray whitespace that breaks the quote-strip below. + if [[ "$value" =~ ^(.*[^[:space:]])[[:space:]]*$ ]]; then + value="${BASH_REMATCH[1]}" + else + value="" + fi + + # Strip quotes if present (both single and double) + value="${value#\'}" + value="${value%\'}" + value="${value#\"}" + value="${value%\"}" + + # Store in global variables + case "$key" in + default_version) DEFAULT_VERSION="$value" ;; + comment) COMMENT="$value" ;; + requires) REQUIRES="$value" ;; + schema) SCHEMA="$value" ;; + module_pathname) MODULE_PATHNAME="$value" ;; + esac + fi + done < "$control_file" + + # Validate required fields + if [ -z "$DEFAULT_VERSION" ]; then + die 1 "Control file missing default_version" + fi + + if [ -z "$COMMENT" ]; then + echo "WARNING: Control file missing comment, using extension name" >&2 + COMMENT="$EXTENSION extension" + fi + + # Warn about C code + if [ -n "$MODULE_PATHNAME" ]; then + cat >&2 <<-EOF + WARNING: Extension uses module_pathname (C code) + pg_tle only supports trusted languages (PL/pgSQL, SQL, etc.) + Generated SQL will likely not work + EOF + fi + + echo " default_version: $DEFAULT_VERSION" >&2 + echo " comment: $COMMENT" >&2 + if [ -n "$REQUIRES" ]; then + echo " requires: $REQUIRES" >&2 + fi + if [ -n "$SCHEMA" ]; then + echo " schema: $SCHEMA" >&2 + fi +} + +discover_sql_files() { + echo "Discovering SQL files for extension: $EXTENSION" >&2 + debug 30 "discover_sql_files: Starting discovery for extension: $EXTENSION" + + # Ensure default_version file exists and has content if base file exists + # This handles the case where make all hasn't generated it yet, or it exists but is empty + local default_version_file="sql/${EXTENSION}--${DEFAULT_VERSION}.sql" + local base_file="sql/${EXTENSION}.sql" + if [ -f "$base_file" ] && ([ ! -f "$default_version_file" ] || [ ! -s "$default_version_file" ]); then + debug 40 "discover_sql_files: Creating default_version file from base file" + cp "$base_file" "$default_version_file" + fi + + # Find versioned files: sql/{ext}--{version}.sql + # Use find to get proper null-delimited output, then filter out upgrade scripts + VERSION_FILES=() # Reset array + debug 40 "discover_sql_files: Reset VERSION_FILES array" + while IFS= read -r -d '' file; do + local basename=$(basename "$file" .sql) + local dash_count=$(echo "$basename" | grep -o -- "--" | wc -l | tr -d '[:space:]') + # Skip upgrade scripts (they have 2 dashes) + if [ "$dash_count" -ne 1 ]; then + continue + fi + # Error on empty version files + if [ ! -s "$file" ]; then + die 1 "Empty version file found: $file" + fi + VERSION_FILES+=("$file") + done < <(find sql/ -maxdepth 1 -name "${EXTENSION}--*.sql" -print0 2>/dev/null | sort -zV) + + # Find upgrade scripts: sql/{ext}--{ver1}--{ver2}.sql + # These have TWO occurrences of "--" in the filename + UPGRADE_FILES=() # Reset array + debug 40 "discover_sql_files: Reset UPGRADE_FILES array" + while IFS= read -r -d '' file; do + # Empty upgrade files are allowed (no-op upgrades) + local basename=$(basename "$file" .sql) + local dash_count=$(echo "$basename" | grep -o -- "--" | wc -l | tr -d '[:space:]') + if [ "$dash_count" -eq 2 ]; then + UPGRADE_FILES+=("$file") + fi + done < <(find sql/ -maxdepth 1 -name "${EXTENSION}--*--*.sql" -print0 2>/dev/null | sort -zV) + + if [ ${#VERSION_FILES[@]} -eq 0 ]; then + die 1 "No versioned SQL files found for $EXTENSION + Expected pattern: sql/${EXTENSION}--{version}.sql + Run 'make' first to generate versioned files from sql/${EXTENSION}.sql" + fi + + echo " Found ${#VERSION_FILES[@]} version file(s):" >&2 + for f in "${VERSION_FILES[@]}"; do + echo " - $f" >&2 + done + + debug 40 "discover_sql_files: Checking UPGRADE_FILES array, count=${#UPGRADE_FILES[@]}" + if array_not_empty "${#UPGRADE_FILES[@]}"; then + echo " Found ${#UPGRADE_FILES[@]} upgrade script(s):" >&2 + debug 30 "discover_sql_files: Iterating over ${#UPGRADE_FILES[@]} upgrade files" + for f in "${UPGRADE_FILES[@]}"; do + echo " - $f" >&2 + done + else + debug 30 "discover_sql_files: No upgrade files found" + fi +} + +extract_version_from_filename() { + local filename="$1" + local basename=$(basename "$filename" .sql) + + # Match patterns: + # - ext--1.0.0 → FROM_VERSION=1.0.0, TO_VERSION="" + # - ext--1.0.0--2.0.0 → FROM_VERSION=1.0.0, TO_VERSION=2.0.0 + + if [[ "$basename" =~ ^${EXTENSION}--([0-9][0-9.]*)(--([0-9][0-9.]*))?$ ]]; then + FROM_VERSION="${BASH_REMATCH[1]}" + TO_VERSION="${BASH_REMATCH[3]}" # Empty for non-upgrade files + return 0 + else + die 1 "Cannot parse version from filename: $filename + Expected format: ${EXTENSION}--{version}.sql or ${EXTENSION}--{ver1}--{ver2}.sql" + fi +} + +validate_delimiter() { + local sql_file="$1" + + if grep -qF "$PGTLE_DELIMITER" "$sql_file"; then + die 1 "SQL file contains reserved pg_tle delimiter: $sql_file + Found: $PGTLE_DELIMITER + This delimiter is used internally by pgtle.sh to wrap SQL content. + You must modify your SQL to not contain this string. If this poses a + serious problem, please open an issue at https://github.com/decibel/pgxntool/issues" + fi +} + +wrap_sql_content() { + local sql_file="$1" + + validate_delimiter "$sql_file" + + # Output wrapped SQL with proper indentation + # Empty files are valid (no-op upgrades) + echo " ${PGTLE_DELIMITER}" + cat "$sql_file" + echo " ${PGTLE_DELIMITER}" +} + +build_requires_array() { + # Input: "plpgsql, other_ext, another" + # Output: 'plpgsql', 'other_ext', 'another' + + # Split on comma, trim whitespace, quote each element + REQUIRES_ARRAY=$(echo "$REQUIRES" | \ + sed 's/[[:space:]]*,[[:space:]]*/\n/g' | \ + sed "s/^[[:space:]]*//;s/[[:space:]]*$//" | \ + sed "s/^/'/;s/$/'/" | \ + paste -sd, -) +} + +generate_header() { + local pgtle_version="$1" + local output_file="$2" + local version_count=${#VERSION_FILES[@]} + local upgrade_count=${#UPGRADE_FILES[@]} + + # Determine version compatibility message + local compat_msg + if [[ "$pgtle_version" == *"+"* ]]; then + local base_version="${pgtle_version%+}" + compat_msg="-- Works on pg_tle >= ${base_version}" + else + local min_version="${pgtle_version%-*}" + local max_version="${pgtle_version#*-}" + compat_msg="-- Works on pg_tle >= ${min_version} and < ${max_version}" + fi + + cat < $to_ver" + echo "SELECT pgtle.install_update_path(" + echo " '${EXTENSION}'," + echo " '${from_ver}'," + echo " '${to_ver}'," + wrap_sql_content "$upgrade_file" + echo ");" + echo +} + +generate_pgtle_sql() { + local pgtle_version="$1" + debug 30 "generate_pgtle_sql: Starting for version $pgtle_version, extension $EXTENSION" + + # Get capability using function (compatible with bash < 4.0) + local capability=$(get_pgtle_capability "$pgtle_version") + local version_dir="pg_tle/${pgtle_version}" + local output_file="${version_dir}/${EXTENSION}.sql" + + # Ensure arrays are initialized (defensive programming) + # Arrays should already be initialized at top level, but ensure they exist + debug 40 "generate_pgtle_sql: Checking array initialization" + debug 30 "generate_pgtle_sql: VERSION_FILES is ${VERSION_FILES+set}, count=${#VERSION_FILES[@]}" + debug 30 "generate_pgtle_sql: UPGRADE_FILES is ${UPGRADE_FILES+set}, count=${#UPGRADE_FILES[@]}" + + if [ -z "${VERSION_FILES+set}" ]; then + echo "WARNING: VERSION_FILES not set, initializing" >&2 + VERSION_FILES=() + fi + if [ -z "${UPGRADE_FILES+set}" ]; then + echo "WARNING: UPGRADE_FILES not set, initializing" >&2 + UPGRADE_FILES=() + fi + + # Create version-specific output directory if needed + mkdir -p "$version_dir" + + echo "Generating: $output_file (pg_tle $pgtle_version)" >&2 + + # Generate SQL to file + { + generate_header "$pgtle_version" "$output_file" + + cat < "$output_file" + + echo " ✓ Generated: $output_file" >&2 +} + +main "$@" + diff --git a/pgtle_versions.md b/pgtle_versions.md new file mode 100644 index 0000000..d2c5c03 --- /dev/null +++ b/pgtle_versions.md @@ -0,0 +1,47 @@ +# pg_tle Version Support Matrix + +This file documents pg_tle version boundaries that affect pgxntool's pg_tle support code. Each boundary represents a backward-incompatible API change. + +## Version Ranges (pgxntool notation) + +### 1.0.0-1.4.0 +- **pg_tle versions:** 1.0.0 through 1.3.x +- **PostgreSQL support:** 11-17 +- **API:** No `pgtle.uninstall_extension()` function, no schema parameter +- **Features:** Basic extension management, custom data types, authentication hooks + +### 1.4.0-1.5.0 +- **pg_tle versions:** 1.4.0 through 1.4.x +- **PostgreSQL support:** 11-17 +- **API:** Added `pgtle.uninstall_extension()` function, no schema parameter +- **Features:** Custom alignment/storage, enhanced warnings + +### 1.5.0+ +- **pg_tle versions:** 1.5.0 and later (tested through 1.5.2) +- **PostgreSQL support:** 12-18 (dropped PG 11) +- **API:** BREAKING CHANGE - `pgtle.install_extension()` now requires schema parameter +- **Features:** Schema parameter support in installation + +## Key API Changes by Version + +**1.4.0:** Added `pgtle.uninstall_extension()` +- Versions before 1.4.0 cannot uninstall extensions + +**1.5.0:** Changed `pgtle.install_extension()` signature +- Added required `schema` parameter +- Dropped PostgreSQL 11 support + +## Version Notation + +- `X.Y.Z+` - Works on pg_tle >= X.Y.Z +- `X.Y.Z-A.B.C` - Works on pg_tle >= X.Y.Z and < A.B.C + +**Boundary conditions:** +- `1.5.0+` means >= 1.5.0 (includes 1.5.0) +- `1.4.0-1.5.0` means >= 1.4.0 and < 1.5.0 (excludes 1.5.0) +- `1.0.0-1.4.0` means >= 1.0.0 and < 1.4.0 (excludes 1.4.0) + +## For Complete Details + +- `pgtle.sh` (comments at top) +- https://github.com/aws/pg_tle diff --git a/pgxntool-sync.sh b/pgxntool-sync.sh new file mode 100755 index 0000000..56896ac --- /dev/null +++ b/pgxntool-sync.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# +# pgxntool-sync.sh - Pull the latest pgxntool via git subtree and reconcile setup files +# +# This performs the two steps needed to update pgxntool inside a project: +# +# 1. git subtree pull -P pgxntool ... (update the embedded pgxntool copy) +# 2. update-setup-files.sh (3-way merge files setup.sh copied out, +# like .gitignore and test/deps.sql) +# +# It is invoked by the `make pgxntool-sync` targets, but can also be run directly +# so you never need make to update pgxntool. +# +# Usage: pgxntool-sync.sh [ []] +# +# repo Git URL (or path) to pull pgxntool from. Defaults to the canonical +# pgxntool repository. +# ref Branch, tag, or commit to pull. Defaults to the `release` tag, which +# always points at the latest released version. +# +# Run from the root of your project (the directory containing pgxntool/). + +set -o errexit -o errtrace -o pipefail +trap 'echo "Error on line ${LINENO}"' ERR + +PGXNTOOL_DIR="$(dirname "${BASH_SOURCE[0]}")" +source "$PGXNTOOL_DIR/lib.sh" + +# Canonical source for pgxntool. `release` is a floating tag that the release +# process force-moves to each new version, so it always names the latest +# released version -- exactly what a plain sync should track. +DEFAULT_REPO="https://github.com/Postgres-Extensions/pgxntool.git" +DEFAULT_REF="release" + +repo=${1:-$DEFAULT_REPO} +ref=${2:-$DEFAULT_REF} + +# We must run from the project root: git subtree pull operates on the pgxntool/ +# prefix and update-setup-files.sh resolves paths relative to the current dir. +[[ -d "pgxntool" ]] || die 1 "pgxntool directory not found. Run from your project root." +# Use rev-parse, not [ -d .git ]: in a worktree .git is a file, not a directory. +git rev-parse --git-dir >/dev/null 2>&1 || die 1 "Not in a git repository. Run from your project root." + +# The old commit is the pgxntool subtree HEAD before the pull; update-setup-files.sh +# needs it as the merge base for files that were copied out of pgxntool. +old_commit=$(git log -1 --format=%H -- pgxntool/) + +git subtree pull -P pgxntool --squash -m "Pull pgxntool from $repo $ref" "$repo" "$ref" + +"$PGXNTOOL_DIR/update-setup-files.sh" "$old_commit" diff --git a/run-test-build.sh b/run-test-build.sh new file mode 100755 index 0000000..86a61eb --- /dev/null +++ b/run-test-build.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# pgxntool/run-test-build.sh - Prepare test/build/ for the test-build target +# +# Sets up the generated sql/ directory and ensures expected/*.out files exist +# so pg_regress can run without aborting on missing files. +# +# Usage: run-test-build.sh TESTDIR +# +# Called by the test-build target in base.mk before running installcheck. + +set -e + +TESTDIR="${1:?Usage: run-test-build.sh TESTDIR}" +BUILD_DIR="$TESTDIR/build" +SQL_DIR="$BUILD_DIR/sql" +EXPECTED_DIR="$BUILD_DIR/expected" + +mkdir -p "$SQL_DIR" +mkdir -p "$EXPECTED_DIR" + +# Verify .sql files exist. This script is only called when test-build is +# enabled, so missing files indicate a misconfiguration. +files=("$BUILD_DIR"/*.sql) +if [ ! -f "${files[0]}" ]; then + echo "ERROR: no .sql files found in $BUILD_DIR/" >&2 + exit 1 +fi + +# Sync .sql files to sql/ directory for pg_regress. +# --checksum: compare by content, not size+mtime. rsync's default "quick check" +# assumes equal size+mtime means files are identical, which isn't safe here — +# builds can produce identical-sized files with different content. Checksum +# comparison also avoids unnecessary writes that could trigger antivirus. +# --times: preserve source mtimes on destination files so make's dependency +# tracking works correctly. +# --delete: remove files from sql/ that no longer exist in build/. +# --include/--exclude: select only *.sql from the directory source +# (--delete requires a directory transfer, not individual file arguments). +rsync -r --checksum --times --delete --include='*.sql' --exclude='*' "$BUILD_DIR/" "$SQL_DIR/" + +# Create empty expected/*.out files for .sql tests (if not already present). +# pg_regress requires an expected file to exist for each test; without it +# pg_regress stops immediately rather than running the test and showing the diff. +for file in "$BUILD_DIR"/*.sql; do + out="$EXPECTED_DIR/$(basename "$file" .sql).out" + [ -f "$out" ] || touch "$out" +done diff --git a/setup.sh b/setup.sh index 881ccaa..54b3633 100755 --- a/setup.sh +++ b/setup.sh @@ -3,7 +3,13 @@ set -o errexit -o errtrace -o pipefail trap 'echo "Error on line ${LINENO}"' ERR -[ -d .git ] || git init +# Source common library functions (error, die, debug) +PGXNTOOL_DIR="$(dirname "${BASH_SOURCE[0]}")" +source "$PGXNTOOL_DIR/lib.sh" + +# Use rev-parse, not [ -d .git ]: in a worktree .git is a file, not a directory, +# so the old check would wrongly re-run `git init` inside a valid working tree. +git rev-parse --git-dir >/dev/null 2>&1 || git init if ! git diff --cached --exit-code; then echo "Git repository is not clean; please commit and try again." >&2 @@ -35,21 +41,44 @@ safecp () { fi } -safecp pgxntool/_.gitignore .gitignore -safecp pgxntool/META.in.json META.in.json +# ============================================================================= +# SETUP FILES +# ============================================================================= +# SETUP_FILES and SETUP_SYMLINKS are defined in lib.sh +# These are also used by update-setup-files.sh for sync updates. +# ============================================================================= + +# Copy tracked setup files (defined in lib.sh) +for entry in "${SETUP_FILES[@]}"; do + src="pgxntool/${entry%%:*}" + dest="${entry##*:}" + # Create parent directory if needed + mkdir -p "$(dirname "$dest")" + safecp "$src" "$dest" +done + +# Create tracked symlinks (defined in lib.sh) +for entry in "${SETUP_SYMLINKS[@]}"; do + dest="${entry%%:*}" + target="${entry##*:}" + mkdir -p "$(dirname "$dest")" + if [ ! -e "$dest" ]; then + echo "Creating symlink $dest -> $target" + ln -s "$target" "$dest" + git add "$dest" + else + echo "$dest already exists" + fi +done +# META.in.json and Makefile are NOT in SETUP_FILES because users heavily customize them +safecp pgxntool/META.in.json META.in.json safecreate Makefile include pgxntool/base.mk make META.json git add META.json -mkdir -p sql test src - -cd test -mkdir -p sql -safecp ../pgxntool/test/deps.sql deps.sql -[ -d pgxntool ] || ln -s ../pgxntool/test/pgxntool . -git add pgxntool +mkdir -p sql test/sql src git status echo "If you won't be creating C code then you can: diff --git a/update-setup-files.sh b/update-setup-files.sh new file mode 100755 index 0000000..f814605 --- /dev/null +++ b/update-setup-files.sh @@ -0,0 +1,181 @@ +#!/usr/bin/env bash +# +# update-setup-files.sh - Update files that were initially copied by setup.sh +# +# This script handles the 3-way merge of setup files after a pgxntool subtree +# update. It compares the old pgxntool version, new pgxntool version, and +# user's current file to determine the appropriate action: +# +# 1. If pgxntool didn't change the file: skip (nothing to do) +# 2. If user hasn't modified the file: auto-update +# 3. If both changed: 3-way merge with conflict markers +# +# Usage: update-setup-files.sh +# +# The old commit is the pgxntool subtree commit BEFORE the sync. + +set -o errexit -o errtrace -o pipefail +trap 'echo "Error on line ${LINENO}"' ERR + +PGXNTOOL_DIR="$(dirname "${BASH_SOURCE[0]}")" +source "$PGXNTOOL_DIR/lib.sh" + +# SETUP_FILES and SETUP_SYMLINKS are defined in lib.sh + +# ============================================================================= +# Functions +# ============================================================================= + +usage() { + echo "Usage: $0 " + echo + echo "Updates setup files after a pgxntool subtree sync." + echo + echo "Arguments:" + echo " old-pgxntool-commit The pgxntool commit hash BEFORE the sync" + exit 1 +} + +# Get file content from a specific commit +# Usage: get_old_content +get_old_content() { + local commit=$1 + local path=$2 + git show "${commit}:pgxntool/${path}" 2>/dev/null +} + +# Get current file content from pgxntool directory +# Usage: get_new_content +get_new_content() { + local path=$1 + cat "pgxntool/${path}" 2>/dev/null +} + +# Process a single setup file +# Usage: process_file +process_file() { + local source=$1 + local dest=$2 + local old_commit=$3 + + # Get the three versions + local old_content new_content user_content + + old_content=$(get_old_content "$old_commit" "$source") || { + debug 20 "Could not get old version of $source (new file in pgxntool?)" + old_content="" + } + + new_content=$(get_new_content "$source") || { + error "Could not read pgxntool/$source" + return 1 + } + + # Check if destination exists + if [[ ! -e "$dest" ]]; then + echo " $dest: creating (file was missing)" + cp "pgxntool/$source" "$dest" + return 0 + fi + + user_content=$(cat "$dest") + + # Step 1: Did pgxntool change this file? + if [[ "$old_content" == "$new_content" ]]; then + debug 30 "$dest: pgxntool unchanged, skipping" + return 0 + fi + + # Step 2: Did user modify their copy? + if [[ "$user_content" == "$old_content" ]]; then + echo " $dest: updated (you hadn't modified it)" + cp "pgxntool/$source" "$dest" + return 0 + fi + + # Step 3: Both changed - need 3-way merge + echo " $dest: attempting 3-way merge..." + + # Create temp files for git merge-file + local tmp_old tmp_new + tmp_old=$(mktemp) + tmp_new=$(mktemp) + trap "rm -f '$tmp_old' '$tmp_new'" RETURN + + echo "$old_content" > "$tmp_old" + echo "$new_content" > "$tmp_new" + + # git merge-file modifies the first file in place + # Returns 0 on clean merge, >0 if conflicts (but still writes result) + if git merge-file -L "yours" -L "old pgxntool" -L "new pgxntool" \ + "$dest" "$tmp_old" "$tmp_new"; then + echo " $dest: merged cleanly (please review)" + else + echo " $dest: CONFLICTS - resolve manually" + fi +} + +# Process a symlink +# Usage: process_symlink +process_symlink() { + local dest=$1 + local target=$2 + + if [[ -L "$dest" ]]; then + local current_target + current_target=$(readlink "$dest") + if [[ "$current_target" == "$target" ]]; then + debug 30 "$dest: symlink unchanged" + else + echo " $dest: symlink points to '$current_target', expected '$target'" + echo " (not auto-fixing - please check manually)" + fi + elif [[ -e "$dest" ]]; then + echo " $dest: exists but is not a symlink (expected symlink to $target)" + else + echo " $dest: creating symlink to $target" + ln -s "$target" "$dest" + fi +} + +# ============================================================================= +# Main +# ============================================================================= + +[[ $# -eq 1 ]] || usage + +old_commit=$1 + +# Verify we're in a git repo with pgxntool subtree. +# Use rev-parse, not [ -d .git ]: in a worktree .git is a file, not a directory. +[[ -d "pgxntool" ]] || die 1 "pgxntool directory not found. Run from project root." +git rev-parse --git-dir >/dev/null 2>&1 || die 1 "Not in a git repository." + +# Verify the old commit is valid +if ! git cat-file -e "${old_commit}^{commit}" 2>/dev/null; then + die 1 "Invalid commit: $old_commit" +fi + +echo "Checking setup files for updates..." +echo + +# Process regular files +for entry in "${SETUP_FILES[@]}"; do + source="${entry%%:*}" + dest="${entry##*:}" + process_file "$source" "$dest" "$old_commit" +done + +# Process symlinks +for entry in "${SETUP_SYMLINKS[@]}"; do + dest="${entry%%:*}" + target="${entry##*:}" + process_symlink "$dest" "$target" +done + +# Prune pgxntool's own dev-only dirs (.github/, .claude/) that the subtree pull +# re-introduces but that don't belong in a consuming project (see lib.sh). +prune_pgxntool_dev_dirs + +echo +echo "Done. Review changes with 'git diff' and commit when ready." diff --git a/verify-results-pgtap.sh b/verify-results-pgtap.sh new file mode 100755 index 0000000..dd8d7e9 --- /dev/null +++ b/verify-results-pgtap.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# pgxntool/verify-results-pgtap.sh - Check pgtap results before 'make results' +# +# Scans pgtap output files for failures and plan mismatches, then checks +# regression.diffs as a fallback. Exits non-zero if any problems are found. +# +# Usage: verify-results-pgtap.sh TESTOUT +# +# Called by the verify-results target in base.mk (pgtap mode). + +set -e + +TESTOUT="${1:?Usage: verify-results-pgtap.sh TESTOUT}" + +# Check for pgtap failures in result files (excluding TODO items) +failed=0 +for f in "$TESTOUT"/results/*.out; do + [ -f "$f" ] || continue + if grep -q '^not ok' "$f"; then + notok=$(grep '^not ok' "$f" | grep -v '# TODO' || true) + if [ -n "$notok" ]; then + echo "ERROR: pgtap failure detected in $f" + echo "$notok" + failed=1 + fi + fi + if grep -q 'Looks like you planned' "$f"; then + echo "ERROR: pgtap plan mismatch in $f" + grep 'Looks like you planned' "$f" + failed=1 + fi +done +if [ $failed -ne 0 ]; then + echo + echo "pgtap failures detected. Cannot run 'make results'." + exit 1 +fi + +# Also check regression.diffs (output mismatch even if pgtap all passed) +if [ -r "$TESTOUT/regression.diffs" ]; then + echo "ERROR: Tests are failing. Cannot run 'make results'." + echo "Fix test failures first, then run 'make results'." + echo + echo "See $TESTOUT/regression.diffs for details:" + cat "$TESTOUT/regression.diffs" + exit 1 +fi From 0ec49aeff2c2d646ff0d322d2150b109aec599bd Mon Sep 17 00:00:00 2001 From: jnasbyupgrade Date: Thu, 16 Jul 2026 12:39:34 -0500 Subject: [PATCH 2/2] Reconcile pgxntool setup files after 2.1.0 sync Auto-updated .gitignore from pgxntool 2.1.0 and pruned pgxntool's dev-only .github/ and .claude/ (not for embedding projects). --- .gitignore | 8 + pgxntool/.claude/CLAUDE.md | 28 -- pgxntool/.claude/development.md | 47 --- pgxntool/.github/workflows/CLAUDE.md | 61 ---- pgxntool/.github/workflows/ci.yml | 294 ------------------ .../.github/workflows/claude-code-review.yml | 102 ------ pgxntool/.github/workflows/claude.yml | 46 --- pgxntool/.github/workflows/protect-label.yml | 146 --------- 8 files changed, 8 insertions(+), 724 deletions(-) delete mode 100644 pgxntool/.claude/CLAUDE.md delete mode 100644 pgxntool/.claude/development.md delete mode 100644 pgxntool/.github/workflows/CLAUDE.md delete mode 100644 pgxntool/.github/workflows/ci.yml delete mode 100644 pgxntool/.github/workflows/claude-code-review.yml delete mode 100644 pgxntool/.github/workflows/claude.yml delete mode 100644 pgxntool/.github/workflows/protect-label.yml diff --git a/.gitignore b/.gitignore index 0c14928..1873c2c 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,14 @@ results/ regression.diffs regression.out +# Generated sql/ directory for test/build +# Created by make test-build. See README.asc for details. +test/build/sql/ + +# Auto-generated schedule file for test/install +# Created by make when test/install/*.sql files exist. +test/install/schedule + # Misc tmp/ .DS_Store diff --git a/pgxntool/.claude/CLAUDE.md b/pgxntool/.claude/CLAUDE.md deleted file mode 100644 index e2e74fe..0000000 --- a/pgxntool/.claude/CLAUDE.md +++ /dev/null @@ -1,28 +0,0 @@ -# Claude Development Notes - -This file contains guidance for Claude Code when working in this repository. -It is excluded from distributions via `.gitattributes export-ignore`. - -## CI Monitoring After Every Push - -**REQUIRED**: After every `git push`, immediately start a background task to -monitor the CI run for that push. If you pushed to both pgxntool and -pgxntool-test, start a background task for each repo — do not monitor them -sequentially. - -Use `gh run watch` or poll with `gh run list` / `gh pr checks` in the -background task. Report failures to the user as soon as they are detected; -do not wait for all jobs to finish before reporting. - -## Multiple Concurrent Sessions - -It is common to have multiple Claude Code sessions open simultaneously across -pgxntool and pgxntool-test. To avoid cross-session interference: - -**If you are asked to do something on an existing PR that you did not open or -are not already working on in this session, immediately ask for confirmation -before proceeding.** For example: "I see PR #32 exists. Were you asking me to -work on that, or did you mean to send this to a different session?" - -This applies to: editing PR branches, pushing to them, closing/reopening them, -adding commits, modifying PR descriptions, or any other PR-level action. diff --git a/pgxntool/.claude/development.md b/pgxntool/.claude/development.md deleted file mode 100644 index 886484a..0000000 --- a/pgxntool/.claude/development.md +++ /dev/null @@ -1,47 +0,0 @@ -# pgxntool Development Guidelines - -**THIS FILE IS FOR PGXNTOOL DEVELOPERS ONLY.** - -If you are an extension developer using pgxntool in your project, this file does not -apply to you. See the top-level `CLAUDE.md` instead. - -## Critical: Work from pgxntool-test, Not Here - -**NEVER make changes to pgxntool directly from this repository.** - -pgxntool development must be done from a checkout of **pgxntool-test**, which contains -the full test infrastructure. Working here directly means you cannot run tests, and -any changes you commit cannot be validated before merging. - -**Correct workflow:** -1. Clone or use an existing checkout of `pgxntool-test` -2. Work in a worktree: both `pgxntool/` and `pgxntool-test/` will be siblings -3. Make changes to `pgxntool/` from within that pgxntool-test context -4. Run the test suite via `make test` in pgxntool-test before committing - -**See:** https://github.com/Postgres-Extensions/pgxntool-test for the full development -workflow. - ---- - -## Makefile Variable Assignment Rules - -**RULE: Do not use `:=` (simply expanded) unless you have a specific need for immediate evaluation.** - -Use `=` (recursively expanded) for standard variable assignments. Reserve `:=` for cases where the right-hand side must be evaluated exactly once at assignment time — for example, when assigning the result of a `$(call ...)` function that references the variable being set (which would cause infinite recursion with `=`). - -When a variable must also override command-line values, combine `override` with `:=` — but only where `override` is genuinely needed. - -## Debug Level Rules (lib.sh `debug` function) - -`debug LEVEL "msg"` prints when `DEBUG >= LEVEL`. LEVEL encodes how noisy/esoteric a message is — how far you'd crank `DEBUG` before you'd want to see it — **not** code nesting depth. A top-level line can warrant a high level if it's esoteric, and loop-body detail is usually high precisely because it's noisy. Judge by signal-to-noise. - -The tiers are anchors, not strict multiples of 10 — any value in range is fine, leaving room to fine-tune between existing calls without renumbering: - -- **10**: Critical errors, important warnings -- **20**: Warnings, significant state changes -- **30**: General debugging, function entry/exit, array operations -- **40**: Verbose details, loop iterations -- **50+**: Maximum verbosity (per-iteration innards) - -Note: The BATS test helper `debug` function (in `tests/lib/helpers.bash` in pgxntool-test) uses a separate 1–5 scale controlled by `$TESTDEBUG`. The two systems are independent. diff --git a/pgxntool/.github/workflows/CLAUDE.md b/pgxntool/.github/workflows/CLAUDE.md deleted file mode 100644 index 48c749c..0000000 --- a/pgxntool/.github/workflows/CLAUDE.md +++ /dev/null @@ -1,61 +0,0 @@ -# .github/workflows — CI Architecture - -## Workflow files - -- **`ci.yml`** — main CI for pgxntool pull requests. Runs `check-test-pr` (verifies - the paired pgxntool-test PR's CI passed), then optionally runs `test` (only for the - commit-with-no-tests path — see below). -- **`protect-label.yml`** — enforces that only maintainers with write access can apply - or remove the `commit-with-no-tests` label. - -## Normal CI flow (paired test PR exists) - -When a pgxntool PR has a corresponding open PR in pgxntool-test with the same branch -name, the `check-test-pr` job polls (up to 20 minutes) for that test PR's CI to -complete and pass. If it passes, pgxntool CI passes — **no tests run here**. Tests run -exactly once, in pgxntool-test's own CI. - -## commit-with-no-tests path - -When a maintainer applies the `commit-with-no-tests` label (and no paired test PR -exists), the `test` job runs tests directly in pgxntool CI against pgxntool-test/master. -This is the rare exception, not the norm. - -## Cross-repo reusable workflow — tradeoffs and constraints - -The `test` job calls a reusable workflow from pgxntool-test: -```yaml -uses: Postgres-Extensions/pgxntool-test/.github/workflows/run-tests.yml@ -``` - -GitHub Actions requires the `uses:` ref to be a **static string** — expressions like -`${{ }}` are not supported in the repo/path portion or the `@ref` suffix in practice. - -### The @branch → @master ref - -While developing on a feature branch where pgxntool-test also has changes, this ref -is set to `@` so CI can find `run-tests.yml` before it lands on master. - -**IMPORTANT**: This ref must be updated to `@master` before pgxntool merges. The -correct merge order is: **pgxntool-test merges first**, then update this ref to -`@master`, then pgxntool merges. - -**For Claude**: Do NOT leave a `@` ref without explicit user approval. The -user merges directly from the PR page — there are no manual steps between merges. -See `.github/workflows/CLAUDE.md` in pgxntool-test for the full picture. - -### Changes to run-tests.yml - -`run-tests.yml` lives in pgxntool-test and is the single source of truth for all test -steps. If it changes, pgxntool's CI uses `@master` — so it won't see the new version -until pgxntool-test merges. This is acceptable because: -- Changes to `run-tests.yml` require a paired test PR (not commit-with-no-tests) -- When a paired test PR exists, pgxntool's `test` job is skipped anyway -- The two scenarios are mutually exclusive in practice - -## Label name - -The label `commit-with-no-tests` is defined as a const (`NO_TEST_LABEL`) in `ci.yml` -and as `LABEL` in `protect-label.yml`. The job-level `if:` condition in -`protect-label.yml` must also use the literal string (YAML can't reference JS consts) -— keep these in sync if the label name ever changes. diff --git a/pgxntool/.github/workflows/ci.yml b/pgxntool/.github/workflows/ci.yml deleted file mode 100644 index 9e3ce90..0000000 --- a/pgxntool/.github/workflows/ci.yml +++ /dev/null @@ -1,294 +0,0 @@ -name: CI - -on: - pull_request: - # We use 'pull_request' (not 'pull_request_target') deliberately. - # 'pull_request_target' runs with write access to the base repo, which is - # a security risk for untrusted fork code. Since this workflow only reads - # from other public repos (no secrets needed), 'pull_request' is correct - # and safe even for fork PRs. - -permissions: - contents: read # required by actions/checkout in the reusable test workflow - pull-requests: read - checks: read - -concurrency: - group: ci-pr-${{ github.event.pull_request.number }} - cancel-in-progress: true - -jobs: - check-test-pr: - name: Check for paired pgxntool-test PR - runs-on: ubuntu-latest - # This check polls until the paired pgxntool-test CI run completes - # (up to 20 minutes). The job timeout gives a few minutes of headroom. - timeout-minutes: 25 - outputs: - run-tests: ${{ steps.check.outputs.run_tests }} - test-ref: ${{ steps.check.outputs.test_ref }} - - steps: - - name: Find paired pgxntool-test PR or check commit-with-no-tests label - id: check - # Pinned to an immutable SHA (supply-chain hardening); comment tracks the tag. - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 - with: - # GITHUB_TOKEN is sufficient for reading public repos. If these repos - # are ever made private, replace with a PAT stored as a secret with - # 'repo' scope on both repos. Note: PAT expiration causes silent - # failures here — the API returns 401 and the job errors out instead - # of failing gracefully with a useful message. - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const branch = context.payload.pull_request.head.ref; - const prNumber = context.payload.pull_request.number; - // Single source of truth for the label name. Must also match the - // literal string in the protect-label.yml job-level `if:` condition - // (YAML expressions can't reference JS constants). - const NO_TEST_LABEL = 'commit-with-no-tests'; - - // master-to-master PRs have no paired test PR by convention. - // Run tests against pgxntool-test/master directly. - // - // If a fork PR's branch is named 'master', that's almost certainly - // a mistake (contributors should use a feature branch), but we - // don't block it — just warn visibly as an annotation on the run. - // Note: pull_request gives a read-only token for fork PRs, so we - // can't post a PR comment back to the upstream repo from here. - // Gate on the BASE branch too: this shortcut is only for - // master-to-master PRs. A PR from master into some other base must - // still go through the normal paired-test lookup below. - if (branch === 'master' && context.payload.pull_request.base.ref === 'master') { - const headRepo = context.payload.pull_request.head.repo; - const isBaseRepo = - headRepo?.owner?.login === context.repo.owner && - headRepo?.name === context.repo.repo; - if (!isBaseRepo) { - core.warning( - `PR head branch is named 'master' but comes from a fork ` + - `(${headRepo?.full_name ?? 'unknown'}). Contributors should ` + - `use a feature branch, not master. Proceeding with tests ` + - `against pgxntool-test/master.` - ); - } - core.setOutput('run_tests', 'true'); - core.setOutput('test_ref', 'master'); - return; - } - - // The owner of this PR's head repo — the contributor's fork owner - // for fork PRs, or the base repo owner for maintainer PRs. - // The paired pgxntool-test PR must come from the SAME owner. - // We never cross-match PRs across different contributors' forks. - const prOwner = context.payload.pull_request.head.repo?.owner?.login; - - // Look for open pgxntool-test PRs with the SAME branch name AND - // the same fork owner. Branch names must match exactly. - // - // The GitHub API's 'head' filter requires "owner:branch" format. - // We list all open PRs and filter locally — safe for repos with - // few open PRs, and avoids needing to know the fork repo name. - // paginate() fetches all pages automatically, so this is correct - // even if pgxntool-test ever exceeds 100 open PRs (the per_page cap). - const prs = await github.paginate(github.rest.pulls.list, { - owner: context.repo.owner, - repo: 'pgxntool-test', - state: 'open', - per_page: 100 - }); - - const matching = prs.filter(pr => - pr.head.ref === branch && - pr.head.repo?.owner?.login === prOwner - ); - if (matching.length > 1) { - core.setFailed( - `Multiple open pgxntool-test PRs from ${prOwner} match branch ` + - `'${branch}'. Cannot determine which one to use.\n\n` + - `Close all but one, then re-run this check.` - ); - return; - } - - const testPR = matching.length === 1 ? matching[0] : null; - - if (testPR) { - // Error if the no-test label is also set — that's contradictory. - // Re-fetch the PR live (not from payload) in case the label was - // added after this workflow was triggered. - const { data: currentPR } = await github.rest.pulls.get({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: prNumber - }); - if (currentPR.labels.some(l => l.name === NO_TEST_LABEL)) { - core.setFailed( - `PR has the '${NO_TEST_LABEL}' label, but a paired ` + - `pgxntool-test PR #${testPR.number} exists on branch '${branch}'.\n\n` + - `Remove the '${NO_TEST_LABEL}' label — it should only be used ` + - `when there is genuinely no paired test PR.` - ); - return; - } - - // A paired test PR exists. Verify its CI passed for the exact - // current HEAD SHA and that the run is recent enough to be valid. - const sha = testPR.head.sha; - const testPRUrl = - `https://github.com/${context.repo.owner}/pgxntool-test/pull/${testPR.number}`; - const recheckUrl = - `https://github.com/${context.repo.owner}/${context.repo.repo}/pull/${prNumber}/checks`; - - core.info(`Found pgxntool-test PR #${testPR.number} (${sha.slice(0, 7)})`); - - // Poll until all check runs for the exact HEAD SHA complete. - // Using 'ref: sha' (not branch name) ensures we only see runs for - // this commit — never stale runs from an older push on the same branch. - // - // We poll rather than fail immediately because both repos are often - // pushed close together. When that happens, pgxntool CI starts while - // pgxntool-test CI may not have queued yet. We wait up to 20 minutes. - const POLL_INTERVAL_MS = 30 * 1000; - const MAX_WAIT_MS = 20 * 60 * 1000; - const waitStart = Date.now(); - let runs; - - while (true) { - // per_page: 100 is intentional here — a single commit will - // not realistically have 100+ CI check runs, so pagination - // is unnecessary. (pulls.list uses paginate() above because - // an active repo could have many open PRs.) - const { data: checks } = await github.rest.checks.listForRef({ - owner: context.repo.owner, - repo: 'pgxntool-test', - ref: sha, - per_page: 100 - }); - runs = checks.check_runs; - - const incomplete = runs.filter(r => r.status !== 'completed'); - if (runs.length > 0 && incomplete.length === 0) break; - - const elapsed = Date.now() - waitStart; - if (elapsed >= MAX_WAIT_MS) { - const mins = Math.round(elapsed / 60000); - if (runs.length === 0) { - core.setFailed( - `pgxntool-test PR #${testPR.number} has no CI runs for ` + - `SHA ${sha.slice(0, 7)} after waiting ${mins} min.\n\n` + - `Push a commit (or manually re-run CI) on the test PR:\n` + - ` Test PR: ${testPRUrl}\n` + - ` Re-run this check: ${recheckUrl}` - ); - } else { - const names = incomplete.map(r => r.name).join(', '); - core.setFailed( - `pgxntool-test PR #${testPR.number} CI did not finish within ` + - `${mins} min for SHA ${sha.slice(0, 7)}: ${names}\n\n` + - ` Test PR: ${testPRUrl}\n` + - ` Re-run this check: ${recheckUrl}` - ); - } - return; - } - - if (runs.length === 0) { - core.info(`No CI runs yet for pgxntool-test PR #${testPR.number} (${sha.slice(0, 7)}); waiting 30s...`); - } else { - const names = incomplete.map(r => r.name).join(', '); - core.info(`pgxntool-test CI still running (${names}); waiting 30s...`); - } - await new Promise(resolve => setTimeout(resolve, POLL_INTERVAL_MS)); - } - - // All checks complete — look for failures. - // 'success', 'skipped', 'neutral' are non-blocking. - const failed = runs.filter( - r => !['success', 'skipped', 'neutral'].includes(r.conclusion) - ); - if (failed.length > 0) { - const names = failed.map(r => `${r.name} (${r.conclusion})`).join(', '); - core.setFailed( - `pgxntool-test PR #${testPR.number} CI failed for ` + - `SHA ${sha.slice(0, 7)}: ${names}\n\n` + - `Fix the test PR CI, then re-run this check:\n` + - ` Test PR: ${testPRUrl}\n` + - ` Re-run this check: ${recheckUrl}` - ); - return; - } - - core.info( - `pgxntool-test PR #${testPR.number} CI passed for ` + - `SHA ${sha.slice(0, 7)} — tests run there, not here.` - ); - core.setOutput('run_tests', 'false'); - core.setOutput('test_ref', sha); - return; - } - - // No paired test PR found. Check for the NO_TEST_LABEL label, - // which a maintainer can apply when a pgxntool change genuinely - // needs no test changes (unusual). - // - // We make a live API call rather than reading from the event - // payload. The payload is a snapshot from when this workflow was - // triggered — a maintainer may have added the label after that. - const { data: pr } = await github.rest.pulls.get({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: prNumber - }); - - if (pr.labels.some(l => l.name === NO_TEST_LABEL)) { - core.info( - `'${NO_TEST_LABEL}' label is present; running tests ` + - "against pgxntool-test/master. The protect-label workflow " + - "ensures only maintainers can apply this label." - ); - core.setOutput('run_tests', 'true'); - core.setOutput('test_ref', 'master'); - return; - } - - // Neither a paired test PR nor the override label was found. - // Fail with a clear, actionable message. - core.setFailed( - `No paired pgxntool-test PR found for branch '${branch}', ` + - `and no '${NO_TEST_LABEL}' label on this PR.\n\n` + - `pgxntool changes should always be paired with matching test\n` + - `changes in pgxntool-test. This check enforces that pairing.\n\n` + - `To resolve:\n` + - ` 1. Open a PR in pgxntool-test from the SAME account (${prOwner}),\n` + - ` on a branch ALSO named '${branch}'. Both the branch name and\n` + - ` the head owner must match exactly for the pairing to work.\n\n` + - ` 2. If this pgxntool change truly needs no test updates (unusual),\n` + - ` ask a maintainer to apply the '${NO_TEST_LABEL}' label.\n` + - ` Only maintainers can apply this label. It is not a normal\n` + - ` shortcut — most pgxntool changes require test updates.\n\n` + - `See: https://github.com/Postgres-Extensions/pgxntool-test#ci-and-contributing` - ); - - test: - needs: check-test-pr - if: needs.check-test-pr.outputs.run-tests == 'true' - # ----------------------------------------------------------------------- - # CROSS-REPO REUSABLE WORKFLOW — READ BEFORE CHANGING THIS REF - # See: .github/workflows/CLAUDE.md for full architecture notes. - # - # The ref must be a static string — GitHub Actions does not support - # expressions in uses:. It points at pgxntool-test's run-tests.yml on - # master. (During feature-branch development this is temporarily set to - # @ so CI can find run-tests.yml before it lands on master, and - # flipped back to @master once pgxntool-test/ has merged.) - # ----------------------------------------------------------------------- - uses: Postgres-Extensions/pgxntool-test/.github/workflows/run-tests.yml@master - with: - # pgxntool: this PR's own branch, on its own account (a fork for fork PRs). - pgxntool-owner: ${{ github.event.pull_request.head.repo.owner.login }} - pgxntool-branch: ${{ github.event.pull_request.head.ref }} - # pgxntool-test: no paired test PR in this path, so use canonical master - # from Postgres-Extensions only (never a fork's master). - pgxntool-test-owner: Postgres-Extensions - pgxntool-test-ref: master diff --git a/pgxntool/.github/workflows/claude-code-review.yml b/pgxntool/.github/workflows/claude-code-review.yml deleted file mode 100644 index fc7e26c..0000000 --- a/pgxntool/.github/workflows/claude-code-review.yml +++ /dev/null @@ -1,102 +0,0 @@ -name: Claude Code Review - -# Runs on PRs INTO this repo. We use pull_request_target (not pull_request) so -# that PRs from a fork can access CLAUDE_CODE_OAUTH_TOKEN — GitHub withholds -# secrets from `pull_request` runs triggered by forks, which is why the plain -# `pull_request` version never worked for fork PRs. -# -# SECURITY: pull_request_target runs in the BASE repo with secrets and a -# write-capable token. The job is gated to PRs from the trusted `jnasbyupgrade` -# fork only — an arbitrary external fork can never trigger this secret-bearing -# job. The workflow file always comes from the base branch (master), so a PR -# cannot modify the reviewer that runs on it. We check out the PR head only for -# read context (persist-credentials: false) and never build or execute PR code. -on: - pull_request_target: - types: [opened, synchronize, reopened, ready_for_review] - -concurrency: - group: claude-review-${{ github.event.pull_request.number }} - cancel-in-progress: true - -jobs: - claude-review: - # Trusted fork only, and skip drafts (don't spend API/CI on unfinished PRs). - # To add more trusted owners, extend the head-owner check. - if: >- - github.event.pull_request.draft == false && - github.event.pull_request.head.repo.owner.login == 'jnasbyupgrade' - runs-on: ubuntu-latest - timeout-minutes: 60 - permissions: - contents: read - pull-requests: write # post the review comments - checks: read # read sibling check-runs for the cost gate - steps: - # COST GATE: the paid Claude review is the last thing to run. Wait for the - # PR head's OTHER check-runs to finish and only proceed if they are clean. - # If any sibling check failed we skip the review to avoid spending money - # reviewing a PR that is already known-broken. Uniform across all repos: - # it discovers sibling checks dynamically (no per-repo workflow names). - # - decision=run : all sibling checks completed with a good conclusion, - # OR no sibling checks exist after a short grace window - # (nothing to gate on), OR the poll timed out is treated - # as skip (see below). - # - decision=skip : at least one sibling check failed/cancelled/etc, or - # we timed out waiting for still-pending checks. - # We exclude this workflow's own check-run (job name `claude-review`) so the - # gate never waits on or fails because of itself. - - name: Wait for CI; skip the paid review if any check failed - id: gate - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - REPO: ${{ github.repository }} - SHA: ${{ github.event.pull_request.head.sha }} - run: | - decision=skip - for i in $(seq 1 72); do # ~24 min max - json=$(gh api "repos/$REPO/commits/$SHA/check-runs" --paginate \ - --jq '[.check_runs[] | select(.name != "claude-review")]' 2>/dev/null) || json='' - [ -z "$json" ] && { sleep 20; continue; } - total=$(jq 'length' <<<"$json") - if [ "$total" -eq 0 ]; then - [ "$i" -ge 9 ] && { decision=run; break; } # ~3 min grace: nothing to gate on - sleep 20; continue - fi - pending=$(jq '[.[]|select(.status!="completed")]|length' <<<"$json") - if [ "$pending" -eq 0 ]; then - bad=$(jq '[.[]|select((.conclusion//"")|test("^(failure|cancelled|timed_out|action_required|stale)$"))]|length' <<<"$json") - [ "$bad" -eq 0 ] && decision=run || decision=skip - break - fi - sleep 20 - done - echo "decision=$decision" >> "$GITHUB_OUTPUT" - echo "gate decision: $decision" - - - name: Check out PR head (read-only context) - if: steps.gate.outputs.decision == 'run' - # Intentionally tracks the major-version tag (not a pinned SHA) so - # upstream fixes are picked up automatically. - uses: actions/checkout@v4 - with: - repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.event.pull_request.head.sha }} - fetch-depth: 1 - persist-credentials: false - - - name: Run Claude Code Review - if: steps.gate.outputs.decision == 'run' - uses: anthropics/claude-code-action@v1 - with: - claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} - # Provide github_token so the action uses it directly for GitHub API - # calls instead of the OIDC->GitHub-App-token exchange, which 401s under - # pull_request_target. GITHUB_TOKEN is repo/workflow-scoped (independent - # of the actor's role) and has pull-requests: write here. - github_token: ${{ secrets.GITHUB_TOKEN }} - # NOTE: plugin_marketplaces can't be pinned — it tracks the - # marketplace repo's default branch (upstream anthropics/claude-code). - plugin_marketplaces: 'https://github.com/anthropics/claude-code.git' - plugins: 'code-review@claude-code-plugins' - prompt: '/code-review:code-review ${{ github.repository }}/pull/${{ github.event.pull_request.number }}' diff --git a/pgxntool/.github/workflows/claude.yml b/pgxntool/.github/workflows/claude.yml deleted file mode 100644 index c85ec00..0000000 --- a/pgxntool/.github/workflows/claude.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: Claude Code - -on: - issue_comment: - types: [created] - pull_request_review_comment: - types: [created] - issues: - types: [opened, assigned] - pull_request_review: - types: [submitted] - -# No concurrency limit: @claude mentions are independent, read-only requests; -# serializing would only delay responses and cancelling would drop them. -jobs: - claude: - if: | - (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) || - (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) || - (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) || - (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude'))) - runs-on: ubuntu-latest - timeout-minutes: 30 - permissions: - contents: read - pull-requests: read - issues: read - id-token: write - actions: read # Required for Claude to read CI results on PRs - steps: - - name: Checkout repository - # Intentionally tracks the major-version tag (not a pinned SHA) so - # upstream fixes are picked up automatically. - uses: actions/checkout@v4 - with: - fetch-depth: 1 - persist-credentials: false - - - name: Run Claude Code - id: claude - uses: anthropics/claude-code-action@v1 - with: - claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} - # Allows Claude to read CI results on PRs - additional_permissions: | - actions: read diff --git a/pgxntool/.github/workflows/protect-label.yml b/pgxntool/.github/workflows/protect-label.yml deleted file mode 100644 index de71ea3..0000000 --- a/pgxntool/.github/workflows/protect-label.yml +++ /dev/null @@ -1,146 +0,0 @@ -name: Protect 'commit-with-no-tests' label - -on: - # IMPORTANT: Must use pull_request_target, NOT pull_request. - # - # 'pull_request' from a fork runs with a read-only GITHUB_TOKEN scoped to - # the fork. It cannot add or remove labels on the upstream repo (write - # operation), and cannot call getCollaboratorPermissionLevel (requires write - # permission to the target repo). - # - # 'pull_request_target' runs in the base repo's context with a token that - # has write access — exactly what we need here. - # - # Security: because pull_request_target has write access, never check out - # or execute code from the PR head in this workflow. This workflow only calls - # the GitHub API via actions/github-script and is safe. - pull_request_target: - types: [labeled, unlabeled] - -jobs: - protect: - # Only fire for the label we care about. All other label changes are - # unaffected by this workflow. - # Note: this literal must match the LABEL const defined in the script below. - if: github.event.label.name == 'commit-with-no-tests' - runs-on: ubuntu-latest - permissions: - pull-requests: write # To add/remove labels - issues: write # GitHub label API goes through the issues endpoint - - steps: - - name: Enforce write-access-only on 'commit-with-no-tests' label - # Pinned to an immutable SHA: this workflow runs as pull_request_target - # with write access, so a moved upstream tag must not change what runs. - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 - with: - script: | - const actor = context.actor; - const prNumber = context.payload.pull_request.number; - const action = context.payload.action; // 'labeled' or 'unlabeled' - // Single source of truth for the label name within this script. - // Must also match the literal in the job-level `if:` condition above - // (YAML expressions can't reference JS constants). - const LABEL = 'commit-with-no-tests'; - - // When this workflow re-adds or removes the label itself, that fires - // this event again with actor = 'github-actions[bot]'. Without this - // guard the job loops forever. We match any '[bot]' suffix to also - // cover other automation (Dependabot, Renovate, etc.). - if (actor.endsWith('[bot]')) { - core.info(`Actor is a bot (${actor}); skipping permission check`); - return; - } - - // Check the actor's effective permission level in this repo. - // - // EDGE CASE — 404 for non-collaborators: This API returns 404 when - // the user is not an explicit collaborator. This is the normal case - // for contributors who forked and opened a PR. If we don't catch - // this error, the job crashes with an unhandled exception and the - // label stays in whatever state the contributor put it in — - // defeating the entire protection. - // - // EDGE CASE — org team members: Users with write access via org - // team membership (not a direct collaborator invite) correctly show - // as 'write' here because the API returns effective permission. - // Exception: if the org has "private member visibility" set and the - // token can't enumerate team membership, they may get a 404 instead. - // If that becomes an issue, add a fallback to - // github.rest.orgs.getMembershipForUser(). - // - // EDGE CASE — other errors: Network blips, API outages, and rate - // limiting all throw here. We fail safe by treating any unexpected - // error as "no write access" and logging for debugging. - let hasWrite = false; - try { - const { data: perm } = await github.rest.repos.getCollaboratorPermissionLevel({ - owner: context.repo.owner, - repo: context.repo.repo, - username: actor - }); - hasWrite = ['admin', 'write'].includes(perm.permission); - } catch (e) { - if (e.status === 404) { - // Not a collaborator — no write access. Expected and normal. - hasWrite = false; - } else { - core.warning( - `Unexpected error checking permissions for ${actor} ` + - `(HTTP ${e.status}): ${e.message}. Treating as no write access.` - ); - hasWrite = false; - } - } - - if (action === 'labeled' && !hasWrite) { - core.info(`${actor} lacks write access; removing '${LABEL}' label`); - await github.rest.issues.removeLabel({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: prNumber, - name: LABEL - }); - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: prNumber, - body: - `@${actor} The \`commit-with-no-tests\` label can only be applied by ` + - `maintainers with write access to this repository.\n\n` + - `If you believe no test changes are needed for this PR, please ask a ` + - `maintainer to apply the label after reviewing. Note that most pgxntool ` + - `changes do require paired test updates — this label should be used sparingly.` - }); - - } else if (action === 'unlabeled' && !hasWrite) { - // Non-writer removed the label. Put it back. - // - // EDGE CASE — brief label-absent window: There is a short window - // between removal and this workflow re-adding the label. During - // that window the label genuinely does not exist. This is harmless - // in practice: the ci.yml workflow reads labels via a live API - // call (not from its cached payload), so a re-run after the label - // is restored will pick it up correctly. - core.info(`${actor} lacks write access; re-adding '${LABEL}' label`); - await github.rest.issues.addLabels({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: prNumber, - labels: [LABEL] - }); - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: prNumber, - body: - `@${actor} The \`commit-with-no-tests\` label can only be removed by ` + - `maintainers with write access to this repository.\n\n` + - `Contact a maintainer if you believe this label was applied in error.` - }); - - } else if (hasWrite) { - core.info( - `${actor} has write access; '${action}' on '${LABEL}' label is approved` - ); - }