From 694f32c2a6d842a0cd8c06b1b0b7af877349aa80 Mon Sep 17 00:00:00 2001 From: Juan Cruz Viotti Date: Tue, 25 Aug 2026 17:15:43 -0300 Subject: [PATCH 1/7] [WIP] Start benchmarking server operations Signed-off-by: Juan Cruz Viotti --- CMakeLists.txt | 2 + benchmark/measure.py | 91 +++++++++++++++++++++++++++ benchmark/server.sh | 75 ++++++++++++++++++++++ enterprise/e2e/auth-keys/benchmark.sh | 14 +++++ test/e2e/html/benchmark.sh | 10 +++ 5 files changed, 192 insertions(+) create mode 100755 benchmark/measure.py create mode 100755 benchmark/server.sh create mode 100755 enterprise/e2e/auth-keys/benchmark.sh create mode 100755 test/e2e/html/benchmark.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 98bf0994e..c82c608ed 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -130,7 +130,9 @@ sourcemeta_target_clang_format(SOURCES enterprise/*.h enterprise/*.cc) sourcemeta_target_shellcheck(SOURCES release.sh + contrib/*.sh test/*.sh + test/e2e/*/*.sh docker/*.sh enterprise/scripts/*.sh enterprise/e2e/*/*.sh diff --git a/benchmark/measure.py b/benchmark/measure.py new file mode 100755 index 000000000..e0c5530a1 --- /dev/null +++ b/benchmark/measure.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 + +"""Measure how long one endpoint takes to answer. + +The connection is opened once and kept, so what is timed is answering a +request rather than establishing a conversation. Requests are made one after +another, which measures latency rather than throughput: what this answers is +how long one caller waits, not how many callers can be served at once. +""" + +import argparse +import http.client +import json +import sys +import time + + +def percentile(samples, fraction): + return samples[min(int(len(samples) * fraction), len(samples) - 1)] + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--name", required=True) + parser.add_argument("--url", required=True) + parser.add_argument("--method", default="GET") + parser.add_argument("--header", action="append", default=[]) + parser.add_argument("--body", default=None) + parser.add_argument("--count", type=int, default=20000) + parser.add_argument("--warmup", type=int, default=1000) + options = parser.parse_args() + + _, _, rest = options.url.partition("//") + authority, _, path = rest.partition("/") + host, _, port = authority.partition(":") + path = "/" + path + + headers = {} + for header in options.header: + key, _, value = header.partition(":") + headers[key.strip()] = value.strip() + + body = options.body.encode() if options.body else None + connection = http.client.HTTPConnection(host, int(port or 80)) + connection.connect() + + def once(): + connection.request(options.method, path, body=body, headers=headers) + response = connection.getresponse() + response.read() + return response.status + + # Answering the first time builds what answering afterwards reuses, so + # what is measured is a warm instance rather than a cold one + for _ in range(options.warmup): + status = once() + if status >= 400: + sys.exit(f"{options.name}: warm up got HTTP {status}") + + samples = [] + for _ in range(options.count): + start = time.perf_counter_ns() + status = once() + samples.append((time.perf_counter_ns() - start) / 1000.0) + # A run that was refused measured something other than what it set out + # to, so it stops rather than reporting a number nobody can trust + if status >= 400: + sys.exit(f"{options.name}: got HTTP {status}") + + samples.sort() + json.dump( + [ + { + "name": f"{options.name} (p50)", + "unit": "us", + "value": round(percentile(samples, 0.50)), + }, + { + "name": f"{options.name} (p99)", + "unit": "us", + "value": round(percentile(samples, 0.99)), + }, + ], + sys.stdout, + indent=2, + ) + sys.stdout.write("\n") + + +if __name__ == "__main__": + main() diff --git a/benchmark/server.sh b/benchmark/server.sh new file mode 100755 index 000000000..00aa31150 --- /dev/null +++ b/benchmark/server.sh @@ -0,0 +1,75 @@ +#!/bin/sh + +set -o errexit +set -o nounset + +if [ "$#" -ne 2 ] +then + echo "Usage: $0 " 1>&2 + exit 1 +fi + +INDEX="$1" +SERVER="$2" +HERE="$(cd "$(dirname "$0")" && pwd)" +ROOT="$(cd "$HERE/.." && pwd)" +PORT="${BENCHMARK_PORT:-8199}" + +# A suite says what it wants measured by carrying a script that says it, so +# wiring a new one up is writing that script and nothing else here +RESULTS="$(mktemp)" +clean() { rm -f "$RESULTS"; } +trap clean EXIT + +for DECLARATION in "$ROOT"/test/e2e/*/benchmark.sh "$ROOT"/enterprise/e2e/*/benchmark.sh +do + [ -f "$DECLARATION" ] || continue + DIRECTORY="$(dirname "$DECLARATION")" + SUITE="${DIRECTORY#"$ROOT"/}" + + OUTPUT="$(mktemp -d)" + SERVER_PID= + + echo "Indexing $SUITE..." 1>&2 + "$INDEX" --skip-banner "$DIRECTORY/one.json" "$OUTPUT/index" >&2 2>/dev/null + + # What a suite needs to admit a caller is what it needs to serve one, so the + # same file the tests run under is the one a measurement runs under + if [ -f "$DIRECTORY/environment" ] + then + set -a + # shellcheck source=/dev/null + . "$DIRECTORY/environment" + set +a + fi + + "$SERVER" "$OUTPUT/index" "$PORT" > /dev/null 2>&1 & + SERVER_PID="$!" + + COUNTER=0 + while [ "$COUNTER" -lt 100 ] + do + if ! kill -0 "$SERVER_PID" 2>/dev/null + then + echo "Server for $SUITE exited before becoming ready" 1>&2 + exit 1 + fi + if nc -z localhost "$PORT" 2>/dev/null + then + break + fi + sleep 0.1 + COUNTER=$((COUNTER + 1)) + done + + # A suite names what it measures without naming itself, since where it lives + # is something this already knows + "$DECLARATION" "http://localhost:$PORT" \ + | jq --arg suite "$SUITE" 'map(.name |= "\($suite): \(.)")' >> "$RESULTS" + + kill "$SERVER_PID" 2>/dev/null || true + wait "$SERVER_PID" 2>/dev/null || true + rm -rf "$OUTPUT" +done + +jq --slurp 'add' < "$RESULTS" diff --git a/enterprise/e2e/auth-keys/benchmark.sh b/enterprise/e2e/auth-keys/benchmark.sh new file mode 100755 index 000000000..e13f6bdeb --- /dev/null +++ b/enterprise/e2e/auth-keys/benchmark.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +set -o errexit +set -o nounset + +BASE="$1" +HERE="$(cd "$(dirname "$0")" && pwd)" +MEASURE="$HERE/../../../benchmark/measure.py" + +# A path a policy governs, so what is measured includes admitting the caller +# rather than only serving them +"$MEASURE" --name "Schema Fetch (gated)" \ + --header "Authorization: Bearer $ONE_E2E_VAULT_KEY" \ + --url "$BASE/vault/secret.json" diff --git a/test/e2e/html/benchmark.sh b/test/e2e/html/benchmark.sh new file mode 100755 index 000000000..50f33f900 --- /dev/null +++ b/test/e2e/html/benchmark.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +set -o errexit +set -o nounset + +BASE="$1" +HERE="$(cd "$(dirname "$0")" && pwd)" +MEASURE="$HERE/../../../benchmark/measure.py" + +"$MEASURE" --name "Schema Fetch" --url "$BASE/test/bundling/single.json" From b0c548a41513383e38e36518cedf3bc2e2830dc0 Mon Sep 17 00:00:00 2001 From: Juan Cruz Viotti Date: Tue, 25 Aug 2026 17:26:01 -0300 Subject: [PATCH 2/7] Simpler Signed-off-by: Juan Cruz Viotti --- .github/actions/e2e/action.yml | 11 +++-- .github/workflows/ci.yml | 17 ++++++++ benchmark/merge.sh | 22 ++++++++++ benchmark/server.sh | 75 ---------------------------------- test/e2e/common.mk | 20 ++++++++- 5 files changed, 65 insertions(+), 80 deletions(-) create mode 100755 benchmark/merge.sh delete mode 100755 benchmark/server.sh diff --git a/.github/actions/e2e/action.yml b/.github/actions/e2e/action.yml index c6c259641..37c828f25 100644 --- a/.github/actions/e2e/action.yml +++ b/.github/actions/e2e/action.yml @@ -16,8 +16,11 @@ runs: shell: bash - name: E2E (${{ inputs.path }}) - run: > - make -C ${{ inputs.path }} - EDITION=${{ inputs.edition }} - HURL='docker run --rm --network=host --volume $$(pwd):/workspace --workdir /workspace ghcr.io/orange-opensource/hurl:8.0.1' + run: | + mkdir -p "$GITHUB_WORKSPACE/benchmark-results" + SLUG="$(printf '%s' '${{ inputs.path }}' | tr '/' '-')" + make -C ${{ inputs.path }} \ + EDITION=${{ inputs.edition }} \ + BENCHMARK_OUTPUT="$GITHUB_WORKSPACE/benchmark-results/$SLUG.json" \ + HURL='docker run --rm --network=host --volume $$(pwd):/workspace --workdir /workspace ghcr.io/orange-opensource/hurl:8.0.1' shell: bash diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9b248acff..f8ff4abfc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -174,6 +174,23 @@ jobs: comment-always: true fail-on-alert: false + - name: Benchmark (server) + run: ./benchmark/merge.sh benchmark-results > benchmark-server.json + - uses: benchmark-action/github-action-benchmark@v1 + if: github.event.pull_request.head.repo.fork == false + with: + name: Benchmark Server (${{ matrix.edition.name }}) + tool: customSmallerIsBetter + output-file-path: benchmark-server.json + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: ${{ github.event_name != 'pull_request' }} + benchmark-data-dir-path: benchmark/${{ matrix.edition.name }}/server + # Latency on a shared runner moves far more than an index does, so + # this is wide enough that only a real regression is heard + alert-threshold: '25%' + comment-always: true + fail-on-alert: false + website: runs-on: ubuntu-latest steps: diff --git a/benchmark/merge.sh b/benchmark/merge.sh new file mode 100755 index 000000000..c2b9cf552 --- /dev/null +++ b/benchmark/merge.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +set -o errexit +set -o nounset + +if [ "$#" -ne 1 ] +then + echo "Usage: $0 " 1>&2 + exit 1 +fi + +DIRECTORY="$1" + +# A run where no suite declared anything is not a failure, it is a run with +# nothing to say, and an empty array is how it says so +if [ ! -d "$DIRECTORY" ] || [ -z "$(ls -A "$DIRECTORY" 2>/dev/null)" ] +then + echo "[]" + exit 0 +fi + +jq --slurp 'add' "$DIRECTORY"/*.json diff --git a/benchmark/server.sh b/benchmark/server.sh deleted file mode 100755 index 00aa31150..000000000 --- a/benchmark/server.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/bin/sh - -set -o errexit -set -o nounset - -if [ "$#" -ne 2 ] -then - echo "Usage: $0 " 1>&2 - exit 1 -fi - -INDEX="$1" -SERVER="$2" -HERE="$(cd "$(dirname "$0")" && pwd)" -ROOT="$(cd "$HERE/.." && pwd)" -PORT="${BENCHMARK_PORT:-8199}" - -# A suite says what it wants measured by carrying a script that says it, so -# wiring a new one up is writing that script and nothing else here -RESULTS="$(mktemp)" -clean() { rm -f "$RESULTS"; } -trap clean EXIT - -for DECLARATION in "$ROOT"/test/e2e/*/benchmark.sh "$ROOT"/enterprise/e2e/*/benchmark.sh -do - [ -f "$DECLARATION" ] || continue - DIRECTORY="$(dirname "$DECLARATION")" - SUITE="${DIRECTORY#"$ROOT"/}" - - OUTPUT="$(mktemp -d)" - SERVER_PID= - - echo "Indexing $SUITE..." 1>&2 - "$INDEX" --skip-banner "$DIRECTORY/one.json" "$OUTPUT/index" >&2 2>/dev/null - - # What a suite needs to admit a caller is what it needs to serve one, so the - # same file the tests run under is the one a measurement runs under - if [ -f "$DIRECTORY/environment" ] - then - set -a - # shellcheck source=/dev/null - . "$DIRECTORY/environment" - set +a - fi - - "$SERVER" "$OUTPUT/index" "$PORT" > /dev/null 2>&1 & - SERVER_PID="$!" - - COUNTER=0 - while [ "$COUNTER" -lt 100 ] - do - if ! kill -0 "$SERVER_PID" 2>/dev/null - then - echo "Server for $SUITE exited before becoming ready" 1>&2 - exit 1 - fi - if nc -z localhost "$PORT" 2>/dev/null - then - break - fi - sleep 0.1 - COUNTER=$((COUNTER + 1)) - done - - # A suite names what it measures without naming itself, since where it lives - # is something this already knows - "$DECLARATION" "http://localhost:$PORT" \ - | jq --arg suite "$SUITE" 'map(.name |= "\($suite): \(.)")' >> "$RESULTS" - - kill "$SERVER_PID" 2>/dev/null || true - wait "$SERVER_PID" 2>/dev/null || true - rm -rf "$OUTPUT" -done - -jq --slurp 'add' < "$RESULTS" diff --git a/test/e2e/common.mk b/test/e2e/common.mk index cc1522d17..d26e8201f 100644 --- a/test/e2e/common.mk +++ b/test/e2e/common.mk @@ -3,6 +3,8 @@ HURL ?= hurl NPM ?= npm NPX ?= npx ROOT := $(dir $(lastword $(MAKEFILE_LIST)))../.. +# Where this suite lives, which is what a measurement it takes is named after +SUITE := $(patsubst $(abspath $(ROOT))/%,%,$(CURDIR)) COMPOSE = compose.yml BASE ?= http://localhost @@ -16,7 +18,7 @@ export EDITION all: $(MAKE) down $(MAKE) up - $(MAKE) test-hurl test-playwright; \ + $(MAKE) test-benchmark test-hurl test-playwright; \ status=$$?; $(MAKE) down; exit $$status .PHONY: up @@ -32,6 +34,22 @@ test-hurl: $(wildcard hurl/*.all.hurl) \ $(wildcard hurl/*.$(EDITION).hurl) +# A suite that carries a script saying what it wants measured is measured +# before its tests run, so what is timed is an instance warmed deliberately +# rather than one that has just served a whole test suite. Nothing is measured +# unless somewhere to put the answer was named, so an ordinary run pays nothing +.PHONY: test-benchmark +test-benchmark: +ifneq ($(wildcard benchmark.sh),) +ifneq ($(BENCHMARK_OUTPUT),) + if [ -f environment ]; then set -a; . ./environment; set +a; fi; \ + ./benchmark.sh $(BASE):$(PORT) > $(BENCHMARK_OUTPUT).part + jq --arg suite "$(SUITE)" 'map(.name |= "\($$suite): \(.)")' \ + < $(BENCHMARK_OUTPUT).part > $(BENCHMARK_OUTPUT) + rm -f $(BENCHMARK_OUTPUT).part +endif +endif + .PHONY: test-playwright test-playwright: ifneq ($(wildcard playwright/),) From a294d7633d715bd30bef9070ded94dceabf4c274 Mon Sep 17 00:00:00 2001 From: Juan Cruz Viotti Date: Tue, 25 Aug 2026 17:53:57 -0300 Subject: [PATCH 3/7] Single Signed-off-by: Juan Cruz Viotti --- .github/workflows/ci.yml | 30 +++++++++++------------------- 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f8ff4abfc..4e064c8a2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -157,36 +157,28 @@ jobs: edition: ${{ matrix.edition.name }} if: matrix.edition.name == 'enterprise' + # The suites measured themselves as they ran, so what is left is to take + # the indexer's own measurements and say all of it at once. This action + # keeps a checkout of the branch it records into, so it is called exactly + # once per job rather than once per kind of measurement - name: Benchmark run: | make docker-benchmark PRESET=Release ${{ matrix.edition.options }} - docker run --rm one-benchmark > benchmark.json + mkdir -p benchmark-results + docker run --rm one-benchmark > benchmark-results/index.json + ./benchmark/merge.sh benchmark-results > benchmark.json + cat benchmark.json - uses: benchmark-action/github-action-benchmark@v1 if: github.event.pull_request.head.repo.fork == false with: - name: Benchmark Index (${{ matrix.edition.name }}) + name: Benchmark (${{ matrix.edition.name }}) tool: customSmallerIsBetter output-file-path: benchmark.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: ${{ github.event_name != 'pull_request' }} benchmark-data-dir-path: benchmark/${{ matrix.edition.name }}/index - alert-threshold: '5%' - comment-always: true - fail-on-alert: false - - - name: Benchmark (server) - run: ./benchmark/merge.sh benchmark-results > benchmark-server.json - - uses: benchmark-action/github-action-benchmark@v1 - if: github.event.pull_request.head.repo.fork == false - with: - name: Benchmark Server (${{ matrix.edition.name }}) - tool: customSmallerIsBetter - output-file-path: benchmark-server.json - github-token: ${{ secrets.GITHUB_TOKEN }} - auto-push: ${{ github.event_name != 'pull_request' }} - benchmark-data-dir-path: benchmark/${{ matrix.edition.name }}/server - # Latency on a shared runner moves far more than an index does, so - # this is wide enough that only a real regression is heard + # Latency on a shared runner moves far more than an index does, and + # one threshold now covers both alert-threshold: '25%' comment-always: true fail-on-alert: false From b5a6d44c6aa78c1595125d8443a8c045c5358966 Mon Sep 17 00:00:00 2001 From: Juan Cruz Viotti Date: Tue, 25 Aug 2026 18:17:50 -0300 Subject: [PATCH 4/7] More Signed-off-by: Juan Cruz Viotti --- .github/workflows/ci.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4e064c8a2..026972283 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -171,7 +171,12 @@ jobs: - uses: benchmark-action/github-action-benchmark@v1 if: github.event.pull_request.head.repo.fork == false with: - name: Benchmark (${{ matrix.edition.name }}) + # TODO: Rename to `Benchmark ()`, as this now carries + # server measurements too. The name keys the series stored on + # `gh-pages`, and only pushes to `main` write there, so renaming + # starts an empty series and suppresses the comparison comment until + # the first push after the rename lands + name: Benchmark Index (${{ matrix.edition.name }}) tool: customSmallerIsBetter output-file-path: benchmark.json github-token: ${{ secrets.GITHUB_TOKEN }} From e1197ca722246a2ce97021c021fe607d0b6f4373 Mon Sep 17 00:00:00 2001 From: Juan Cruz Viotti Date: Wed, 26 Aug 2026 10:08:57 -0300 Subject: [PATCH 5/7] More Signed-off-by: Juan Cruz Viotti --- .github/actions/e2e/action.yml | 2 +- enterprise/e2e/auth-keys/benchmark.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/e2e/action.yml b/.github/actions/e2e/action.yml index 37c828f25..c45cb9d11 100644 --- a/.github/actions/e2e/action.yml +++ b/.github/actions/e2e/action.yml @@ -21,6 +21,6 @@ runs: SLUG="$(printf '%s' '${{ inputs.path }}' | tr '/' '-')" make -C ${{ inputs.path }} \ EDITION=${{ inputs.edition }} \ - BENCHMARK_OUTPUT="$GITHUB_WORKSPACE/benchmark-results/$SLUG.json" \ + BENCHMARK_OUTPUT="$GITHUB_WORKSPACE/benchmark-results/server-$SLUG.json" \ HURL='docker run --rm --network=host --volume $$(pwd):/workspace --workdir /workspace ghcr.io/orange-opensource/hurl:8.0.1' shell: bash diff --git a/enterprise/e2e/auth-keys/benchmark.sh b/enterprise/e2e/auth-keys/benchmark.sh index e13f6bdeb..0d44fd503 100755 --- a/enterprise/e2e/auth-keys/benchmark.sh +++ b/enterprise/e2e/auth-keys/benchmark.sh @@ -9,6 +9,6 @@ MEASURE="$HERE/../../../benchmark/measure.py" # A path a policy governs, so what is measured includes admitting the caller # rather than only serving them -"$MEASURE" --name "Schema Fetch (gated)" \ +"$MEASURE" --name "Schema Gated" \ --header "Authorization: Bearer $ONE_E2E_VAULT_KEY" \ --url "$BASE/vault/secret.json" From f9d03a8cb86da68d568eaebf3434daba01b8587d Mon Sep 17 00:00:00 2001 From: Juan Cruz Viotti Date: Wed, 26 Aug 2026 10:17:16 -0300 Subject: [PATCH 6/7] Simpler Signed-off-by: Juan Cruz Viotti --- .github/workflows/ci.yml | 4 +- .gitignore | 1 + benchmark/Dockerfile | 5 +- benchmark/entrypoint.sh | 22 +++++++ benchmark/measure.py | 89 ++++++++++++--------------- enterprise/e2e/auth-keys/benchmark.py | 20 ++++++ enterprise/e2e/auth-keys/benchmark.sh | 14 ----- test/e2e/common.mk | 10 +-- test/e2e/html/benchmark.py | 9 +++ test/e2e/html/benchmark.sh | 10 --- 10 files changed, 104 insertions(+), 80 deletions(-) create mode 100755 benchmark/entrypoint.sh create mode 100755 enterprise/e2e/auth-keys/benchmark.py delete mode 100755 enterprise/e2e/auth-keys/benchmark.sh create mode 100755 test/e2e/html/benchmark.py delete mode 100755 test/e2e/html/benchmark.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 026972283..6b3af73b2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -165,8 +165,8 @@ jobs: run: | make docker-benchmark PRESET=Release ${{ matrix.edition.options }} mkdir -p benchmark-results - docker run --rm one-benchmark > benchmark-results/index.json - ./benchmark/merge.sh benchmark-results > benchmark.json + docker run --rm --volume "$PWD/benchmark-results:/results:ro" \ + one-benchmark > benchmark.json cat benchmark.json - uses: benchmark-action/github-action-benchmark@v1 if: github.event.pull_request.head.repo.fork == false diff --git a/.gitignore b/.gitignore index 01cc628e4..c8824b7fa 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ Brewfile.lock.json .DS_Store /node_modules /.cache +__pycache__ diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile index 2b47be1a5..e97815684 100644 --- a/benchmark/Dockerfile +++ b/benchmark/Dockerfile @@ -6,5 +6,8 @@ COPY benchmark/index-add-update-rebuild.sh /benchmark/index-add-update-rebuild.s COPY benchmark/index-custom-meta-schema.sh /benchmark/index-custom-meta-schema.sh COPY benchmark/index-n.sh /benchmark/index-n.sh COPY benchmark/index-ref-fanout.sh /benchmark/index-ref-fanout.sh +COPY benchmark/merge.sh /benchmark/merge.sh +COPY benchmark/entrypoint.sh /benchmark/entrypoint.sh RUN /benchmark/index.sh /usr/bin/sourcemeta-one-index > /benchmark.json -ENTRYPOINT [ "cat", "/benchmark.json" ] +# Mount whatever the suites measured at /results to have it merged in +ENTRYPOINT [ "/benchmark/entrypoint.sh" ] diff --git a/benchmark/entrypoint.sh b/benchmark/entrypoint.sh new file mode 100755 index 000000000..8f4a0e803 --- /dev/null +++ b/benchmark/entrypoint.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +set -o errexit +set -o nounset + +# What the indexer measured of itself was taken when this image was built. What +# the suites measured of a running instance was taken on the way here, and is +# mounted rather than built in, since it could only be taken with a server up. +# Both are said at once, with the indexer first, so the answer reads as one +# group after another +RESULTS="$(mktemp -d)" +clean() { rm -rf "$RESULTS"; } +trap clean EXIT + +cp /benchmark.json "$RESULTS/index.json" + +if [ -d /results ] +then + find /results -maxdepth 1 -name '*.json' -exec cp {} "$RESULTS/" \; +fi + +/benchmark/merge.sh "$RESULTS" diff --git a/benchmark/measure.py b/benchmark/measure.py index e0c5530a1..4e3ac8aca 100755 --- a/benchmark/measure.py +++ b/benchmark/measure.py @@ -1,6 +1,11 @@ #!/usr/bin/env python3 -"""Measure how long one endpoint takes to answer. +"""Measure how long endpoints take to answer. + +A suite declares what it wants measured by importing `run` from here and +naming the requests. Where the suite lives and where the instance is listening +arrive as arguments, so a declaration says what to measure and nothing about +where it is being measured from. The connection is opened once and kept, so what is timed is answering a request rather than establishing a conversation. Requests are made one after @@ -8,84 +13,72 @@ how long one caller waits, not how many callers can be served at once. """ -import argparse import http.client import json import sys import time +WARMUP = 1000 +COUNT = 20000 -def percentile(samples, fraction): - return samples[min(int(len(samples) * fraction), len(samples) - 1)] +def _percentile(samples, fraction): + return samples[min(int(len(samples) * fraction), len(samples) - 1)] -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--name", required=True) - parser.add_argument("--url", required=True) - parser.add_argument("--method", default="GET") - parser.add_argument("--header", action="append", default=[]) - parser.add_argument("--body", default=None) - parser.add_argument("--count", type=int, default=20000) - parser.add_argument("--warmup", type=int, default=1000) - options = parser.parse_args() - _, _, rest = options.url.partition("//") - authority, _, path = rest.partition("/") +def measure(suite, base, name, path, method="GET", headers=None, body=None): + _, _, rest = base.partition("//") + authority, _, prefix = rest.partition("/") host, _, port = authority.partition(":") - path = "/" + path + target = "/" + prefix + path if prefix else path - headers = {} - for header in options.header: - key, _, value = header.partition(":") - headers[key.strip()] = value.strip() - - body = options.body.encode() if options.body else None + payload = body.encode() if body else None connection = http.client.HTTPConnection(host, int(port or 80)) connection.connect() def once(): - connection.request(options.method, path, body=body, headers=headers) + connection.request(method, target, body=payload, headers=headers or {}) response = connection.getresponse() response.read() return response.status # Answering the first time builds what answering afterwards reuses, so # what is measured is a warm instance rather than a cold one - for _ in range(options.warmup): + for _ in range(WARMUP): status = once() if status >= 400: - sys.exit(f"{options.name}: warm up got HTTP {status}") + sys.exit(f"{suite}: {name}: warm up got HTTP {status}") samples = [] - for _ in range(options.count): + for _ in range(COUNT): start = time.perf_counter_ns() status = once() samples.append((time.perf_counter_ns() - start) / 1000.0) # A run that was refused measured something other than what it set out # to, so it stops rather than reporting a number nobody can trust if status >= 400: - sys.exit(f"{options.name}: got HTTP {status}") + sys.exit(f"{suite}: {name}: got HTTP {status}") + connection.close() samples.sort() - json.dump( - [ - { - "name": f"{options.name} (p50)", - "unit": "us", - "value": round(percentile(samples, 0.50)), - }, - { - "name": f"{options.name} (p99)", - "unit": "us", - "value": round(percentile(samples, 0.99)), - }, - ], - sys.stdout, - indent=2, - ) + return [ + { + "name": f"{suite}: {name} ({label})", + "unit": "us", + "value": round(_percentile(samples, fraction)), + } + for label, fraction in (("p50", 0.50), ("p99", 0.99)) + ] + + +def run(measurements): + if len(sys.argv) != 3: + sys.exit(f"Usage: {sys.argv[0]} ") + + suite, base = sys.argv[1], sys.argv[2] + entries = [] + for measurement in measurements: + entries.extend(measure(suite, base, **measurement)) + + json.dump(entries, sys.stdout, indent=2) sys.stdout.write("\n") - - -if __name__ == "__main__": - main() diff --git a/enterprise/e2e/auth-keys/benchmark.py b/enterprise/e2e/auth-keys/benchmark.py new file mode 100755 index 000000000..b462eda36 --- /dev/null +++ b/enterprise/e2e/auth-keys/benchmark.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 + +import os + +from measure import run + +# A path a policy governs, so what is measured includes admitting the caller +# rather than only serving them. The credential comes from the same file the +# tests run under, and its absence is loud rather than a refusal to explain +run( + [ + { + "name": "Schema Gated", + "path": "/vault/secret.json", + "headers": { + "Authorization": f"Bearer {os.environ['ONE_E2E_VAULT_KEY']}" + }, + }, + ] +) diff --git a/enterprise/e2e/auth-keys/benchmark.sh b/enterprise/e2e/auth-keys/benchmark.sh deleted file mode 100755 index 0d44fd503..000000000 --- a/enterprise/e2e/auth-keys/benchmark.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh - -set -o errexit -set -o nounset - -BASE="$1" -HERE="$(cd "$(dirname "$0")" && pwd)" -MEASURE="$HERE/../../../benchmark/measure.py" - -# A path a policy governs, so what is measured includes admitting the caller -# rather than only serving them -"$MEASURE" --name "Schema Gated" \ - --header "Authorization: Bearer $ONE_E2E_VAULT_KEY" \ - --url "$BASE/vault/secret.json" diff --git a/test/e2e/common.mk b/test/e2e/common.mk index d26e8201f..b886e90d4 100644 --- a/test/e2e/common.mk +++ b/test/e2e/common.mk @@ -40,13 +40,13 @@ test-hurl: # unless somewhere to put the answer was named, so an ordinary run pays nothing .PHONY: test-benchmark test-benchmark: -ifneq ($(wildcard benchmark.sh),) +ifneq ($(wildcard benchmark.py),) ifneq ($(BENCHMARK_OUTPUT),) if [ -f environment ]; then set -a; . ./environment; set +a; fi; \ - ./benchmark.sh $(BASE):$(PORT) > $(BENCHMARK_OUTPUT).part - jq --arg suite "$(SUITE)" 'map(.name |= "\($$suite): \(.)")' \ - < $(BENCHMARK_OUTPUT).part > $(BENCHMARK_OUTPUT) - rm -f $(BENCHMARK_OUTPUT).part + PYTHONPATH=$(abspath $(ROOT))/benchmark \ + python3 ./benchmark.py $(SUITE) $(BASE):$(PORT) \ + > $(BENCHMARK_OUTPUT).part + mv $(BENCHMARK_OUTPUT).part $(BENCHMARK_OUTPUT) endif endif diff --git a/test/e2e/html/benchmark.py b/test/e2e/html/benchmark.py new file mode 100755 index 000000000..587a9ad68 --- /dev/null +++ b/test/e2e/html/benchmark.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python3 + +from measure import run + +run( + [ + {"name": "Schema Fetch", "path": "/test/bundling/single.json"}, + ] +) diff --git a/test/e2e/html/benchmark.sh b/test/e2e/html/benchmark.sh deleted file mode 100755 index 50f33f900..000000000 --- a/test/e2e/html/benchmark.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh - -set -o errexit -set -o nounset - -BASE="$1" -HERE="$(cd "$(dirname "$0")" && pwd)" -MEASURE="$HERE/../../../benchmark/measure.py" - -"$MEASURE" --name "Schema Fetch" --url "$BASE/test/bundling/single.json" From 282613a9d763423b5ce833fee160b4127e98763c Mon Sep 17 00:00:00 2001 From: Juan Cruz Viotti Date: Wed, 26 Aug 2026 10:35:03 -0300 Subject: [PATCH 7/7] More Signed-off-by: Juan Cruz Viotti --- benchmark/Dockerfile | 1 - benchmark/entrypoint.sh | 2 +- benchmark/merge.sh | 22 ---------------------- 3 files changed, 1 insertion(+), 24 deletions(-) delete mode 100755 benchmark/merge.sh diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile index e97815684..846808e9e 100644 --- a/benchmark/Dockerfile +++ b/benchmark/Dockerfile @@ -6,7 +6,6 @@ COPY benchmark/index-add-update-rebuild.sh /benchmark/index-add-update-rebuild.s COPY benchmark/index-custom-meta-schema.sh /benchmark/index-custom-meta-schema.sh COPY benchmark/index-n.sh /benchmark/index-n.sh COPY benchmark/index-ref-fanout.sh /benchmark/index-ref-fanout.sh -COPY benchmark/merge.sh /benchmark/merge.sh COPY benchmark/entrypoint.sh /benchmark/entrypoint.sh RUN /benchmark/index.sh /usr/bin/sourcemeta-one-index > /benchmark.json # Mount whatever the suites measured at /results to have it merged in diff --git a/benchmark/entrypoint.sh b/benchmark/entrypoint.sh index 8f4a0e803..713fa39c5 100755 --- a/benchmark/entrypoint.sh +++ b/benchmark/entrypoint.sh @@ -19,4 +19,4 @@ then find /results -maxdepth 1 -name '*.json' -exec cp {} "$RESULTS/" \; fi -/benchmark/merge.sh "$RESULTS" +jq --slurp 'add' "$RESULTS"/*.json diff --git a/benchmark/merge.sh b/benchmark/merge.sh deleted file mode 100755 index c2b9cf552..000000000 --- a/benchmark/merge.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/sh - -set -o errexit -set -o nounset - -if [ "$#" -ne 1 ] -then - echo "Usage: $0 " 1>&2 - exit 1 -fi - -DIRECTORY="$1" - -# A run where no suite declared anything is not a failure, it is a run with -# nothing to say, and an empty array is how it says so -if [ ! -d "$DIRECTORY" ] || [ -z "$(ls -A "$DIRECTORY" 2>/dev/null)" ] -then - echo "[]" - exit 0 -fi - -jq --slurp 'add' "$DIRECTORY"/*.json