diff --git a/.gitignore b/.gitignore index 2278803..cbb2426 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ # them; commit one only when a document refers to it. /benchmarks/analysis/out/ /benchmarks/analysis/.venv/ +/benchmarks/analysis/__pycache__/ # Editor/IDE .idea/ diff --git a/CLAUDE.md b/CLAUDE.md index f0763c8..ed56d7a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -299,6 +299,19 @@ path: the byte-counting `net.Conn` must never end up in a production transfer. Phases are wall clock, operation samples are cumulative across the parallel upload workers; do not present the two as the same kind of number. +`benchmarks/analysis/` is the optional Python layer that reads those documents +and draws them (`benchdata.py` knows the schema, `plot.py` draws, +`test_plot.py` checks both offline). It consumes canonical files only and must +never become an input to the harness. Two rules make it survive the stored +history: every field it reads is optional, because the results on disk span +several schema versions, and every figure prints its provenance plus the +caveats the stored file justifies (an unshaped link profile, failed repeats, +refused connections). Its self-checks load *every* result committed under +`benchmarks/`, so a reader that only understands the newest schema fails +instead of silently dropping the older files. `out/` is ignored except for the +gallery `benchmarks/analysis/README.md` refers to; regenerate those files with +the commands printed under each image when a new sweep or release lands. + ## Behavior worth knowing before you change it - Uploads from **Windows runners** have no local permission bits to mirror: diff --git a/benchmarks/README.md b/benchmarks/README.md index eef9f2e..7c3218f 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -17,7 +17,10 @@ their measurements and outputs diffed, so every result stored here reads the same way whichever implementation wrote it. The analysis in [`analysis/`](analysis/) reads these files and is optional; the -commands above need nothing from it. +commands above need nothing from it. It plots the sweep as a grid and as +curves, where a deployment's wall clock goes, what the round-trips cost, the +delete sweeps, the measurement against the link's own control, and what +easySFTP picks for itself against what a sweep would have picked. These numbers set no threshold and fail no build. They exist to see where the time goes (issues #158 and #169), so read them as one host's behaviour on @@ -33,7 +36,7 @@ one day, not as a guarantee. | A connections/concurrency sweep | `matrix/matrix-*.md` / `.json` / `.csv` | | Runtime and throughput across releases | `trend.csv` | | An older result | `archive//…` | -| A sweep as a picture | `analysis/README.md` | +| A sweep as a picture, where the time went, what auto picked | `analysis/README.md` | ## Layout diff --git a/benchmarks/analysis/README.md b/benchmarks/analysis/README.md index 12d2fa3..f45bc21 100644 --- a/benchmarks/analysis/README.md +++ b/benchmarks/analysis/README.md @@ -1,9 +1,11 @@ # Benchmark analysis (optional) -Downstream reading of benchmark results: heatmaps of a sweep, throughput -trends across releases. This directory is **not** part of running, storing or -validating a benchmark. `cmd/easysftp-bench` does all of that and needs nothing -from here (issue #190). +Downstream reading of benchmark results: the sweep as a grid and as curves, +where a deployment's wall clock goes, what the round-trips cost, what easySFTP +picks for itself against what a sweep would have picked, and throughput across +releases. This directory is **not** part of running, storing or validating a +benchmark. `cmd/easysftp-bench` does all of that and needs nothing from here +(issue #190). The relationship is one way. Everything in here consumes the canonical outputs the Go harness writes, and nothing in here may become an input to them: @@ -18,40 +20,78 @@ cmd/easysftp-bench -> results.json / matrix.json / *.csv / index.json / trend. A chart is a reading of the data, never a replacement for it. If a plot and the JSON disagree, the JSON is right. Do not commit generated plots by default; `out/` is ignored. Attach them to the run or the issue that needed them, and -commit one only when a document refers to it. +commit one only when a document refers to it. The gallery below is that +exception, and the commands under it regenerate exactly those files. + +``` +benchdata.py reads the stored documents; knows the schema, not matplotlib +plot.py draws; one function per command +test_plot.py self-checks, run offline against what is stored here +out/ output, ignored except for the gallery below +``` ## Use ```bash python -m venv .venv && . .venv/bin/activate # Windows: .venv\Scripts\activate -pip install -r benchmarks/analysis/requirements.txt ``` -A heatmap per scenario, build and link profile, from the newest stored sweep: - ```bash -python benchmarks/analysis/plot.py heatmap +pip install -r benchmarks/analysis/requirements.txt ``` -A specific sweep, and the release trend: +Every command takes zero or more stored files and falls back to the newest one +of the kind it needs (`benchmarks/latest.json` for a run, the newest +`benchmarks/matrix/matrix-*` for a sweep). All of them write into +`benchmarks/analysis/out/` unless `--out` says otherwise and print the path of +every file they wrote. + +| Command | What it draws | Reads | +|---|---|---| +| `heatmap` | throughput over the connections x concurrency grid | `matrix.csv` | +| `scaling` | the same cells as curves, against the linear reference | `matrix.json` | +| `auto` | what easySFTP picks for itself against the best cell | `matrix.json` | +| `canary` | whether the line held still for the whole sweep | `matrix.json` | +| `phases` | where the wall clock of a deployment goes | either | +| `operations` | per round-trip cost and share of the work | either | +| `deletes` | the delete sweeps, the only measurement of deletion | either | +| `link` | measured throughput against the link's own control | either | +| `trend` | throughput per scenario across releases | `trend.csv` | +| `report` | everything the given files support, plus `report.md` | both | + +"either" means a stored run (`release-*.json`, `manual-*.json`, `latest.json`) +or a sweep. For a run it draws the measured result; for a sweep it draws the +**best cell** of each scenario and profile, taken from `scaling[].best`, since +a sweep has hundreds of cells and only the one that won is worth taking apart. + +Everything at once, for the newest release result and the newest sweep: ```bash -python benchmarks/analysis/plot.py heatmap benchmarks/matrix/matrix-20260730T224549Z-main.csv +python benchmarks/analysis/plot.py report ``` -```bash -python benchmarks/analysis/plot.py trend -``` +Common options, valid on every command: -Both write PNGs into `benchmarks/analysis/out/` unless `--out` says otherwise, -and print the path of every file they wrote. +| Option | Effect | +|---|---| +| `--scenario small,large` | only these scenarios | +| `--profile baseline` | only these link profiles | +| `--out DIR` | write somewhere else | +| `--format png\|svg\|pdf`, `--dpi N` | how to write it | +| `--metric mib_per_s\|files_per_s\|median_ms` | what `heatmap` colors and ranks by | +| `--include-deletes` | `phases`: add the delete sweeps as their own bars | ## Reading the plots +Every figure carries its provenance under it, and any caveat the stored file +justifies in red: a run measured with `tc` unavailable has link profile names +that say what was *asked for*, not what happened, and a chart cannot tell that +by itself. + A **heatmap** cell is one measured configuration: `connections` down, `concurrency` across, colored by throughput (brighter is faster) and labelled -with the median duration. Two things to look for, and they are the reason the -grid exists at all: +with the median duration. The fastest cell is boxed in red. Two things to look +for, and they are the reason the grid exists at all: - Where the bright region stops. Scaling that flattens along `concurrency` but keeps improving along `connections` means the run is bounded per connection, @@ -61,15 +101,132 @@ grid exists at all: the numbers until the axis is extended. `matrix.json` says the same thing in `scaling[].best_at_axis_max`, and that field is the authority. -A **trend** point is one stored release measurement of one scenario. Compare -points only within the same `environment` and, since issue #184, the same -`link`: a faster line and a faster easySFTP look identical here otherwise. The -trend chart cannot tell them apart and does not try. +**scaling** is the same cells read as curves, against the axis that actually +varied for that scenario (`scenario.AxisFor` caps both axes at the file count, +so `single` sweeps only `request_concurrency` and `small` only the other two). +The dotted line is perfect scaling from the leftmost point of the same series; +the gap to it is what the setting is worth. The y axis is scaled to what was +measured, so the reference normally leaves the frame. + +**auto** scores easySFTP's own choice against the fastest cell of the same +scenario and profile. The picked coordinate is read back from the run's own +counters, so it is what easySFTP did, not what the plot assumes. A policy +within roughly 15% on every profile is defensible, one that only wins on the +house line is not (issue #184, phase 5; the policy itself is issue #156). + +**canary** is one fixed cell repeated at the start, the middle and the end of +each profile's grid. A spread larger than the deltas the sweep is read for +means the server or the line moved during the run, and the whole run is a poor +comparison basis. + +**phases** and **operations** are two different kinds of number and the plots +say so. A phase is wall clock and the bars add up to roughly the run's +duration. An operation total is cumulative across the parallel upload workers, +so it is normally *larger* than the phase it happened in: read it for its share +of the work and its per call cost, never as elapsed time. `file_upload` is the +umbrella around the `sftp_*` calls of one file, so its total contains theirs +and the share is taken over the others only. + +**deletes** is the clean deployment measured before every run (issue #184, +phase 4). It is a pure delete sweep and the only measurement of deletion in the +harness. Everything outside the per-file upload path runs over one connection +(`session.do`), so a rate that does not move with `connections` or +`concurrency` is the expected shape, not a measurement error. + +**link** puts the measured throughput next to the control `cmd/linkprobe` took +over the same path with x/crypto/ssh and pkg/sftp. That control imports nothing +from `internal/uploader`, which is what makes it one: read against it, a slower +easySFTP and a slower line stop looking the same. + +A **trend** point is one stored release measurement of one scenario, with a +marker per runner. Compare points only within the same `environment` and, since +issue #184, the same `link`: a faster line and a faster easySFTP look identical +here otherwise. The trend chart cannot tell them apart and does not try. + +## The committed gallery + +`out/` is ignored, but these files are committed because this page refers to +them. They are the newest stored sweep +(`matrix/matrix-20260816T125322Z-main`) and the newest release result +(`latest.json`, v3.5.0), and they are regenerated by exactly the commands +under each image. + +| The grid | Where scaling stops paying | +|---|---| +| ![heatmap](out/heatmap-matrix-20260816T125322Z-main-small-candidate-baseline-default.png) | ![scaling](out/scaling-matrix-20260816T125322Z-main-small-candidate-baseline-all.png) | + +```bash +python benchmarks/analysis/plot.py heatmap benchmarks/matrix/matrix-20260816T125322Z-main.csv --scenario small --profile baseline +``` + +```bash +python benchmarks/analysis/plot.py scaling benchmarks/matrix/matrix-20260816T125322Z-main.json --scenario small --profile baseline +``` + +| The policy against the grid | Did the line hold still | +|---|---| +| ![auto](out/auto-matrix-20260816T125322Z-main.png) | ![canary](out/canary-matrix-20260816T125322Z-main.png) | + +```bash +python benchmarks/analysis/plot.py auto benchmarks/matrix/matrix-20260816T125322Z-main.json +``` + +```bash +python benchmarks/analysis/plot.py canary benchmarks/matrix/matrix-20260816T125322Z-main.json +``` + +| Against the line itself | Deletion | +|---|---| +| ![link](out/link-matrix-20260816T125322Z-main.png) | ![deletes](out/deletes-matrix-20260816T125322Z-main-small.png) | + +```bash +python benchmarks/analysis/plot.py link benchmarks/matrix/matrix-20260816T125322Z-main.json +``` + +```bash +python benchmarks/analysis/plot.py deletes benchmarks/matrix/matrix-20260816T125322Z-main.json --scenario small +``` + +| Where the wall clock goes | What the round-trips cost | +|---|---| +| ![phases](out/phases-release-v3.5.0.png) | ![operations](out/operations-release-v3.5.0-small-baseline.png) | + +```bash +python benchmarks/analysis/plot.py phases --include-deletes +``` + +```bash +python benchmarks/analysis/plot.py operations --scenario small +``` + +Across releases: + +![trend](out/trend-releases.png) + +```bash +python benchmarks/analysis/plot.py trend +``` + +## Self-checks + +```bash +python -m unittest discover -s benchmarks/analysis +``` + +They need matplotlib and the stored results, nothing else: no benchmark is run +and no server is contacted. Two things are asserted, and they are the two ways +this layer breaks. Every result committed under `benchmarks/` is loaded, so a +reader that only understands the newest schema version fails instead of +silently dropping the older files; and every command is drawn into a temporary +directory, so a plot that raises on a field that moved fails here rather than +on the day someone needs the picture. ## Adding to this Keep it reproducible and keep it optional: read only the canonical files, pin what you import in `requirements.txt`, and do not reach into `internal/` or -shell out to the harness. If an analysis turns out to be something the project -needs on every run, that is an argument for putting it in the Go harness, not -for making this directory a dependency of it. +shell out to the harness. Schema knowledge belongs in `benchdata.py`, drawing +in `plot.py`, and a new command needs a row in the table above, a paragraph +under "Reading the plots" and a case in `test_plot.py`. If an analysis turns +out to be something the project needs on every run, that is an argument for +putting it in the Go harness, not for making this directory a dependency of it. diff --git a/benchmarks/analysis/benchdata.py b/benchmarks/analysis/benchdata.py new file mode 100644 index 0000000..e506751 --- /dev/null +++ b/benchmarks/analysis/benchdata.py @@ -0,0 +1,430 @@ +#!/usr/bin/env python3 +"""Reading the canonical benchmark documents. + +`plot.py` draws; this module reads. It knows the shapes `cmd/easysftp-bench` +writes (`results.json`, `matrix.json`, their CSV exports and `trend.csv`) and +nothing about matplotlib, so a new plot does not have to relearn the schema. + +Two rules hold everywhere in here, both learned from the stored files: + +- Every field is optional. The results span several schema versions: version 1 + (up to v3.3.2) has no `link`, no `phases`, no `operations`; the sweeps from + before issue #184 have no `link_profile`, no `request_concurrency` and no + `auto` rows. Asking for a key that is not there is how a plot silently drops + older results, or worse, shows the wrong axis. +- Nothing here writes into `benchmarks/` and nothing here runs a benchmark. + The relationship is one way (`benchmarks/analysis/README.md`). +""" + +from __future__ import annotations + +import csv +import json +from pathlib import Path + +REPO = Path(__file__).resolve().parents[2] +BENCHMARKS = REPO / "benchmarks" +DEFAULT_OUT = BENCHMARKS / "analysis" / "out" + +# The phases cmd/easysftp-bench records, in the order benchmarks/README.md +# lists them. Fixed so one phase keeps one color across every figure of a +# report, which is the only reason two stacked bars can be compared by eye. +PHASE_ORDER = [ + "connect", + "local_scan", + "remote_scan", + "hash", + "create_dirs", + "sweep_stale_temps", + "upload", + "delete_sweep", + "manifest_read", + "manifest_write", + "prune_dirs", + "cleanup", +] + +# Same idea for the per round-trip operations. `file_upload` is deliberately +# first: it is the umbrella around the sftp_* calls of one file, so its total +# contains theirs and the two must never be added together. +OPERATION_ORDER = [ + "file_upload", + "sftp_open", + "sftp_write", + "sftp_chmod", + "sftp_setstat", + "sftp_rename", + "sftp_remove", + "sftp_rmdir", + "sftp_mkdirall", + "sftp_stat", + "sftp_readdir", + "ssh_connect", +] + +# An operation that wraps other operations. Its total is not additional time. +UMBRELLA_OPERATIONS = {"file_upload"} + +# The three settings a matrix cell is a point in. +AXIS_KEYS = ("concurrency", "connections", "request_concurrency") + + +# -------------------------------------------------------------------------- +# Small shared helpers +# -------------------------------------------------------------------------- + + +def rows(path): + """A CSV as dicts, with empty strings left alone: '' is 'not recorded'.""" + with open(path, newline="", encoding="utf-8") as handle: + return list(csv.DictReader(handle)) + + +def number(row, column): + """A numeric column or field, or None when it is absent or empty.""" + value = row.get(column) + if value is None: + return None + if isinstance(value, (int, float)): + return float(value) + text = str(value).strip() + if not text: + return None + try: + return float(text) + except ValueError: + return None + + +def slug(text): + keep = [c if c.isalnum() or c in "-_." else "-" for c in str(text)] + return "".join(keep).strip("-") or "none" + + +def human_ms(value): + if value is None: + return "?" + if value < 1000: + return f"{value:.0f} ms" + if value < 60_000: + return f"{value / 1000:.1f} s" + return f"{value / 60_000:.1f} min" + + +def axis_label(key, value): + """A cell coordinate as it should be read: null request_concurrency is + 'default', which is not the same as any measured number.""" + if value is None or value == "": + return "default" if key == "request_concurrency" else "?" + if isinstance(value, float) and value.is_integer(): + return str(int(value)) + return str(value) + + +def axis_layout(points, keys=AXIS_KEYS): + """Which of the three settings to put where, from what actually varies. + + A sweep does not use the same axes for every scenario: `scenario.AxisFor` + caps both axes at the file count and `scenario.SweepsRequests` only lets + request_concurrency through for large files, so `single` varies only + request_concurrency while `small` varies only the other two. Hard-coding + concurrency onto x draws a one-point line for half the scenarios. + + Returns (x_key, series_key, facet_keys): the widest axis across the bottom, + the next widest as one line each, and any remaining axis that still varies + as a separate figure. Ties keep the order in `keys`. + """ + ranked = sorted( + ( + (len({point.get(key) for point in points}), -index, key) + for index, key in enumerate(keys) + ), + reverse=True, + ) + order = [key for _, _, key in ranked] + facets = [ + key for key in order[2:] if len({point.get(key) for point in points}) > 1 + ] + return order[0], order[1], facets + + +# -------------------------------------------------------------------------- +# The stored documents +# -------------------------------------------------------------------------- + + +class Document: + """One stored result: the envelope plus the measurement under `.benchmark`. + + The accessors below are the whole schema surface the plots are allowed to + know about. Add to them rather than reaching into `.benchmark` from a plot, + so the version tolerance stays in one place. + """ + + def __init__(self, path, envelope): + self.path = Path(path) + self.envelope = envelope if isinstance(envelope, dict) else {} + benchmark = self.envelope.get("benchmark") + # A bare results.json/matrix.json (a run artifact that was never + # stored) has no envelope around it. Both are worth plotting. + if not isinstance(benchmark, dict): + benchmark = self.envelope + self.envelope = {} + self.benchmark = benchmark + + # -- provenance --------------------------------------------------------- + + @property + def kind(self): + kind = self.envelope.get("kind") or self.benchmark.get("benchmark_kind") + if kind: + return kind + return "matrix" if self.benchmark.get("cells") else "standard" + + @property + def is_matrix(self): + return bool(self.benchmark.get("cells")) or self.kind == "matrix" + + @property + def ref(self): + return self.benchmark.get("candidate_ref") or "?" + + @property + def runner(self): + return self.benchmark.get("runner") or "?" + + @property + def recorded_at(self): + return self.envelope.get("recorded_at") or "" + + @property + def scenarios(self): + return self.benchmark.get("scenarios") or {} + + # -- measurements ------------------------------------------------------- + + @property + def results(self): + """The upload aggregates: results[] for a standard run, cells[] for a + sweep. One row per configuration in both cases.""" + return self.benchmark.get("results") or self.benchmark.get("cells") or [] + + @property + def deletes(self): + return self.benchmark.get("deletes") or [] + + @property + def scaling(self): + return self.benchmark.get("scaling") or [] + + @property + def auto(self): + return self.benchmark.get("auto") or [] + + @property + def canary(self): + return self.benchmark.get("canary") or [] + + @property + def link(self): + return self.benchmark.get("link") or {} + + @property + def probes(self): + return self.link.get("probes") or [] + + @property + def shaping(self): + return self.link.get("shaping") or {} + + # -- derived ------------------------------------------------------------ + + def profile(self, row): + return row.get("link_profile") or "baseline" + + def throughput(self, row): + """MiB/s and files/s, computed when the row does not carry them. + + Sweep cells written before the CSV export did carry both; the delete + rows never do. Recomputing from bytes and the median keeps one formula + in one place instead of three that drift. + """ + mib = number(row, "mib_per_s") + files_per_s = number(row, "files_per_s") + median = number(row, "median_ms") + if median and median > 0: + if mib is None and number(row, "bytes") is not None: + mib = number(row, "bytes") / (1024 * 1024) / (median / 1000) + if files_per_s is None and number(row, "files") is not None: + files_per_s = number(row, "files") / (median / 1000) + return mib, files_per_s + + def controls(self, profile): + """What the link probe measured for one profile, averaged over the + probes taken for it (normally one at the start and one at the end). + + This is the control: measured with x/crypto/ssh and pkg/sftp, not with + easySFTP's uploader. Comparing a run against it separates the line from + easySFTP, which is the only reason it exists. + """ + taken = [p for p in self.probes if (p.get("profile") or "baseline") == profile] + if not taken: + return None + + def mean(values): + values = [v for v in values if v is not None] + return sum(values) / len(values) if values else None + + return { + "probes": len(taken), + "rtt_p50_ms": mean([(p.get("rtt_ms") or {}).get("p50") for p in taken]), + "handshake_ms": mean([p.get("handshake_ms") for p in taken]), + "single_mib_per_s": mean( + [(p.get("control") or {}).get("single_stream_mib_per_s") for p in taken] + ), + "n_mib_per_s": mean( + [(p.get("control") or {}).get("n_stream_mib_per_s") for p in taken] + ), + "streams": next( + ( + (p.get("control") or {}).get("streams") + for p in taken + if (p.get("control") or {}).get("streams") + ), + None, + ), + } + + def best_cell(self, scenario, label, profile): + """The fastest measured cell of one scenario, build and profile. + + Read out of `scaling[].best` where the harness recorded it, so a plot + and `matrix.md` never disagree about which cell won. + """ + for entry in self.scaling: + if ( + entry.get("scenario") == scenario + and entry.get("label") == label + and (entry.get("link_profile") or "baseline") == profile + ): + best = entry.get("best") or {} + coordinates = {key: best.get(key) for key in AXIS_KEYS} + for cell in self.results: + if ( + cell.get("scenario") == scenario + and cell.get("label") == label + and self.profile(cell) == profile + and all(cell.get(k) == v for k, v in coordinates.items()) + ): + return cell, entry + return None, entry + return None, None + + def caveats(self): + """What a reader has to know before comparing anything in this file. + + Every figure carries these under it. They are not decoration: a sweep + measured with shaping unavailable has profile names that say what was + asked for, not what happened, and a chart cannot tell that by itself. + """ + notes = [] + + shaping = self.shaping + # "baseline" is the real line and is never shaped, so a run that asked + # for nothing else is not missing anything and gets no caveat. + requested = [ + profile for profile in (shaping.get("requested") or []) if profile != "baseline" + ] + applied = [ + profile for profile in (shaping.get("applied") or []) if profile != "baseline" + ] + if requested and not shaping.get("available", True): + reason = shaping.get("reason") or "reason not recorded" + notes.append( + "link shaping was not available (" + + reason + + "): every profile was measured on the real line, so the " + "profile names say what was asked for, not what happened" + ) + elif requested and applied and set(requested) != set(applied): + notes.append( + "shaping applied for " + + " ".join(applied) + + " only, out of the requested " + + " ".join(requested) + ) + + failed = sum( + int(row.get("failed_runs") or 0) for row in self.results + ) + sum(int(row.get("failed_sweeps") or 0) for row in self.deletes) + if failed: + notes.append(f"{failed} repeat(s) failed and are not in these aggregates") + + refused = sum( + int(row.get("connections_refused") or row.get("refused_connections") or 0) + for row in self.results + ) + if refused: + notes.append( + f"{refused} connection(s) were refused and fell back to the first one" + ) + + errors = sum(int(row.get("errors") or 0) for row in self.results) + if errors: + notes.append(f"{errors} operation error(s) were recorded") + + return notes + + def provenance(self): + parts = [self.kind, self.ref, self.runner] + if self.recorded_at: + parts.append(self.recorded_at) + return ", ".join(str(p) for p in parts if p and p != "?") + + +def load(path): + with open(path, encoding="utf-8") as handle: + return Document(path, json.load(handle)) + + +# -------------------------------------------------------------------------- +# Finding the stored files +# -------------------------------------------------------------------------- + + +def newest_matrix_json(): + found = sorted((BENCHMARKS / "matrix").glob("matrix-*.json")) + if not found: + raise SystemExit( + "no sweep found under benchmarks/matrix; pass a matrix.json explicitly" + ) + return found[-1] + + +def newest_matrix_csv(): + found = sorted((BENCHMARKS / "matrix").glob("matrix-*.csv")) + if not found: + raise SystemExit( + "no sweep found under benchmarks/matrix; pass a matrix.csv explicitly" + ) + return found[-1] + + +def newest_release_json(): + """`latest.json` when it is there: it is the copy of the newest official + release result and the one link that must never move.""" + latest = BENCHMARKS / "latest.json" + if latest.exists(): + return latest + found = sorted((BENCHMARKS / "releases").glob("release-*.json")) + if not found: + raise SystemExit( + "no release result found under benchmarks/releases; pass one explicitly" + ) + return found[-1] + + +def documents(paths, default): + """Load the given paths, or the newest stored file of the right kind.""" + chosen = [Path(p) for p in paths] or [default()] + return [load(path) for path in chosen] diff --git a/benchmarks/analysis/out/auto-matrix-20260816T125322Z-main.png b/benchmarks/analysis/out/auto-matrix-20260816T125322Z-main.png new file mode 100644 index 0000000..684bc7a Binary files /dev/null and b/benchmarks/analysis/out/auto-matrix-20260816T125322Z-main.png differ diff --git a/benchmarks/analysis/out/canary-matrix-20260816T125322Z-main.png b/benchmarks/analysis/out/canary-matrix-20260816T125322Z-main.png new file mode 100644 index 0000000..4faafbb Binary files /dev/null and b/benchmarks/analysis/out/canary-matrix-20260816T125322Z-main.png differ diff --git a/benchmarks/analysis/out/deletes-matrix-20260816T125322Z-main-small.png b/benchmarks/analysis/out/deletes-matrix-20260816T125322Z-main-small.png new file mode 100644 index 0000000..cf73559 Binary files /dev/null and b/benchmarks/analysis/out/deletes-matrix-20260816T125322Z-main-small.png differ diff --git a/benchmarks/analysis/out/heatmap-matrix-20260730T224549Z-main-calib-100x64k-candidate-baseline-default.png b/benchmarks/analysis/out/heatmap-matrix-20260730T224549Z-main-calib-100x64k-candidate-baseline-default.png deleted file mode 100644 index b2fbfab..0000000 Binary files a/benchmarks/analysis/out/heatmap-matrix-20260730T224549Z-main-calib-100x64k-candidate-baseline-default.png and /dev/null differ diff --git a/benchmarks/analysis/out/heatmap-matrix-20260730T224549Z-main-large-candidate-baseline-default.png b/benchmarks/analysis/out/heatmap-matrix-20260730T224549Z-main-large-candidate-baseline-default.png deleted file mode 100644 index d735d2f..0000000 Binary files a/benchmarks/analysis/out/heatmap-matrix-20260730T224549Z-main-large-candidate-baseline-default.png and /dev/null differ diff --git a/benchmarks/analysis/out/heatmap-matrix-20260730T224549Z-main-single-candidate-baseline-default.png b/benchmarks/analysis/out/heatmap-matrix-20260730T224549Z-main-single-candidate-baseline-default.png deleted file mode 100644 index a01be77..0000000 Binary files a/benchmarks/analysis/out/heatmap-matrix-20260730T224549Z-main-single-candidate-baseline-default.png and /dev/null differ diff --git a/benchmarks/analysis/out/heatmap-matrix-20260730T224549Z-main-small-candidate-baseline-default.png b/benchmarks/analysis/out/heatmap-matrix-20260730T224549Z-main-small-candidate-baseline-default.png deleted file mode 100644 index 134d0aa..0000000 Binary files a/benchmarks/analysis/out/heatmap-matrix-20260730T224549Z-main-small-candidate-baseline-default.png and /dev/null differ diff --git a/benchmarks/analysis/out/heatmap-matrix-20260730T224549Z-main-sync-candidate-baseline-default.png b/benchmarks/analysis/out/heatmap-matrix-20260730T224549Z-main-sync-candidate-baseline-default.png deleted file mode 100644 index ecdf983..0000000 Binary files a/benchmarks/analysis/out/heatmap-matrix-20260730T224549Z-main-sync-candidate-baseline-default.png and /dev/null differ diff --git a/benchmarks/analysis/out/heatmap-matrix-20260816T125322Z-main-small-candidate-baseline-default.png b/benchmarks/analysis/out/heatmap-matrix-20260816T125322Z-main-small-candidate-baseline-default.png new file mode 100644 index 0000000..0b864ff Binary files /dev/null and b/benchmarks/analysis/out/heatmap-matrix-20260816T125322Z-main-small-candidate-baseline-default.png differ diff --git a/benchmarks/analysis/out/link-matrix-20260816T125322Z-main.png b/benchmarks/analysis/out/link-matrix-20260816T125322Z-main.png new file mode 100644 index 0000000..5b09ad7 Binary files /dev/null and b/benchmarks/analysis/out/link-matrix-20260816T125322Z-main.png differ diff --git a/benchmarks/analysis/out/operations-release-v3.5.0-small-baseline.png b/benchmarks/analysis/out/operations-release-v3.5.0-small-baseline.png new file mode 100644 index 0000000..adb4c9d Binary files /dev/null and b/benchmarks/analysis/out/operations-release-v3.5.0-small-baseline.png differ diff --git a/benchmarks/analysis/out/phases-release-v3.5.0.png b/benchmarks/analysis/out/phases-release-v3.5.0.png new file mode 100644 index 0000000..f67076a Binary files /dev/null and b/benchmarks/analysis/out/phases-release-v3.5.0.png differ diff --git a/benchmarks/analysis/out/scaling-matrix-20260816T125322Z-main-small-candidate-baseline-all.png b/benchmarks/analysis/out/scaling-matrix-20260816T125322Z-main-small-candidate-baseline-all.png new file mode 100644 index 0000000..54976d8 Binary files /dev/null and b/benchmarks/analysis/out/scaling-matrix-20260816T125322Z-main-small-candidate-baseline-all.png differ diff --git a/benchmarks/analysis/out/trend-releases.png b/benchmarks/analysis/out/trend-releases.png index 961fb3d..e7b3e94 100644 Binary files a/benchmarks/analysis/out/trend-releases.png and b/benchmarks/analysis/out/trend-releases.png differ diff --git a/benchmarks/analysis/plot.py b/benchmarks/analysis/plot.py index b2f9e34..db835d4 100644 --- a/benchmarks/analysis/plot.py +++ b/benchmarks/analysis/plot.py @@ -1,23 +1,37 @@ #!/usr/bin/env python3 """Plots of stored benchmark results. -Reads the canonical CSVs the Go harness writes and nothing else: a sweep's -matrix.csv and the top-level trend.csv. It never runs a benchmark, never writes -into benchmarks/ outside the output directory, and is not needed by anything -that does (issue #190). - -Columns are read by name, because the stored files span several schema -versions: the sweeps from before issue #184 have no link_profile and no -request_concurrency, and asking for a column that is not there is how a plot -silently starts showing the wrong axis. - - python benchmarks/analysis/plot.py heatmap [matrix.csv ...] [--out DIR] - python benchmarks/analysis/plot.py trend [trend.csv] [--out DIR] +Reads the canonical files the Go harness writes and nothing else: a sweep's +`matrix.json` / `matrix.csv`, a run's `results.json` (`latest.json` is the +newest official one) and the top-level `trend.csv`. It never runs a benchmark, +never writes into `benchmarks/` outside the output directory, and is not needed +by anything that does (issue #190). + +Fields are read by name and every one of them is optional, because the stored +files span several schema versions: the sweeps from before issue #184 have no +link profile, no `request_concurrency` and no `auto` rows, and version 1 +results have no phases and no operations at all. Asking for something that is +not there is how a plot silently starts showing the wrong axis. + + python benchmarks/analysis/plot.py [file ...] [options] + + heatmap throughput over the connections x concurrency grid matrix.csv + scaling the same grid as curves, with the linear reference matrix.json + auto what easySFTP picks for itself against the best cell matrix.json + canary whether the line held still for the whole sweep matrix.json + phases where the wall clock of a deployment goes either + operations per round-trip cost and share of the work either + deletes the delete sweeps, the only measurement of deletion either + link measured throughput against the link's own control either + trend throughput per scenario across releases trend.csv + report every plot the given files support, plus report.md + +Run `plot.py --help` for the options of one command. """ import argparse -import csv import sys +import textwrap from collections import defaultdict from datetime import datetime from pathlib import Path @@ -27,101 +41,202 @@ matplotlib.use("Agg") # no display on a runner, and none needed import matplotlib.pyplot as plt # noqa: E402 -REPO = Path(__file__).resolve().parents[2] -BENCHMARKS = REPO / "benchmarks" -DEFAULT_OUT = BENCHMARKS / "analysis" / "out" +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import benchdata as bench # noqa: E402 +from benchdata import ( # noqa: E402 + AXIS_KEYS, + BENCHMARKS, + DEFAULT_OUT, + OPERATION_ORDER, + PHASE_ORDER, + UMBRELLA_OPERATIONS, + axis_label, + axis_layout, + human_ms, + number, + slug, +) + +COLORS = matplotlib.colormaps["tab20"] +UNKNOWN = "#9e9e9e" + +# Higher is better for every metric but the duration. +METRICS = { + "mib_per_s": ("MiB/s", True), + "files_per_s": ("files/s", True), + "median_ms": ("median ms", False), +} + + +# -------------------------------------------------------------------------- +# Drawing helpers shared by every command +# -------------------------------------------------------------------------- + +def color_for(name, order): + if name in order: + return COLORS(order.index(name) % 20) + return UNKNOWN -def rows(path): - """The CSV as dicts, with empty strings left alone: '' is 'not recorded'.""" - with open(path, newline="", encoding="utf-8") as handle: - return list(csv.DictReader(handle)) +def sort_key(value): + """Sorts axis values with a null (an unset request_concurrency) first: it + is not a number and it is not zero, it is 'whatever easySFTP picked'.""" + if value is None or value == "": + return (0, 0.0, "") + if isinstance(value, (int, float)): + return (1, float(value), "") + return (2, 0.0, str(value)) -def number(row, column): - """A numeric column, or None when it is absent or empty.""" - text = (row.get(column) or "").strip() - if not text: - return None - try: - return float(text) - except ValueError: - return None +def footer(figure, doc, extra=(), warnings=()): + """Provenance and caveats under every figure. -def slug(text): - keep = [c if c.isalnum() or c in "-_." else "-" for c in str(text)] - return "".join(keep).strip("-") or "none" + Not decoration. Two results measured on different runners or over + different links are not comparable and a chart cannot tell them apart, so + it has to say what it is a chart of. + """ + lines = [(str(text), "#404040") for text in extra if text] + lines += [(str(text), "crimson") for text in warnings if text] + if doc is not None: + lines += [("caveat: " + note, "crimson") for note in doc.caveats()] + lines.append((doc.provenance(), "#707070")) + + width = int(max(60, figure.get_figwidth() * 15)) + wrapped = [] + for text, color in lines: + for piece in textwrap.wrap(text, width) or [""]: + wrapped.append((piece, color)) + if not wrapped: + figure.tight_layout() + return + + # In figure fractions, so it has to come from the figure's height: a fixed + # fraction writes the lines on top of each other on a short figure. + step = 10.0 / (72.0 * figure.get_figheight()) + figure.tight_layout(rect=(0, min(step * len(wrapped) + step, 0.45), 1, 1)) + for index, (piece, color) in enumerate(reversed(wrapped)): + figure.text( + 0.01, step * (0.4 + index), piece, fontsize=7, color=color, va="bottom" + ) + + +def save(figure, args, name): + args.out.mkdir(parents=True, exist_ok=True) + target = args.out / f"{name}.{args.format}" + figure.savefig(target, dpi=args.dpi) + plt.close(figure) + return target -def newest_matrix(): - sweeps = sorted((BENCHMARKS / "matrix").glob("matrix-*.csv")) - if not sweeps: - sys.exit("no sweep found under benchmarks/matrix; pass one explicitly") - return [sweeps[-1]] +def selected(args, scenario=None, profile=None): + """The scenario and profile filters, both optional and both exact.""" + if args.scenario and scenario is not None and scenario not in args.scenario: + return False + if args.profile and profile is not None and profile not in args.profile: + return False + return True -def heatmap(paths, out): +def document_stem(doc): + """What a file made from this document is called. + + The stored name, except for `latest.json`: it is a copy of a release + result, and a plot called "latest" stops being true the day after. + """ + if doc.path.stem == "latest" and doc.envelope.get("version"): + return f"release-{doc.envelope['version']}" + return doc.path.stem + + +def sibling_document(path): + """The JSON next to a CSV, for the provenance and the caveats a flat + export cannot carry. Absent is normal, not an error.""" + candidate = Path(path).with_suffix(".json") + if candidate.exists(): + try: + return bench.load(candidate) + except (ValueError, OSError): + return None + return None + + +# -------------------------------------------------------------------------- +# heatmap: the grid as a grid +# -------------------------------------------------------------------------- + + +def heatmap(args): """One heatmap per scenario, build, link profile and request_concurrency. Those four are what a cell is not: mixing any of them into one grid would put two different configurations in the same square. """ written = [] - for path in paths: + for path in args.paths or [bench.newest_matrix_csv()]: + doc = sibling_document(path) grids = defaultdict(dict) - for row in rows(path): + for row in bench.rows(path): connections = number(row, "connections") concurrency = number(row, "concurrency") median = number(row, "median_ms") + scenario = row.get("scenario", "?") + profile = row.get("link_profile") or "baseline" if connections is None or concurrency is None or median is None: continue + if not selected(args, scenario, profile): + continue key = ( - row.get("scenario", "?"), + scenario, row.get("build", "?"), - row.get("link_profile") or "baseline", + profile, row.get("request_concurrency") or "default", ) - grids[key][(int(connections), int(concurrency))] = ( - median, - number(row, "mib_per_s"), - ) + grids[key][(int(connections), int(concurrency))] = { + "median_ms": median, + "mib_per_s": number(row, "mib_per_s"), + "files_per_s": number(row, "files_per_s"), + } for key, cells in sorted(grids.items()): - scenario, build, profile, requests = key if len(cells) < 2: # A single cell is a number, not a grid. matrix.json has it. continue - written.append(_draw_heatmap(path, key, cells, out)) - - if not written: - sys.exit("nothing to plot: no sweep in those files had a grid") - for item in written: - print(item) + written.append(_draw_heatmap(path, doc, key, cells, args)) + return written -def _draw_heatmap(source, key, cells, out): +def _draw_heatmap(source, doc, key, cells, args): scenario, build, profile, requests = key + metric, (unit, higher_is_better) = args.metric, METRICS[args.metric] connections = sorted({c for c, _ in cells}) concurrency = sorted({c for _, c in cells}) - speeds = [ - [ - (cells.get((row, column)) or (None, None))[1] - for column in concurrency - ] + values = [ + [(cells.get((row, column)) or {}).get(metric) for column in concurrency] for row in connections ] + measured = [ + (coordinate, cell[metric]) + for coordinate, cell in cells.items() + if cell.get(metric) is not None + ] + best = ( + (max if higher_is_better else min)(measured, key=lambda item: item[1])[0] + if measured + else None + ) figure, axes = plt.subplots( - figsize=(1.5 + 1.2 * len(concurrency), 1.5 + 0.9 * len(connections)) + figsize=(2.0 + 1.2 * len(concurrency), 2.2 + 0.9 * len(connections)) ) image = axes.imshow( - [[value if value is not None else float("nan") for value in line] for line in speeds], - cmap="viridis", + [[v if v is not None else float("nan") for v in line] for line in values], + cmap="viridis" if higher_is_better else "viridis_r", aspect="auto", ) - figure.colorbar(image, ax=axes, label="MiB/s") + figure.colorbar(image, ax=axes, label=unit) axes.set_xticks(range(len(concurrency)), [str(c) for c in concurrency]) axes.set_yticks(range(len(connections)), [str(c) for c in connections]) @@ -140,91 +255,1250 @@ def _draw_heatmap(source, key, cells, out): if cell is None: continue axes.text( - x, y, f"{cell[0]:.0f} ms", - ha="center", va="center", fontsize=8, color="white", + x, + y, + f"{cell['median_ms']:.0f} ms", + ha="center", + va="center", + fontsize=8, + color="white", ) - figure.tight_layout() - out.mkdir(parents=True, exist_ok=True) - name = f"heatmap-{slug(Path(source).stem)}-{slug(scenario)}-{slug(build)}-{slug(profile)}-{slug(requests)}.png" - target = out / name - figure.savefig(target, dpi=140) - plt.close(figure) - return target + notes = [] + if best is not None: + y, x = connections.index(best[0]), concurrency.index(best[1]) + axes.add_patch( + plt.Rectangle( + (x - 0.5, y - 0.5), 1, 1, fill=False, edgecolor="crimson", linewidth=2.5 + ) + ) + edges = [ + name + for name, value, axis in ( + ("connections", best[0], connections), + ("concurrency", best[1], concurrency), + ) + if value == max(axis) and len(axis) > 1 + ] + note = ( + f"best cell (red): connections {best[0]}, concurrency {best[1]}, " + f"{cells[best][metric]:.2f} {unit}" + ) + if edges: + # The reason the grid exists: a best cell on the largest swept + # value is a cut-off, not an optimum. matrix.json says the same in + # scaling[].best_at_axis_max, and that field is the authority. + note += "; it sits on the largest swept " + " and ".join(edges) + note += ", so the optimum was bounded from below, not measured" + notes.append(note) + + footer(figure, doc, notes) + return save( + figure, + args, + "heatmap-" + + "-".join( + slug(part) + for part in (Path(source).stem, scenario, build, profile, requests) + ), + ) + + +# -------------------------------------------------------------------------- +# scaling: the same grid, read as curves +# -------------------------------------------------------------------------- + + +def scaling(args): + """Throughput against the axis that was actually swept, per scenario. + + The heatmap answers "which cell won". This answers "where did it stop + paying", which is the question `advanced.connections` and + `advanced.concurrency` are set from. The dashed line is what perfect + scaling from the leftmost point would have looked like; the gap to it is + the answer. + """ + written = [] + for doc in bench.documents(args.paths, bench.newest_matrix_json): + if not doc.scaling: + continue + for entry in doc.scaling: + scenario = entry.get("scenario", "?") + profile = entry.get("link_profile") or "baseline" + if not selected(args, scenario, profile): + continue + points = entry.get("points") or [] + if len(points) < 2: + continue + x_key, series_key, facet_keys = axis_layout(points) + facet_key = facet_keys[0] if facet_keys else None + facets = ( + sorted({p.get(facet_key) for p in points}, key=sort_key) + if facet_key + else [None] + ) + for facet in facets: + chosen = [ + p for p in points if facet_key is None or p.get(facet_key) == facet + ] + if len(chosen) < 2: + continue + written.append( + _draw_scaling( + doc, entry, chosen, x_key, series_key, facet_key, facet, args + ) + ) + return written + + +def _draw_scaling(doc, entry, points, x_key, series_key, facet_key, facet, args): + scenario = entry.get("scenario", "?") + label = entry.get("label", "?") + profile = entry.get("link_profile") or "baseline" + best = entry.get("best") or {} + + x_values = sorted({p.get(x_key) for p in points}, key=sort_key) + positions = {value: index for index, value in enumerate(x_values)} + series = sorted({p.get(series_key) for p in points}, key=sort_key) + + figure, (top, bottom) = plt.subplots( + 2, 1, figsize=(2.5 + 1.0 * len(x_values), 7.2), sharex=True + ) + ceilings = {top: 0.0, bottom: 0.0} + for index, value in enumerate(series): + line = sorted( + (p for p in points if p.get(series_key) == value), + key=lambda p: sort_key(p.get(x_key)), + ) + color = COLORS(index * 2 % 20) + name = f"{series_key} {axis_label(series_key, value)}" + for axes, field in ((top, "mib_per_s"), (bottom, "files_per_s")): + xs = [positions[p.get(x_key)] for p in line if p.get(field) is not None] + ys = [p[field] for p in line if p.get(field) is not None] + if not ys: + continue + axes.plot(xs, ys, marker="o", color=color, label=name) + ceilings[axes] = max(ceilings[axes], max(ys)) + + # Perfect scaling from the leftmost measured point of this series. + # Only drawn where the axis is numeric: "default" has no multiple. + numeric = [p for p in line if isinstance(p.get(x_key), (int, float))] + if len(numeric) >= 3: + first = numeric[0] + for axes, field in ((top, "mib_per_s"), (bottom, "files_per_s")): + if first.get(field) in (None, 0): + continue + axes.plot( + [positions[p.get(x_key)] for p in numeric], + [ + first[field] * (p[x_key] / first[x_key]) + for p in numeric + ], + linestyle=":", + linewidth=1, + color=color, + alpha=0.45, + ) + + if best.get(x_key) in positions: + for axes, field in ((top, "mib_per_s"), (bottom, "files_per_s")): + if best.get(field) is not None: + axes.scatter( + [positions[best[x_key]]], + [best[field]], + marker="*", + s=190, + color="crimson", + zorder=5, + label="best cell" if axes is top else None, + ) + + # Scaled to what was measured, so the reference may leave the frame: an + # axis stretched to fit perfect scaling flattens the curves the plot is for. + for axes, ceiling in ceilings.items(): + if ceiling > 0: + axes.set_ylim(0, ceiling * 1.15) + + top.set_ylabel("MiB/s") + bottom.set_ylabel("files/s") + bottom.set_xlabel(x_key) + bottom.set_xticks( + range(len(x_values)), [axis_label(x_key, value) for value in x_values] + ) + for axes in (top, bottom): + axes.grid(alpha=0.3) + top.legend(fontsize=8) + + title = f"{scenario} / {label} / {profile}" + if facet_key is not None: + title += f" / {facet_key} {axis_label(facet_key, facet)}" + top.set_title( + f"{title}\n{doc.scenarios.get(scenario, '')}".rstrip(), fontsize=10 + ) + + notes = [ + "dotted: perfect scaling from the leftmost point of the same series; the " + "y axis is scaled to what was measured, so it normally leaves the frame" + ] + # `scaling[].best_at_axis_max` is the authority for this, but the sweeps + # from before issue #184 do not carry it, so it is derived there instead of + # the plot going quiet about the one thing the grid exists to show. + edges = entry.get("best_at_axis_max") + derived = edges is None + if derived: + edges = [ + key + for key in AXIS_KEYS + if isinstance(best.get(key), (int, float)) + and len({p.get(key) for p in entry.get("points") or []}) > 1 + and best[key] == max( + p.get(key) + for p in entry.get("points") or [] + if isinstance(p.get(key), (int, float)) + ) + ] + if edges: + notes.append( + "the best cell sits on the largest swept " + + " and ".join(edges) + + ": the optimum was bounded from below, not measured, and nothing " + "should be fitted to these numbers until the axis is extended" + + (" (derived here: this sweep predates the stored field)" if derived else "") + ) + footer(figure, doc, notes) + return save( + figure, + args, + "scaling-" + + "-".join( + slug(part) + for part in ( + document_stem(doc), + scenario, + label, + profile, + f"{facet_key}{axis_label(facet_key, facet)}" if facet_key else "all", + ) + ), + ) + + +# -------------------------------------------------------------------------- +# phases: where the wall clock goes +# -------------------------------------------------------------------------- + + +def _upload_rows(doc, args): + """One (title, name, row) per (scenario, profile): the result of a standard + run, the best cell of a sweep. A sweep has hundreds of cells and stacking + all of them says nothing; the cell that won is the one worth taking apart. + """ + picked = [] + if doc.is_matrix: + for entry in doc.scaling: + scenario = entry.get("scenario", "?") + profile = entry.get("link_profile") or "baseline" + if not selected(args, scenario, profile): + continue + cell, _ = doc.best_cell(scenario, entry.get("label"), profile) + if cell is None: + continue + short = { + "connections": "conn", + "concurrency": "conc", + "request_concurrency": "req", + } + coordinates = [ + f"{short[key]} {axis_label(key, cell.get(key))}" for key in AXIS_KEYS + ] + picked.append( + ( + f"{scenario} / {profile}\nbest cell: " + ", ".join(coordinates), + slug(f"{scenario}-{profile}-" + "-".join(coordinates)), + cell, + ) + ) + else: + for row in doc.results: + scenario = row.get("scenario", "?") + profile = doc.profile(row) + if not selected(args, scenario, profile): + continue + picked.append((f"{scenario} / {profile}", slug(f"{scenario}-{profile}"), row)) + return picked + + +def phases(args): + written = [] + for doc in bench.documents(args.paths, bench.newest_release_json): + rows = [ + (title, row) + for title, _, row in _upload_rows(doc, args) + if row.get("phases") + ] + if args.include_deletes: + rows += [ + (f"{row.get('scenario', '?')} / {doc.profile(row)}\ndelete sweep", row) + for row in doc.deletes + if row.get("phases") + and selected(args, row.get("scenario"), doc.profile(row)) + ] + if not rows: + continue + written.append(_draw_phases(doc, rows, args)) + return written + + +def _draw_phases(doc, rows, args): + names = [ + name + for name in PHASE_ORDER + if any( + any(phase.get("name") == name for phase in row.get("phases", [])) + for _, row in rows + ) + ] + extra = sorted( + { + phase.get("name") + for _, row in rows + for phase in row.get("phases", []) + if phase.get("name") not in names + } + ) + names += extra + + figure, axes = plt.subplots(figsize=(13, 2.4 + 0.85 * len(rows))) + offsets = [0.0] * len(rows) + for name in names: + widths = [ + next( + ( + number(phase, "median_ms") or 0.0 + for phase in row.get("phases", []) + if phase.get("name") == name + ), + 0.0, + ) + for _, row in rows + ] + if not any(widths): + continue + axes.barh( + range(len(rows)), + widths, + left=offsets, + color=color_for(name, PHASE_ORDER), + label=name, + height=0.62, + ) + offsets = [a + b for a, b in zip(offsets, widths)] + + for index, total in enumerate(offsets): + axes.text(total, index, f" {human_ms(total)}", va="center", fontsize=8) + + axes.set_yticks(range(len(rows)), [label for label, _ in rows], fontsize=8) + axes.invert_yaxis() + axes.set_xlabel("wall clock, median of the repeats (ms)") + axes.set_xlim(0, max(offsets) * 1.12 if offsets else 1) + axes.set_title("Where the wall clock of a deployment goes", fontsize=11) + axes.legend(fontsize=8, loc="upper left", bbox_to_anchor=(1.01, 1)) + axes.grid(axis="x", alpha=0.3) + + footer( + figure, + doc, + [ + "phases are wall clock and add up to roughly the run's duration; " + "the per round-trip costs inside them are in the operations plot " + "and are cumulative across the parallel workers, so they are not " + "the same kind of number" + ], + ) + return save(figure, args, f"phases-{slug(document_stem(doc))}") + + +# -------------------------------------------------------------------------- +# operations: what the round-trips cost +# -------------------------------------------------------------------------- + + +def operations(args): + written = [] + for doc in bench.documents(args.paths, bench.newest_release_json): + for title, key, row in _upload_rows(doc, args): + if row.get("operations"): + written.append(_draw_operations(doc, title, key, row, args)) + return written + + +def _draw_operations(doc, title, key, row, args): + ordered = sorted( + row.get("operations", []), + key=lambda op: number(op, "median_total_ms") or number(op, "total_ms") or 0.0, + ) + names = [op.get("name", "?") for op in ordered] + totals = [ + number(op, "median_total_ms") or number(op, "total_ms") or 0.0 for op in ordered + ] + counts = [int(op.get("count") or 0) for op in ordered] + ticks = [ + f"{name} (n={count}){' *' if name in UMBRELLA_OPERATIONS else ''}" + for name, count in zip(names, counts) + ] + + figure, (left, right) = plt.subplots( + 1, 2, figsize=(13, 1.8 + 0.5 * len(ordered)), sharey=True + ) + + colors = [color_for(name, OPERATION_ORDER) for name in names] + left.barh(range(len(ordered)), totals, color=colors, height=0.65) + for index, (total, name) in enumerate(zip(totals, names)): + share = "" + billed = sum( + value + for value, other in zip(totals, names) + if other not in UMBRELLA_OPERATIONS + ) + if billed and name not in UMBRELLA_OPERATIONS: + share = f" ({total / billed * 100:.0f}%)" + left.text(total, index, f" {human_ms(total)}{share}", va="center", fontsize=8) + left.set_xlabel("cumulative across the parallel workers (ms)") + left.set_xlim(0, max(totals) * 1.25 if totals else 1) + left.set_yticks(range(len(ordered)), ticks, fontsize=8) + left.grid(axis="x", alpha=0.3) + left.set_title("share of the work", fontsize=10) + + for index, op in enumerate(ordered): + percentiles = [ + (number(op, "p50_ms"), "o", "p50"), + (number(op, "p90_ms"), "s", "p90"), + (number(op, "p99_ms"), "^", "p99"), + (number(op, "max_ms"), "|", "max"), + ] + spread = [value for value, _, _ in percentiles if value is not None] + if spread: + right.plot( + [min(spread), max(spread)], + [index, index], + color=colors[index], + alpha=0.5, + linewidth=1.5, + ) + for value, marker, name in percentiles: + if value is None: + continue + right.scatter( + [value], + [index], + marker=marker, + color=colors[index], + s=46, + label=name if index == 0 else None, + ) + right.set_xscale("log") + right.set_xlabel("per call (ms, log scale)") + right.grid(axis="x", alpha=0.3) + right.set_title("per call cost", fontsize=10) + right.legend(fontsize=8, loc="lower right") + + figure.suptitle(f"Operations: {title.replace(chr(10), ' | ')}", fontsize=11) + footer( + figure, + doc, + [ + "* file_upload is the umbrella around the sftp_* calls of one file: " + "its total contains theirs, so the two are never added together and " + "the share is taken over the others only", + "operation totals are cumulative across the parallel upload workers " + "and are normally larger than the phase they happened in; read them " + "for the share of the work and the per call cost, never as elapsed time", + ], + ) + return save(figure, args, f"operations-{slug(document_stem(doc))}-{key}") + + +# -------------------------------------------------------------------------- +# deletes: the only measurement of deletion there is +# -------------------------------------------------------------------------- + + +def deletes(args): + """The clean deployment that runs before every measured run. + + It is a pure delete sweep and it is instrumented on purpose (issue #184, + phase 4): nothing else in the harness measures deletion, and its numbers + are deliberately kept out of the upload aggregates. + """ + written = [] + for doc in bench.documents(args.paths, bench.newest_release_json): + rows = [ + row + for row in doc.deletes + if selected(args, row.get("scenario"), doc.profile(row)) + ] + if not rows: + continue + varies = len({tuple(row.get(key) for key in AXIS_KEYS) for row in rows}) > 1 + if varies: + for scenario in sorted({row.get("scenario", "?") for row in rows}): + chosen = [row for row in rows if row.get("scenario") == scenario] + if len(chosen) > 1: + written.append(_draw_delete_curve(doc, scenario, chosen, args)) + else: + written.append(_draw_delete_bars(doc, rows, args)) + return written + + +def _delete_rate(doc, row): + rate = number(row, "deletes_per_s") + if rate is not None: + return rate + _, files_per_s = doc.throughput( + {"files": row.get("files_deleted"), "median_ms": row.get("median_ms")} + ) + return files_per_s + + +def _draw_delete_bars(doc, rows, args): + labels = [f"{row.get('scenario', '?')}\n{doc.profile(row)}" for row in rows] + rates = [_delete_rate(doc, row) or 0.0 for row in rows] + figure, axes = plt.subplots(figsize=(2.5 + 1.1 * len(rows), 5.4)) + bars = axes.bar(range(len(rows)), rates, color=COLORS(4)) + for bar, row, rate in zip(bars, rows, rates): + axes.text( + bar.get_x() + bar.get_width() / 2, + rate, + f"{rate:.1f}/s\n{human_ms(number(row, 'median_ms'))}\n" + f"{int(row.get('files_deleted') or 0)} files", + ha="center", + va="bottom", + fontsize=8, + ) + axes.set_xticks(range(len(rows)), labels, fontsize=8) + axes.set_ylabel("deletes/s") + axes.set_ylim(0, max(rates) * 1.35 if rates else 1) + axes.grid(axis="y", alpha=0.3) + axes.set_title("Delete sweeps (the pre-run clean deployment)", fontsize=11) + + footer(figure, doc, [_DELETE_NOTE]) + return save(figure, args, f"deletes-{slug(document_stem(doc))}") + + +def _draw_delete_curve(doc, scenario, rows, args): + """The delete rate against the widest swept axis. + + One bold line per profile through the median of every cell measured at + that coordinate, with the cells themselves behind it. A line per + (profile, connections) pair would be a dozen crossing lines saying the + same thing, and the thing they say is whether the rate moves at all. + """ + x_key, _, _ = axis_layout(rows) + x_values = sorted({row.get(x_key) for row in rows}, key=sort_key) + positions = {value: index for index, value in enumerate(x_values)} + + figure, axes = plt.subplots(figsize=(3.5 + 0.95 * len(x_values), 5.8)) + for index, profile in enumerate(sorted({doc.profile(row) for row in rows})): + color = COLORS(index * 2 % 20) + measured = [ + (positions[row.get(x_key)], _delete_rate(doc, row)) + for row in rows + if doc.profile(row) == profile and _delete_rate(doc, row) is not None + ] + if not measured: + continue + axes.scatter( + [x for x, _ in measured], + [y for _, y in measured], + color=color, + s=14, + alpha=0.35, + ) + middles = [] + for position in range(len(x_values)): + at = sorted(y for x, y in measured if x == position) + if at: + middles.append((position, at[(len(at) - 1) // 2])) + axes.plot( + [x for x, _ in middles], + [y for _, y in middles], + marker="o", + color=color, + linewidth=2, + label=profile, + ) + + axes.set_xticks( + range(len(x_values)), [axis_label(x_key, value) for value in x_values] + ) + axes.set_xlabel(x_key) + axes.set_ylabel("deletes/s") + axes.grid(alpha=0.3) + axes.legend(fontsize=8, title="link profile") + axes.set_title( + f"Delete sweeps: {scenario}\n" + "line: median of the cells at that coordinate, dots: the cells", + fontsize=10, + ) + + footer( + figure, + doc, + [ + _DELETE_NOTE, + "everything outside the per-file upload path runs over one " + "connection (`session.do`), so a rate that does not move with these " + "axes is the expected shape, not a measurement error", + ], + ) + return save(figure, args, f"deletes-{slug(document_stem(doc))}-{slug(scenario)}") + + +_DELETE_NOTE = ( + "the clean deployment measured before every run: a pure delete sweep, and " + "the only measurement of deletion in the harness; its numbers are kept out " + "of the upload aggregates on purpose" +) + + +# -------------------------------------------------------------------------- +# link: the measurement against the line it was measured over +# -------------------------------------------------------------------------- + + +def link(args): + """How much of the line a run actually used. + + `cmd/linkprobe` measures the path with x/crypto/ssh and pkg/sftp, importing + nothing from `internal/uploader`: a control taken through the code under + test would not be a control. Read against it, a slower easySFTP and a + slower line stop looking the same. + """ + written = [] + for doc in bench.documents(args.paths, bench.newest_release_json): + profiles = [ + profile + for profile in sorted({doc.profile(row) for row in doc.results}) + if selected(args, profile=profile) and doc.controls(profile) + ] + if not profiles: + continue + written.append(_draw_link(doc, profiles, args)) + return written + + +def _draw_link(doc, profiles, args): + scenarios = sorted( + { + row.get("scenario", "?") + for row in doc.results + if selected(args, row.get("scenario")) + } + ) + measured = {} + for profile in profiles: + for scenario in scenarios: + if doc.is_matrix: + entry = next( + ( + item + for item in doc.scaling + if item.get("scenario") == scenario + and (item.get("link_profile") or "baseline") == profile + ), + None, + ) + value = (entry.get("best") or {}).get("mib_per_s") if entry else None + else: + row = next( + ( + item + for item in doc.results + if item.get("scenario") == scenario + and doc.profile(item) == profile + ), + None, + ) + value = doc.throughput(row)[0] if row else None + measured[(profile, scenario)] = value + + series = [("control, 1 stream", "single_mib_per_s"), ("control, N streams", "n_mib_per_s")] + width = 0.8 / (len(series) + len(scenarios)) + figure, (top, bottom) = plt.subplots( + 2, + 1, + figsize=(max(9.0, 3.0 + 2.4 * len(profiles)), 8.0), + height_ratios=(3, 1), + sharex=True, + ) + + for index, (name, field) in enumerate(series): + values = [(doc.controls(profile) or {}).get(field) or 0.0 for profile in profiles] + top.bar( + [p + index * width for p in range(len(profiles))], + values, + width=width, + color="#555555" if index == 0 else "#999999", + label=name, + ) + for index, scenario in enumerate(scenarios): + offset = (len(series) + index) * width + values = [measured[(profile, scenario)] or 0.0 for profile in profiles] + bars = top.bar( + [p + offset for p in range(len(profiles))], + values, + width=width, + color=COLORS(index * 2 % 20), + label=scenario + (" (best cell)" if doc.is_matrix else ""), + ) + for bar, profile, value in zip(bars, profiles, values): + control = (doc.controls(profile) or {}).get("n_mib_per_s") + if control and value: + top.text( + bar.get_x() + bar.get_width() / 2, + value, + f"{value / control * 100:.0f}%", + ha="center", + va="bottom", + fontsize=7, + ) + + top.set_ylabel("MiB/s") + top.set_title("Measured throughput against the link's own control", fontsize=11) + top.legend(fontsize=8, ncol=2) + top.grid(axis="y", alpha=0.3) + + rtts = [(doc.controls(profile) or {}).get("rtt_p50_ms") or 0.0 for profile in profiles] + bottom.bar(range(len(profiles)), rtts, width=0.45, color="#b05a7a") + for index, value in enumerate(rtts): + bottom.text(index, value, f" {value:.1f} ms", va="center", fontsize=8) + bottom.set_ylabel("RTT p50 (ms)") + bottom.set_xticks( + [p + 0.4 - width / 2 for p in range(len(profiles))], profiles, fontsize=9 + ) + bottom.grid(axis="y", alpha=0.3) + + footer( + figure, + doc, + [ + "the percentage over a bar is its share of the N stream control", + "the control is measured with x/crypto/ssh and pkg/sftp, not with " + "easySFTP's uploader: it separates the line from easySFTP, not " + "pkg/sftp from the line", + ], + ) + return save(figure, args, f"link-{slug(document_stem(doc))}") + + +# -------------------------------------------------------------------------- +# auto: the policy against the settings a sweep would have chosen +# -------------------------------------------------------------------------- + + +def auto(args): + """What easySFTP picks for itself, scored against the best cell. + + `auto` chooses a coordinate rather than sitting at one, which is why it is + not a build label in the grid. The regret is the whole point of measuring + it: a policy within roughly 15% on every profile is defensible, one that + only wins on the house line is not. Changing the policy is issue #156; + this only measures it. + """ + written = [] + for doc in bench.documents(args.paths, bench.newest_matrix_json): + rows = [ + row + for row in doc.auto + if selected(args, row.get("scenario"), row.get("link_profile") or "baseline") + ] + if not rows: + continue + written.append(_draw_auto(doc, rows, args)) + return written + + +def _draw_auto(doc, rows, args): + scenarios = sorted({row.get("scenario", "?") for row in rows}) + profiles = sorted({row.get("link_profile") or "baseline" for row in rows}) + width = 0.8 / max(len(profiles), 1) + + figure, (top, bottom) = plt.subplots( + 2, 1, figsize=(3.0 + 2.4 * len(scenarios), 8.4), sharex=True + ) + + for index, profile in enumerate(profiles): + offset = index * width + picked = [ + next( + ( + row + for row in rows + if row.get("scenario") == scenario + and (row.get("link_profile") or "baseline") == profile + ), + None, + ) + for scenario in scenarios + ] + regrets = [number(row or {}, "regret_percent") or 0.0 for row in picked] + bars = top.bar( + [p + offset for p in range(len(scenarios))], + regrets, + width=width, + color=COLORS(index * 2 % 20), + label=profile, + ) + for bar, row in zip(bars, picked): + if row is None: + continue + chosen = row.get("chosen") or {} + top.text( + bar.get_x() + bar.get_width() / 2, + bar.get_height(), + "/".join( + axis_label(key, chosen.get(key)) + for key in ("connections", "concurrency", "request_concurrency") + ), + ha="center", + va="bottom", + fontsize=7, + rotation=90, + ) + + # Pale bar first, solid and narrower on top of it: the same x, so the + # visible pale part above the solid one is exactly what was left on the + # table. + bottom.bar( + [p + offset for p in range(len(scenarios))], + [number((row or {}).get("best") or {}, "mib_per_s") or 0.0 for row in picked], + width=width, + color=COLORS(index * 2 % 20), + alpha=0.3, + edgecolor=COLORS(index * 2 % 20), + linewidth=0.6, + ) + bottom.bar( + [p + offset for p in range(len(scenarios))], + [number(row or {}, "mib_per_s") or 0.0 for row in picked], + width=width * 0.55, + color=COLORS(index * 2 % 20), + ) + + # In the legend rather than on the axes: the bars fill the width, and a + # label written over them is unreadable exactly where it matters. + top.axhline( + 15, + linestyle="--", + color="crimson", + linewidth=1, + label="15%, the line a defensible policy stays under", + ) + # Headroom for the rotated coordinate labels, which sit on top of the bars. + top.set_ylim( + 0, max([number(row, "regret_percent") or 0.0 for row in rows] + [16.0]) * 1.35 + ) + top.set_ylabel("regret (%) against the best cell") + top.set_title( + "What easySFTP picks for itself (label: connections/concurrency/request_concurrency)", + fontsize=11, + ) + top.legend(fontsize=8, ncol=2) + top.grid(axis="y", alpha=0.3) + + bottom.set_ylabel("MiB/s") + bottom.set_xticks( + [p + 0.4 - width / 2 for p in range(len(scenarios))], scenarios, fontsize=9 + ) + bottom.legend( + handles=[ + matplotlib.patches.Patch(facecolor="#555555", label="what auto reached"), + matplotlib.patches.Patch( + facecolor="#555555", alpha=0.3, label="the fastest cell measured" + ), + ], + fontsize=8, + ) + bottom.grid(axis="y", alpha=0.3) + bottom.set_title("color: the link profile, as above", fontsize=9) -def trend(paths, out): + outside = [ + f"{row.get('scenario')} / {row.get('link_profile') or 'baseline'}" + for row in rows + if row.get("chosen_in_grid") is False + ] + notes = [ + "the picked settings are read back from the run's own counters, so they " + "are what easySFTP did and not what this script assumes" + ] + if outside: + notes.append( + "picked a coordinate that is not in the grid for " + ", ".join(outside) + + ": the regret there is against the nearest measured cells, not an " + "exact pair" + ) + footer(figure, doc, notes) + return save(figure, args, f"auto-{slug(document_stem(doc))}") + + +# -------------------------------------------------------------------------- +# canary: did the line hold still for the whole sweep +# -------------------------------------------------------------------------- + + +def canary(args): + """One fixed cell, measured at the start, the middle and the end. + + A spread larger than the deltas the sweep is read for means the server or + the line moved during the run, and the whole thing is a poor comparison + basis. That is a property of the run, not of any cell in it. + """ + written = [] + for doc in bench.documents(args.paths, bench.newest_matrix_json): + rows = [ + row + for row in doc.canary + if selected(args, row.get("scenario"), row.get("link_profile") or "baseline") + ] + if not rows: + continue + written.append(_draw_canary(doc, rows, args)) + return written + + +def _draw_canary(doc, rows, args): + # "mid" is what the harness writes; the longer spelling is accepted so a + # rename does not silently sort the end of the sweep into the middle. + order = ["start", "mid", "middle", "end"] + stages = [stage for stage in order if any(row.get("at") == stage for row in rows)] + stages += sorted({row.get("at") for row in rows if row.get("at") not in order}) + profiles = sorted({row.get("link_profile") or "baseline" for row in rows}) + + figure, axes = plt.subplots(figsize=(8.5, 5.4)) + spreads = [] + for index, profile in enumerate(profiles): + line = [] + for position, stage in enumerate(stages): + row = next( + ( + row + for row in rows + if (row.get("link_profile") or "baseline") == profile + and row.get("at") == stage + ), + None, + ) + duration = number(row or {}, "duration_ms") + if duration is not None: + line.append((position, duration)) + if not line: + continue + axes.plot( + [p for p, _ in line], + [d for _, d in line], + marker="o", + color=COLORS(index * 2 % 20), + label=profile, + ) + durations = [d for _, d in line] + spread = (max(durations) - min(durations)) / min(durations) * 100 + spreads.append((profile, spread)) + axes.annotate( + f"spread {spread:.1f}%", + (line[-1][0], line[-1][1]), + fontsize=8, + textcoords="offset points", + xytext=(6, 0), + ) + + axes.set_xticks(range(len(stages)), stages) + axes.set_xlim(-0.25, len(stages) - 1 + 0.75) # room for the spread labels + axes.set_ylabel("duration of the fixed cell (ms)") + axes.set_xlabel("when in the profile's grid") + axes.grid(alpha=0.3) + axes.legend(fontsize=8) + sample = rows[0] + axes.set_title( + "Drift check: one fixed cell repeated through the sweep\n" + f"{sample.get('scenario', '?')}, connections {sample.get('connections')}, " + f"concurrency {sample.get('concurrency')}", + fontsize=10, + ) + + notes = [ + "a spread larger than the deltas the sweep is read for means the server " + "or the line moved during the run, and the whole run is a poor " + "comparison basis" + ] + failed = [row for row in rows if row.get("exit_code") not in (0, None)] + if failed: + notes.append(f"{len(failed)} canary run(s) exited non zero") + footer(figure, doc, notes) + return save(figure, args, f"canary-{slug(document_stem(doc))}") + + +# -------------------------------------------------------------------------- +# trend: across the stored releases +# -------------------------------------------------------------------------- + + +def trend(args): """Throughput per scenario across the stored release measurements. Releases only: a manual result is a measurement of whatever someone was trying at the time, and a sweep is not a comparison basis at all. Both are in trend.csv and both are dropped here. """ - path = paths[0] + path = (args.paths or [BENCHMARKS / "trend.csv"])[0] series = defaultdict(list) runners = set() - for row in rows(path): + for row in bench.rows(path): if row.get("kind") != "release": continue - speed = number(row, "mib_per_s") + scenario = row.get("scenario", "?") + if not selected(args, scenario): + continue recorded = row.get("recorded_at") or "" + speed = number(row, "mib_per_s") if speed is None or not recorded: continue try: when = datetime.strptime(recorded, "%Y-%m-%dT%H:%M:%SZ") except ValueError: continue - label = row.get("version") or row.get("label") or "?" - series[row.get("scenario", "?")].append((when, speed, label)) - runners.add(row.get("runner") or "unknown") + runner = row.get("runner") or "unknown" + series[scenario].append( + { + "when": when, + "mib_per_s": speed, + "files_per_s": number(row, "files_per_s"), + "label": row.get("version") or row.get("label") or "?", + "runner": runner, + } + ) + runners.add(runner) if not series: - sys.exit(f"no release measurement in {path}") + return [] - figure, axes = plt.subplots(figsize=(9, 5)) - for scenario, points in sorted(series.items()): - points.sort() - axes.plot( - [p[0] for p in points], [p[1] for p in points], - marker="o", label=scenario, + markers = {runner: mark for runner, mark in zip(sorted(runners), "os^Dv*Xp")} + figure, (top, bottom) = plt.subplots(2, 1, figsize=(10, 8.4), sharex=True) + for index, (scenario, points) in enumerate(sorted(series.items())): + points.sort(key=lambda point: point["when"]) + color = COLORS(index * 2 % 20) + for axes, field in ((top, "mib_per_s"), (bottom, "files_per_s")): + usable = [point for point in points if point.get(field) is not None] + if not usable: + continue + axes.plot( + [point["when"] for point in usable], + [point[field] for point in usable], + color=color, + label=scenario if axes is top else None, + zorder=2, + ) + for point in usable: + axes.scatter( + [point["when"]], + [point[field]], + color=color, + marker=markers.get(point["runner"], "o"), + zorder=3, + ) + # One label per release, over the highest scenario measured for it: one + # per point puts three labels on top of each other on release day. + by_release = defaultdict(list) + for points in series.values(): + for point in points: + by_release[point["when"]].append(point) + for when, points in by_release.items(): + top.annotate( + points[0]["label"], + (when, max(point["mib_per_s"] for point in points)), + fontsize=8, + ha="center", + textcoords="offset points", + xytext=(0, 8), ) - for when, speed, label in points: - axes.annotate(label, (when, speed), fontsize=7, - textcoords="offset points", xytext=(0, 6)) - axes.set_ylabel("MiB/s") - axes.set_xlabel("release") - axes.legend(title="scenario") - axes.grid(alpha=0.3) - axes.set_title("Release benchmarks over time") - # Not decoration: two points measured on different runners or over - # different links are not comparable, and the chart cannot tell them apart. + top.set_ylabel("MiB/s") + bottom.set_ylabel("files/s") + bottom.set_xlabel("release") + top.legend(title="scenario", fontsize=8) + for axes in (top, bottom): + axes.grid(alpha=0.3) + top.set_title("Release benchmarks over time", fontsize=11) + + notes = [ + "marker per runner: " + + ", ".join(f"{mark} {runner}" for runner, mark in markers.items()) + ] + warnings = [] if len(runners) > 1: - figure.text( - 0.5, 0.01, - "measured on more than one runner: these points are not comparable", - ha="center", fontsize=8, color="crimson", + warnings.append( + "measured on more than one runner: points with different markers are " + "not comparable, and since issue #184 the link matters as much as " + "the runner does" ) + footer(figure, None, notes, warnings) + return [save(figure, args, "trend-releases")] - figure.tight_layout() - out.mkdir(parents=True, exist_ok=True) - target = out / "trend-releases.png" - figure.savefig(target, dpi=140) - plt.close(figure) - print(target) + +# -------------------------------------------------------------------------- +# report: everything the given files support, and an index of it +# -------------------------------------------------------------------------- + +REPORT_SECTIONS = [ + ("heatmap", "The grid", "One measured configuration per square: `connections` down, `concurrency` across, colored by throughput and labelled with the median duration. The red square is the fastest cell."), + ("scaling", "Where scaling stops paying", "The same cells as curves against the axis that was actually swept. The dotted line is perfect scaling from the leftmost point; the gap to it is what the setting is worth."), + ("auto", "The policy against the grid", "What easySFTP picks for itself, scored against the fastest cell of the same scenario and profile (issue #184, phase 5; the policy itself is issue #156)."), + ("canary", "Did the line hold still", "One fixed cell repeated through the sweep. A large spread makes the whole run a poor comparison basis."), + ("phases", "Where the wall clock goes", "Phases are wall clock and add up to roughly the run's duration."), + ("operations", "What the round-trips cost", "Cumulative work per operation on the left, per call cost on the right. The two panels are different kinds of number."), + ("deletes", "Deletion", "The clean deployment measured before every run, which is the only measurement of deletion in the harness."), + ("link", "Against the line itself", "Measured throughput next to the control the link probe took over the same path."), + ("trend", "Across releases", "One point per stored release measurement, comparable only within one runner and one link."), +] + + +def report(args): + """Every plot the given files support, plus a Markdown index of them.""" + paths = args.paths or [bench.newest_release_json(), bench.newest_matrix_json()] + matrices, standards = [], [] + for path in paths: + path = Path(path) + if path.suffix == ".csv": + continue + document = bench.load(path) + (matrices if document.is_matrix else standards).append(path) + + produced = {} + for name, function, targets in ( + ("heatmap", heatmap, [Path(p).with_suffix(".csv") for p in matrices]), + ("scaling", scaling, matrices), + ("auto", auto, matrices), + ("canary", canary, matrices), + ("phases", phases, standards + matrices), + ("operations", operations, standards + matrices), + ("deletes", deletes, standards + matrices), + ("link", link, standards + matrices), + ("trend", trend, []), + ): + targets = [path for path in targets if Path(path).exists()] + if not targets and name != "trend": + continue + scoped = argparse.Namespace(**vars(args)) + scoped.paths = targets + try: + produced[name] = function(scoped) + except SystemExit as failure: # a missing input is not a failed report + print(f"{name}: {failure}", file=sys.stderr) + produced[name] = [] + + written = [path for paths in produced.values() for path in paths] + if not written: + return [] + + lines = [ + "# Benchmark analysis", + "", + "Generated by `benchmarks/analysis/plot.py report`. Every image here is a", + "reading of a stored result; if a plot and the JSON disagree, the JSON is", + "right (`benchmarks/analysis/README.md`).", + "", + "## Sources", + "", + ] + for path in paths: + lines.append(f"- `{Path(path).relative_to(bench.REPO).as_posix()}`") + lines.append("") + + for name, title, description in REPORT_SECTIONS: + images = produced.get(name) or [] + if not images: + continue + lines += [f"## {title}", "", description, ""] + for image in images: + lines.append(f"![{image.stem}]({image.name})") + lines.append("") + + args.out.mkdir(parents=True, exist_ok=True) + target = args.out / "report.md" + target.write_text("\n".join(lines), encoding="utf-8") + return written + [target] + + +# -------------------------------------------------------------------------- +# CLI +# -------------------------------------------------------------------------- + +COMMANDS = { + "heatmap": heatmap, + "scaling": scaling, + "phases": phases, + "operations": operations, + "deletes": deletes, + "link": link, + "auto": auto, + "canary": canary, + "trend": trend, + "report": report, +} + + +def csv_list(text): + return [piece.strip() for piece in text.split(",") if piece.strip()] def main(): - parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) - parser.add_argument("command", choices=["heatmap", "trend"]) - parser.add_argument("paths", nargs="*", type=Path) + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument("command", choices=sorted(COMMANDS)) + parser.add_argument( + "paths", + nargs="*", + type=Path, + help="stored files to read; the newest of the right kind when omitted", + ) parser.add_argument("--out", type=Path, default=DEFAULT_OUT) + parser.add_argument("--format", default="png", choices=["png", "svg", "pdf"]) + parser.add_argument("--dpi", type=int, default=140) + parser.add_argument( + "--scenario", + type=csv_list, + default=[], + help="only these scenarios, comma separated", + ) + parser.add_argument( + "--profile", + type=csv_list, + default=[], + help="only these link profiles, comma separated", + ) + parser.add_argument( + "--metric", + default="mib_per_s", + choices=sorted(METRICS), + help="what the heatmap colors and ranks by", + ) + parser.add_argument( + "--include-deletes", + action="store_true", + help="phases: add the delete sweeps as their own bars", + ) args = parser.parse_args() - if args.command == "heatmap": - heatmap(args.paths or newest_matrix(), args.out) - else: - trend(args.paths or [BENCHMARKS / "trend.csv"], args.out) + written = COMMANDS[args.command](args) + if not written: + sys.exit( + f"nothing to plot: no input {args.command} can read had the fields it " + "needs, or the filters excluded everything" + ) + for path in written: + print(path) if __name__ == "__main__": diff --git a/benchmarks/analysis/requirements.txt b/benchmarks/analysis/requirements.txt index 4c8b44f..e5d28e6 100644 --- a/benchmarks/analysis/requirements.txt +++ b/benchmarks/analysis/requirements.txt @@ -1,5 +1,6 @@ # Pinned so a plot made today can be remade next year. Only what plot.py -# imports directly; numpy arrives with matplotlib. +# imports directly; numpy arrives with matplotlib. The self-checks +# (test_plot.py) use unittest from the standard library and add nothing here. # # This file is for the optional analysis layer only. Running, storing or # validating a benchmark needs none of it (issue #190). diff --git a/benchmarks/analysis/test_plot.py b/benchmarks/analysis/test_plot.py new file mode 100644 index 0000000..093de4f --- /dev/null +++ b/benchmarks/analysis/test_plot.py @@ -0,0 +1,236 @@ +#!/usr/bin/env python3 +"""Self-checks for the analysis layer. + + python -m unittest discover -s benchmarks/analysis + +Two things are worth asserting here and nothing else is. First, that the +readers keep reading every result already committed: the stored files span +several schema versions and the newest one is not the only one on disk, so +`benchdata` is exercised against all of them rather than against a fixture +written today. Second, that every command draws something from the newest +stored files, into a temporary directory, because a plot that raises on a +field that moved is the failure mode this layer actually has. + +Nothing in here writes into `benchmarks/`, and nothing in here needs a +benchmark to have been run. +""" + +import json +import sys +import tempfile +import unittest +from argparse import Namespace +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import benchdata as bench # noqa: E402 +import plot # noqa: E402 + +STORED = sorted( + path + for path in bench.BENCHMARKS.rglob("*.json") + if path.name not in {"index.json"} +) + + +def options(out, **overrides): + settings = { + "paths": [], + "out": Path(out), + "format": "png", + "dpi": 70, # small: these files are thrown away at the end of the test + "scenario": [], + "profile": [], + "metric": "mib_per_s", + "include_deletes": False, + } + settings.update(overrides) + return Namespace(**settings) + + +class Helpers(unittest.TestCase): + def test_number_treats_blank_as_absent(self): + row = {"a": "", "b": "3.5", "c": 7, "d": "x", "e": None} + self.assertIsNone(bench.number(row, "a")) + self.assertIsNone(bench.number(row, "missing")) + self.assertIsNone(bench.number(row, "d")) + self.assertIsNone(bench.number(row, "e")) + self.assertEqual(bench.number(row, "b"), 3.5) + self.assertEqual(bench.number(row, "c"), 7.0) + + def test_axis_layout_follows_what_varies(self): + # A payload that only sweeps request_concurrency (`single`, one file) + # must not get concurrency on the x axis: that is a one point line. + points = [ + {"connections": 1, "concurrency": 1, "request_concurrency": value} + for value in (1, 16, 64) + ] + x, series, facets = bench.axis_layout(points) + self.assertEqual(x, "request_concurrency") + self.assertEqual(facets, []) + self.assertIn(series, ("concurrency", "connections")) + + points = [ + {"connections": c, "concurrency": w, "request_concurrency": None} + for c in (1, 2, 4) + for w in (1, 2, 4, 8) + ] + x, series, facets = bench.axis_layout(points) + self.assertEqual((x, series), ("concurrency", "connections")) + self.assertEqual(facets, []) + + def test_axis_label_keeps_null_apart_from_a_number(self): + self.assertEqual(bench.axis_label("request_concurrency", None), "default") + self.assertEqual(bench.axis_label("request_concurrency", 16), "16") + self.assertEqual(bench.axis_label("connections", 4.0), "4") + + def test_sort_key_puts_the_unset_value_first(self): + self.assertEqual( + sorted([16, None, 1], key=plot.sort_key), [None, 1, 16] + ) + + +class Documents(unittest.TestCase): + def test_every_stored_result_loads(self): + self.assertTrue(STORED, "no stored result found to read") + for path in STORED: + with self.subTest(path=path.name): + doc = bench.load(path) + self.assertIn(doc.kind, {"release", "manual", "matrix", "standard"}) + self.assertTrue(doc.results, "a stored result with no rows") + self.assertTrue(doc.provenance()) + self.assertIsInstance(doc.caveats(), list) + + def test_throughput_is_recomputed_when_a_row_lacks_it(self): + doc = bench.Document("x.json", {}) + mib, files = doc.throughput( + {"bytes": 1024 * 1024, "files": 10, "median_ms": 1000} + ) + self.assertAlmostEqual(mib, 1.0) + self.assertAlmostEqual(files, 10.0) + self.assertEqual(doc.throughput({"median_ms": 0}), (None, None)) + + def test_a_baseline_only_run_gets_no_shaping_caveat(self): + # "baseline" is the real line and is never shaped, so asking for it and + # getting it is not a caveat. + doc = bench.Document( + "x.json", + { + "benchmark": { + "link": { + "shaping": { + "available": False, + "reason": "no profile asked for shaping", + "requested": ["baseline"], + "applied": ["baseline"], + } + } + } + }, + ) + self.assertEqual(doc.caveats(), []) + + def test_an_unshaped_profile_is_a_caveat(self): + doc = bench.Document( + "x.json", + { + "benchmark": { + "link": { + "shaping": { + "available": False, + "reason": "tc is not installed", + "requested": ["baseline", "+50ms"], + "applied": ["baseline"], + } + } + } + }, + ) + self.assertEqual(len(doc.caveats()), 1) + self.assertIn("say what was asked for", doc.caveats()[0]) + + def test_a_bare_measurement_without_an_envelope_still_loads(self): + # results.json as the harness writes it, before store wraps it. + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "results.json" + path.write_text( + json.dumps( + { + "benchmark_kind": "standard", + "candidate_ref": "x (1234567)", + "runner": "test", + "results": [{"scenario": "small", "median_ms": 1}], + } + ), + encoding="utf-8", + ) + doc = bench.load(path) + self.assertEqual(doc.kind, "standard") + self.assertEqual(len(doc.results), 1) + + +class Commands(unittest.TestCase): + """Every command against the newest stored files of its kind.""" + + def run_command(self, name, **overrides): + with tempfile.TemporaryDirectory() as directory: + written = plot.COMMANDS[name](options(directory, **overrides)) + self.assertTrue(written, f"{name} drew nothing") + for path in written: + self.assertTrue(path.exists(), f"{name} named a file it did not write") + self.assertGreater(path.stat().st_size, 0) + return [path.name for path in written] + + def test_heatmap(self): + self.run_command("heatmap", scenario=["small"], profile=["baseline"]) + + def test_scaling(self): + self.run_command("scaling", scenario=["small"], profile=["baseline"]) + + def test_auto(self): + self.run_command("auto", scenario=["small"]) + + def test_canary(self): + self.run_command("canary") + + def test_phases(self): + self.run_command("phases", include_deletes=True) + + def test_operations(self): + names = self.run_command("operations", scenario=["small"]) + for name in names: + # The file is named after the row it shows, not after whatever a + # loop inside the drawing code left behind. + self.assertIn("small", name) + + def test_deletes(self): + self.run_command("deletes") + + def test_link(self): + self.run_command("link") + + def test_trend(self): + self.run_command("trend") + + def test_report_writes_an_index_of_what_it_drew(self): + with tempfile.TemporaryDirectory() as directory: + written = plot.report( + options(directory, scenario=["small"], profile=["baseline"]) + ) + index = Path(directory) / "report.md" + self.assertIn(index, written) + text = index.read_text(encoding="utf-8") + for path in written: + if path.suffix == ".png": + self.assertIn(path.name, text, "an image the index does not list") + + def test_a_filter_that_matches_nothing_draws_nothing(self): + with tempfile.TemporaryDirectory() as directory: + self.assertEqual( + plot.scaling(options(directory, scenario=["no-such-scenario"])), [] + ) + + +if __name__ == "__main__": + unittest.main()