diff --git a/CLAUDE.md b/CLAUDE.md index ff2bba47..f1b16aab 100755 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -33,27 +33,34 @@ pytest tests/integration -v ## CLI Usage -The main entry point is `mlpstorage` with nested subcommands: +The main entry point is `mlpstorage`. Benchmark commands sit under a required +submission-mode positional (`closed`, `open`, `whatif`); commands that touch +storage take a trailing `file|object` selector. Only training has a model +positional — the other benchmarks select models via flags. `mlpstorage +--help_all` prints the complete hand-curated command reference (kept in +lockstep with the parser by `tests/unit/test_help_all_parity.py`). ```bash -# Training benchmarks (unet3d, resnet50, cosmoflow) -mlpstorage training datasize ... # Calculate required dataset size -mlpstorage training datagen ... # Generate synthetic data -mlpstorage training run ... # Execute benchmark -mlpstorage training configview ... # View final configuration +# Training (model positional; closed/open models: unet3d, retinanet) +mlpstorage closed training unet3d datasize ... # Calculate required dataset size +mlpstorage closed training unet3d datagen file ... # Generate synthetic data +mlpstorage closed training unet3d run file ... # Execute benchmark +mlpstorage closed training unet3d configview file ... # View final configuration -# Checkpointing benchmarks (llama3-8b, llama3-70b, llama3-405b, llama3-1t) -mlpstorage checkpointing run ... -mlpstorage checkpointing datagen ... -mlpstorage checkpointing validate ... +# Checkpointing (--model/-m required: llama3-8b, llama3-70b, llama3-405b, llama3-1t) +mlpstorage closed checkpointing datasize --model llama3-8b ... +mlpstorage closed checkpointing run file --model llama3-8b ... +mlpstorage closed checkpointing configview file --model llama3-8b ... # Other benchmarks -mlpstorage vectordb run ... # Vector database (PREVIEW) -mlpstorage kvcache run ... # KV cache - -# Utilities -mlpstorage reports reportgen ... # Generate submission reports -mlpstorage history list/replay ... # Command history +mlpstorage closed vectordb run file ... # Vector database (index via --vdb-index) +mlpstorage closed kvcache run ... # KV cache (no file|object; model fixed in closed) + +# Utilities (top-level, no mode positional) +mlpstorage reports reportgen ... # Generate submission reports +mlpstorage history show/rerun ... # Command history +mlpstorage init # Pin orgname to a results-dir +mlpstorage validate # Rules.md submission checker ``` ## Architecture @@ -132,20 +139,20 @@ When running the `mlpstorage` CLI for manual testing or integration tests, use: ```bash # Generate dataset for unet3d with 4 processes -mlpstorage training datagen \ - --model unet3d \ +mlpstorage closed training unet3d datagen file \ --num-processes 4 \ --data-dir /databases/mlps-v3.0/data/ \ - --results-dir /databases/mlps-v3.0/results + --results-dir /databases/mlps-v3.0/results \ + --systemname dev-system -# Run training benchmark for unet3d with 2 h100 accelerators -mlpstorage training run \ - --model unet3d \ +# Run training benchmark for unet3d with 2 b200 accelerators +mlpstorage closed training unet3d run file \ --num-accelerators 2 \ - --accelerator-type h100 \ + --accelerator-type b200 \ --client-host-memory-in-gb 64 \ --data-dir /databases/mlps-v3.0/data/ \ - --results-dir /databases/mlps-v3.0/results + --results-dir /databases/mlps-v3.0/results \ + --systemname dev-system ``` **Note**: These benchmarks require MPI (OpenMPI) to be installed. Install with: @@ -160,10 +167,10 @@ sudo yum install openmpi ## Key Constants From `mlpstorage/config.py`: -- Training models: `cosmoflow`, `resnet50`, `unet3d` +- Training models: `unet3d`, `retinanet` (closed/open); whatif adds `cosmoflow`, `resnet50`, `dlrm`, `flux` - LLM models (checkpointing): `llama3-8b`, `llama3-70b`, `llama3-405b`, `llama3-1t` -- Accelerators: `h100`, `a100` -- Submission categories: `CLOSED`, `OPEN` +- Accelerators: `b200`, `mi355` (closed/open); whatif adds `h100`, `a100` +- Submission modes: `closed`, `open`, `whatif` ## GSD Workflow diff --git a/mlpstorage_py/cli/common_args.py b/mlpstorage_py/cli/common_args.py index adcac735..f4139884 100755 --- a/mlpstorage_py/cli/common_args.py +++ b/mlpstorage_py/cli/common_args.py @@ -293,7 +293,8 @@ def add_universal_arguments(parser, req_results, req_systemname=False): output_control.add_argument( "--stream-log-level", type=str, - default="INFO" + default="INFO", + help="Logging level for console output (default: INFO)" ) output_control.add_argument( '--quiet', @@ -360,11 +361,13 @@ def add_mpi_arguments(parser): ) mpi_options.add_argument( '--oversubscribe', - action="store_true" + action="store_true", + help="Allow launching more MPI ranks than available CPU slots" ) mpi_options.add_argument( '--allow-run-as-root', - action="store_true" + action="store_true", + help="Permit MPI execution as the root user (OpenMPI --allow-run-as-root)" ) mpi_options.add_argument( '--mpi-btl', diff --git a/mlpstorage_py/cli/help_formatter.py b/mlpstorage_py/cli/help_formatter.py index 9d5375b5..039a05ff 100644 --- a/mlpstorage_py/cli/help_formatter.py +++ b/mlpstorage_py/cli/help_formatter.py @@ -20,14 +20,22 @@ SYNOPSIS_TEXT = """\ SYNOPSIS - mlpstorage [OPTIONS] + mlpstorage training [OPTIONS] + mlpstorage checkpointing [OPTIONS] + mlpstorage vectordb [OPTIONS] + mlpstorage kvcache [OPTIONS] mlpstorage (reports|history|lockfile|version) [subcommand] [OPTIONS] mlpstorage init mlpstorage validate [OPTIONS] mlpstorage rules-coverage [--rules-md PATH] — required first positional for benchmark commands - — required second positional (see per-benchmark choices below) + — training only: required model positional + (choices vary by mode; see the tree below). + The other benchmarks select models via flags: + checkpointing --model/-m (required), + vectordb --vdb-index, kvcache --model/-m + (open|whatif only; fixed in closed) — required storage selector for commands that touch storage (absent on datasize; absent on all kvcache commands)""" @@ -36,52 +44,47 @@ │ ├── closed ────────────────────────────────────────────────────── │ ├── training -│ │ └── unet3d | retinanet +│ │ └── unet3d | retinanet ← model positional │ │ ├── datasize {TR_DATASIZE_CLOSED} │ │ ├── datagen file | object {TR_DATAGEN_CLOSED} │ │ ├── run file | object {TR_RUN_CLOSED} │ │ └── configview file | object {TR_CONFIGVIEW_CLOSED} │ │ -│ ├── checkpointing -│ │ └── llama3-8b | llama3-70b | llama3-405b | llama3-1t -│ │ ├── datasize {CK_DATASIZE_CLOSED} -│ │ ├── run file | object {CK_RUN_CLOSED} -│ │ └── configview file | object {CK_CONFIGVIEW_CLOSED} +│ ├── checkpointing ← model via --model/-m flag (required) +│ │ ├── datasize {CK_DATASIZE_CLOSED} +│ │ ├── run file | object {CK_RUN_CLOSED} +│ │ └── configview file | object {CK_CONFIGVIEW_CLOSED} │ │ -│ ├── vectordb -│ │ └── DISKANN | HNSW | AISAQ -│ │ ├── datasize {VDB_DATASIZE_CLOSED} -│ │ ├── datagen file | object {VDB_DATAGEN_CLOSED} -│ │ └── run file | object {VDB_RUN_CLOSED} +│ ├── vectordb ← index via --vdb-index flag +│ │ ├── datasize {VDB_DATASIZE_CLOSED} +│ │ ├── datagen file | object {VDB_DATAGEN_CLOSED} +│ │ └── run file | object {VDB_RUN_CLOSED} │ │ -│ └── kvcache ← no model positional in closed +│ └── kvcache ← model fixed in closed │ ├── datasize {KV_DATASIZE_CLOSED} │ └── run {KV_RUN_CLOSED} │ ├── open ──────────────────────────────────────────────────────── │ ├── training -│ │ └── unet3d | retinanet +│ │ └── unet3d | retinanet ← model positional │ │ ├── datasize {TR_DATASIZE_OPEN} │ │ ├── datagen file | object {TR_DATAGEN_OPEN} │ │ ├── run file | object {TR_RUN_OPEN} │ │ └── configview file | object {TR_CONFIGVIEW_OPEN} │ │ -│ ├── checkpointing -│ │ └── llama3-8b | llama3-70b | llama3-405b | llama3-1t -│ │ ├── datasize {CK_DATASIZE_OPEN} -│ │ ├── run file | object {CK_RUN_OPEN} -│ │ └── configview file | object {CK_CONFIGVIEW_OPEN} +│ ├── checkpointing ← model via --model/-m flag (required) +│ │ ├── datasize {CK_DATASIZE_OPEN} +│ │ ├── run file | object {CK_RUN_OPEN} +│ │ └── configview file | object {CK_CONFIGVIEW_OPEN} │ │ -│ ├── vectordb -│ │ └── DISKANN | HNSW | AISAQ | IVF_FLAT | IVF_SQ8 | FLAT -│ │ ├── datasize {VDB_DATASIZE_OPEN} -│ │ ├── datagen file | object {VDB_DATAGEN_OPEN} -│ │ └── run file | object {VDB_RUN_OPEN} +│ ├── vectordb ← index via --vdb-index flag +│ │ ├── datasize {VDB_DATASIZE_OPEN} +│ │ ├── datagen file | object {VDB_DATAGEN_OPEN} +│ │ └── run file | object {VDB_RUN_OPEN} │ │ -│ └── kvcache -│ └── tiny-1b | mistral-7b | llama2-7b | llama3.1-8b | llama3.1-70b-instruct -│ ├── datasize {KV_DATASIZE_OPEN} -│ └── run {KV_RUN_OPEN} +│ └── kvcache ← model via --model/-m flag (default: tiny-1b) +│ ├── datasize {KV_DATASIZE_OPEN} +│ └── run {KV_RUN_OPEN} │ ├── whatif ────────────────────────────────────────────────────── │ ├── training @@ -91,29 +94,26 @@ │ │ ├── run file | object {TR_RUN_WHATIF} │ │ └── configview file | object {TR_CONFIGVIEW_WHATIF} │ │ -│ ├── checkpointing -│ │ └── llama3-8b | llama3-70b | llama3-405b | llama3-1t -│ │ ├── datasize {CK_DATASIZE_WHATIF} -│ │ ├── run file | object {CK_RUN_WHATIF} -│ │ └── configview file | object {CK_CONFIGVIEW_WHATIF} +│ ├── checkpointing ← model via --model/-m flag (required) +│ │ ├── datasize {CK_DATASIZE_WHATIF} +│ │ ├── run file | object {CK_RUN_WHATIF} +│ │ └── configview file | object {CK_CONFIGVIEW_WHATIF} │ │ -│ ├── vectordb -│ │ └── DISKANN | HNSW | AISAQ | IVF_FLAT | IVF_SQ8 | FLAT -│ │ ├── datasize {VDB_DATASIZE_WHATIF} -│ │ ├── datagen file | object {VDB_DATAGEN_WHATIF} -│ │ └── run file | object {VDB_RUN_WHATIF} +│ ├── vectordb ← index via --vdb-index flag +│ │ ├── datasize {VDB_DATASIZE_WHATIF} +│ │ ├── datagen file | object {VDB_DATAGEN_WHATIF} +│ │ └── run file | object {VDB_RUN_WHATIF} │ │ -│ └── kvcache -│ └── tiny-1b | mistral-7b | llama2-7b | llama3.1-8b | llama3.1-70b-instruct -│ ├── datasize {KV_DATASIZE_WHATIF} -│ └── run {KV_RUN_WHATIF} +│ └── kvcache ← model via --model/-m flag (default: tiny-1b) +│ ├── datasize {KV_DATASIZE_WHATIF} +│ └── run {KV_RUN_WHATIF} │ ├── reports │ └── reportgen {RP_REPORTGEN} │ ├── history -│ ├── list {HI_LIST} -│ └── replay {HI_REPLAY} +│ ├── show {HI_SHOW} +│ └── rerun {HI_RERUN} │ ├── lockfile │ ├── generate {LF_GENERATE} @@ -121,31 +121,35 @@ │ ├── init Pin orgname to a results-dir via the mlperf-results.yaml sentinel │ -├── validate Run the Rules.md submission checker -│ --submitters CSV Comma-separated submitter allowlist (default: all) -│ --mlperf-version VERSION Spec version (default: v5.1) -│ --csv PATH Summary CSV path (default: summary.csv) -│ --skip-output-file Suppress per-submission output file -│ --reference-checksum MD5 Override REFERENCE_CHECKSUMS for code/ MD5 check +├── validate {VALIDATE} │ -├── rules-coverage Reconcile Rules.md IDs against @rule-decorated checks -│ --rules-md PATH Path to Rules.md (default: project-root Rules.md) +├── rules-coverage {RULES_COVERAGE} │ └── version {VERSION} Common argument groups -CORE_STD — Standard arguments, all three modes +CORE_STD — Standard arguments, every benchmark command and most utilities --results-dir/-rd PATH Benchmark results directory + (default: MLPERF_RESULTS_DIR env var, else a tempdir) + --systemname/-sn NAME System-under-test name — folder under results/ + (default: MLPERF_SYSTEMNAME env var) --config-file/-c PATH YAML overrides file (applied after CLI args) --debug Enable debug output --verbose Enable verbose output --stream-log-level LEVEL Logging level (default: INFO) + --quiet Suppress the run configuration summary table --dry-run Print the command that would execute; do not run --verify-lockfile PATH Validate installed packages against lockfile --skip-validation Skip MPI/SSH/DLIO pre-run environment checks - -OPEN_STD — Additional standard arguments, open and whatif modes only + --skip-ssh-check Skip only the SSH connectivity preflight (for + scheduler-launched runs: PALS mpiexec, srun) + --skip-fs-separation-gate Bypass the CAP-03 same-filesystem hard gate + (probe still runs; Rules.md 3.4.2/4.4.2/5.4.2 + still fail at validation time) + +OPEN_STD — Additional standard arguments, open and whatif modes + (all commands except kvcache datasize) --loops N Repeat benchmark N times (default: 1) --allow-invalid-params/-aip Do not abort on invalid DLIO parameters @@ -156,7 +160,9 @@ --allow-run-as-root Permit execution as root (OpenMPI flag) --mpi-params PARAM... Additional raw MPI parameters (repeatable) -TIMESERIES — Time-series host metrics, open and whatif run commands only +TIMESERIES — Time-series host metrics, run commands only + (training and checkpointing: open and whatif modes; + vectordb and kvcache: all three modes) --timeseries-interval SECS Sample interval in seconds (default: 10.0) --skip-timeseries Disable time-series collection entirely --max-timeseries-samples N Per-host sample cap (default: 3600) @@ -166,20 +172,21 @@ TR_DATASIZE_CLOSED Required: --max-accelerators/-ma N - --accelerator-type/-g {b200,mi355} + --accelerator-type/-at {b200,mi355} --client-host-memory-in-gb/-cm N - --data-dir/-dd PATH + --systemname/-sn NAME (or MLPERF_SYSTEMNAME) Optional: + --data-dir/-dd PATH --num-client-hosts/-nc N Derived from --hosts count if unset --dlio-bin-path/-dp PATH --exec-type/-et {mpi,docker} (default: mpi) --hosts/-s HOST... (default: 127.0.0.1) + --params/-p/--param KEY=VALUE... DLIO overrides (CLOSED: restricted subset) + MPI_ARGS + CORE_STD (--results-dir optional) TR_DATASIZE_OPEN - = TR_DATASIZE_CLOSED plus: - --params/-p KEY=VALUE... DLIO parameter overrides (repeatable) + = TR_DATASIZE_CLOSED (flags identical; --params unrestricted) + OPEN_STD TR_DATASIZE_WHATIF @@ -191,19 +198,21 @@ TR_DATAGEN_CLOSED Required: --num-processes/-np N - --results-dir/-rd PATH - --data-dir/-dd PATH + --systemname/-sn NAME (or MLPERF_SYSTEMNAME) + --data-dir/-dd PATH (required with file storage; object mode + may supply data_dir via --config-file) [storage positional: file | object] Optional: --dlio-bin-path/-dp PATH --exec-type/-et {mpi,docker} (default: mpi) --hosts/-s HOST... (default: 127.0.0.1) + --o-direct Route I/O through s3dlio's O_DIRECT local-fs mode + --params/-p/--param KEY=VALUE... DLIO overrides (CLOSED: restricted subset) + MPI_ARGS - + CORE_STD + + CORE_STD (--results-dir optional) TR_DATAGEN_OPEN - = TR_DATAGEN_CLOSED plus: - --params/-p KEY=VALUE... + = TR_DATAGEN_CLOSED (flags identical; --params unrestricted) + OPEN_STD TR_DATAGEN_WHATIF @@ -214,23 +223,27 @@ TR_RUN_CLOSED Required: --num-accelerators/-na N - --accelerator-type/-g {b200,mi355} + --accelerator-type/-at {b200,mi355} --client-host-memory-in-gb/-cm N - --checkpoint-folder/-cf PATH - --results-dir/-rd PATH - --data-dir/-dd PATH + --results-dir/-rd PATH (or MLPERF_RESULTS_DIR) + --systemname/-sn NAME (or MLPERF_SYSTEMNAME) + --data-dir/-dd PATH (required with file storage; object mode + may supply data_dir via --config-file) [storage positional: file | object] Optional: --num-client-hosts/-nc N --dlio-bin-path/-dp PATH --exec-type/-et {mpi,docker} (default: mpi) --hosts/-s HOST... (default: 127.0.0.1) + --o-direct Route I/O through s3dlio's O_DIRECT local-fs mode + --drop-caches-timeout-seconds N Per-call timeout for the per-epoch + page-cache flush + --params/-p/--param KEY=VALUE... DLIO overrides (CLOSED: restricted subset) + MPI_ARGS + CORE_STD TR_RUN_OPEN - = TR_RUN_CLOSED plus: - --params/-p KEY=VALUE... + = TR_RUN_CLOSED (flags identical; --params unrestricted) + OPEN_STD + TIMESERIES @@ -243,60 +256,78 @@ TR_CONFIGVIEW_CLOSED Required: --num-accelerators/-na N - --results-dir/-rd PATH - --data-dir/-dd PATH + --accelerator-type/-at {b200,mi355} + --client-host-memory-in-gb/-cm N + --results-dir/-rd PATH (or MLPERF_RESULTS_DIR) + --systemname/-sn NAME (or MLPERF_SYSTEMNAME) [storage positional: file | object] Optional: + --data-dir/-dd PATH + --num-client-hosts/-nc N --dlio-bin-path/-dp PATH + --exec-type/-et {mpi,docker} (default: mpi) + --hosts/-s HOST... (default: 127.0.0.1) + --o-direct + --params/-p/--param KEY=VALUE... + + MPI_ARGS + CORE_STD TR_CONFIGVIEW_OPEN - = TR_CONFIGVIEW_CLOSED plus: - --params/-p KEY=VALUE... + = TR_CONFIGVIEW_CLOSED (flags identical; --params unrestricted) + OPEN_STD TR_CONFIGVIEW_WHATIF - = TR_CONFIGVIEW_OPEN (model positional choices differ; flags identical) + = TR_CONFIGVIEW_OPEN but: + --accelerator-type choices: {h100,a100,b200,mi355} Placeholder definitions — CHECKPOINTING CK_DATASIZE_CLOSED Required: + --model/-m {llama3-8b,llama3-70b,llama3-405b,llama3-1t} + --num-processes/-np N --client-host-memory-in-gb/-cm N Optional: --hosts/-s HOST... (default: 127.0.0.1) + --exec-type/-et {mpi,docker} (default: mpi) + --dlio-bin-path/-dp PATH --num-checkpoints-read/-ncr N (default: 10; closed allows 10 or 0) --num-checkpoints-write/-ncw N (default: 10; closed allows 10 or 0) - + CORE_STD (--results-dir optional) + --checkpoint-subset (8B at 8 processes only; sizes a Subset run) + + MPI_ARGS + + CORE_STD (--results-dir and --systemname optional) Note: closed runs use 10/10 by default. Use 10/0 then 0/10 in two invocations when a cache flush is required between phases (see Rules.md §4.7.1 and checkpointing/README.md). CK_DATASIZE_OPEN = CK_DATASIZE_CLOSED plus: - --dlio-bin-path/-dp PATH - --params/-p KEY=VALUE... + --params/-p KEY=VALUE... DLIO parameter overrides (repeatable) + OPEN_STD Note: open allows any non-negative integer for --num-checkpoints-read/-write CK_DATASIZE_WHATIF - = CK_DATASIZE_OPEN (model positional choices identical; flags identical) + = CK_DATASIZE_OPEN (--model choices identical; flags identical) ────────────────────────────────────────────────────────────────── CK_RUN_CLOSED Required: + --model/-m {llama3-8b,llama3-70b,llama3-405b,llama3-1t} --num-processes/-np N --checkpoint-folder/-cf PATH --client-host-memory-in-gb/-cm N - --results-dir/-rd PATH + --results-dir/-rd PATH (or MLPERF_RESULTS_DIR) + --systemname/-sn NAME (or MLPERF_SYSTEMNAME) [storage positional: file | object] Optional: --checkpoint-subset (8B at 8 processes only; declares a Subset run) --exec-type/-et {mpi,docker} (default: mpi) --hosts/-s HOST... (default: 127.0.0.1) + --dlio-bin-path/-dp PATH --num-checkpoints-read/-ncr N (default: 10; closed allows 10 or 0) --num-checkpoints-write/-ncw N (default: 10; closed allows 10 or 0) + --o-direct Route I/O through s3dlio's O_DIRECT local-fs mode + MPI_ARGS + CORE_STD Note: closed runs use 10/10 by default. Use 10/0 then 0/10 in two @@ -311,7 +342,6 @@ CK_RUN_OPEN = CK_RUN_CLOSED plus: - --dlio-bin-path/-dp PATH --params/-p KEY=VALUE... + OPEN_STD + TIMESERIES @@ -319,16 +349,27 @@ and any non-negative integer for --num-checkpoints-read/-write CK_RUN_WHATIF - = CK_RUN_OPEN (model positional choices identical; flags identical) + = CK_RUN_OPEN (--model choices identical; flags identical) ────────────────────────────────────────────────────────────────── CK_CONFIGVIEW_CLOSED Required: - --results-dir/-rd PATH + --model/-m {llama3-8b,llama3-70b,llama3-405b,llama3-1t} + --num-processes/-np N + --client-host-memory-in-gb/-cm N + --results-dir/-rd PATH (or MLPERF_RESULTS_DIR) + --systemname/-sn NAME (or MLPERF_SYSTEMNAME) [storage positional: file | object] Optional: + --checkpoint-subset + --exec-type/-et {mpi,docker} (default: mpi) + --hosts/-s HOST... (default: 127.0.0.1) --dlio-bin-path/-dp PATH + --num-checkpoints-read/-ncr N (default: 10) + --num-checkpoints-write/-ncw N (default: 10) + --o-direct + + MPI_ARGS + CORE_STD CK_CONFIGVIEW_OPEN @@ -343,28 +384,39 @@ VDB_DATASIZE_CLOSED Optional: + --vdb-engine {milvus} (default: milvus) + --vdb-index {DISKANN,HNSW,AISAQ} Index family; names the result path + vector_database///... + --index-type {DISKANN,HNSW,AISAQ} Milvus index for storage estimation + (defaults to --vdb-index) --dimension N (default: 1536) --num-vectors N (default: 1,000,000) - --index-type {DISKANN,HNSW,AISAQ} (default: DISKANN) --num-shards N (default: 1) --vector-dtype {FLOAT_VECTOR} (default: FLOAT_VECTOR) - + CORE_STD (--results-dir optional) + + CORE_STD (--results-dir and --systemname optional) VDB_DATASIZE_OPEN - = VDB_DATASIZE_CLOSED but: - --index-type choices: {DISKANN,HNSW,AISAQ,IVF_FLAT,IVF_SQ8,FLAT} + = VDB_DATASIZE_CLOSED plus: + --vdb-index {DISKANN,HNSW,AISAQ,IVF_FLAT,IVF_SQ8,FLAT} (open widens choices) + --index-type {DISKANN,HNSW,AISAQ,IVF_FLAT,IVF_SQ8,FLAT} + --params KEY=VALUE... + OPEN_STD VDB_DATASIZE_WHATIF - = VDB_DATASIZE_OPEN (algorithm positional choices differ; flags identical) + = VDB_DATASIZE_OPEN (flags identical) ────────────────────────────────────────────────────────────────── VDB_DATAGEN_CLOSED Required: - --results-dir/-rd PATH + --results-dir/-rd PATH (or MLPERF_RESULTS_DIR) + --systemname/-sn NAME (or MLPERF_SYSTEMNAME) [storage positional: file | object] Optional: + --vdb-engine {milvus} (default: milvus) + --vdb-index {DISKANN,HNSW,AISAQ} + --index-type {DISKANN,HNSW,AISAQ} Milvus index to create during load + (defaults to --vdb-index) --host/-s IP Milvus server address (default: 127.0.0.1) --port/-p N Milvus port (default: 19530) --collection NAME @@ -377,22 +429,51 @@ --batch-size N (default: 1,000) --chunk-size N (default: 10,000) --force + VDB storage location (recorded for Rules.md 5.4.1; overrides config storage.*): + --storage-root PATH Where the engine stores its data + (must differ from --results-dir) + --storage-type TYPE Storage medium, e.g. local_fs, s3 + (default: local_fs) + Distributed launch: + --distributed Fan datagen out across --hosts via MPI + --hosts HOST... (no -s short form here; -s is --host) + --npernode/--num-processes-per-client N (default: 1) + --mpi-impl {mpich,openmpi} (default: mpich) + --coordination {filesystem,mpi} (default: filesystem) + --rank-output-dir PATH (default: /tmp/mlps_vdb) + --seed N (default: 42) + --ready-timeout SECS (default: 7200) + + MPI_ARGS + CORE_STD VDB_DATAGEN_OPEN - = VDB_DATAGEN_CLOSED + = VDB_DATAGEN_CLOSED plus: + --vdb-index {DISKANN,HNSW,AISAQ,IVF_FLAT,IVF_SQ8,FLAT} (open widens choices) + --index-type {DISKANN,HNSW,AISAQ,IVF_FLAT,IVF_SQ8,FLAT} + --M N HNSW M parameter (default: 16) + --ef-construction N (default: 200) + --max-degree N (default: 16) + --inline-pq N (default: 16) + --search-list-size N (default: 200) + --metric-type {COSINE,L2,IP} (default: COSINE) + --compact Compact the collection after load + --monitor-interval SECS (default: 5) + --params KEY=VALUE... + OPEN_STD VDB_DATAGEN_WHATIF - = VDB_DATAGEN_OPEN (algorithm positional choices differ; flags identical) + = VDB_DATAGEN_OPEN (flags identical) ────────────────────────────────────────────────────────────────── VDB_RUN_CLOSED Required: - --results-dir/-rd PATH + --results-dir/-rd PATH (or MLPERF_RESULTS_DIR) + --systemname/-sn NAME (or MLPERF_SYSTEMNAME) [storage positional: file | object] Optional: + --vdb-engine {milvus} (default: milvus) + --vdb-index {DISKANN,HNSW,AISAQ} Index already loaded in the target collection --host/-s IP (default: 127.0.0.1) --port/-p N (default: 19530) --collection NAME @@ -400,18 +481,40 @@ --num-query-processes N (default: 1) --batch-size N (default: 1) --report-count N (default: 100) - --mode {timed,query_count,sweep} (default: timed) - --runtime N Mutually exclusive with --queries + --benchmark-mode {timed,query_count,sweep} (default: timed) + --runtime N Seconds; mutually exclusive with --queries --queries N Mutually exclusive with --runtime + --num-query-vectors N (default: 1000) + --search-limit N (default: 10) + --search-ef N (default: 200) + --recall-k N K for recall@k (defaults to --search-limit) + --gt-collection NAME Ground-truth FLAT collection + (default: _flat_gt) + --vector-dim N (default: 1536) + VDB storage location (recorded for Rules.md 5.4.1; overrides config storage.*): + --storage-root PATH + --storage-type TYPE + Distributed launch: + --distributed + --hosts HOST... (no -s short form here; -s is --host) + --npernode/--num-processes-per-client N (default: 1) + --mpi-impl {mpich,openmpi} (default: mpich) + --coordination {filesystem,mpi} (default: filesystem) + --rank-output-dir PATH (default: /tmp/mlps_vdb) + --seed N (default: 42) + --ready-timeout SECS (default: 7200) + + MPI_ARGS + + TIMESERIES + CORE_STD VDB_RUN_OPEN - = VDB_RUN_CLOSED + = VDB_RUN_CLOSED plus: + --vdb-index {DISKANN,HNSW,AISAQ,IVF_FLAT,IVF_SQ8,FLAT} (open widens choices) + --params KEY=VALUE... + OPEN_STD - + TIMESERIES VDB_RUN_WHATIF - = VDB_RUN_OPEN (algorithm positional choices differ; flags identical) + = VDB_RUN_OPEN (flags identical) Placeholder definitions — KVCACHE @@ -419,24 +522,20 @@ No object storage support at any level. KV_DATASIZE_CLOSED - (No model positional — closed runs fixed phase sequence using + (Model and cache-tier sizes fixed in closed: the phase sequence uses llama3.1-8b + llama3.1-70b-instruct automatically) - Required: - --num-users/-nu N Optional: - --cache-dir PATH - + CORE_STD (--results-dir optional) + --cache-dir PATH NVMe cache tier directory + (default: subdirectory of results) + + CORE_STD (--results-dir and --systemname optional) Note: --gpu-mem-gb=16.0 and --cpu-mem-gb=32.0 fixed; not shown KV_DATASIZE_OPEN - Required: - --num-users/-nu N - Optional: - --cache-dir PATH + = KV_DATASIZE_CLOSED plus: --gpu-mem-gb FLOAT (default: 16.0) --cpu-mem-gb FLOAT (default: 32.0) - + CORE_STD (--results-dir optional) - + OPEN_STD + Note: OPEN_STD (--loops / --allow-invalid-params) is NOT available on + kvcache datasize in any mode KV_DATASIZE_WHATIF = KV_DATASIZE_OPEN (flags identical) @@ -444,37 +543,33 @@ ────────────────────────────────────────────────────────────────── KV_RUN_CLOSED - (No model positional — fixed 3-phase sequence) + (Fixed 3-phase sequence; model pair and load parameters are pinned) Required: - --num-users/-nu N - --results-dir/-rd PATH + --results-dir/-rd PATH (or MLPERF_RESULTS_DIR) + --systemname/-sn NAME (or MLPERF_SYSTEMNAME) Optional: --cache-dir PATH --kvcache-bin-path PATH - --npernode/--num-processes-per-client N (default: 1) --exec-type/-et {mpi,docker} (default: mpi) --num-processes/-np N --hosts/-s HOST... (default: 127.0.0.1) + MPI_ARGS + + TIMESERIES + CORE_STD Note: the following are fixed in closed and not shown: - duration=60s, generation-mode=realistic, performance-profile=throughput, + gpu-mem-gb=16.0, cpu-mem-gb=32.0, duration=60s, + generation-mode=realistic, performance-profile=throughput, seed=42, trials=3, inter-option-delay=90s, disable-multi-turn=False, disable-prefix-caching=False, enable-rag=True, rag-num-docs=10, enable-autoscaling=True, autoscaler-mode=qos KV_RUN_OPEN - Required: - --num-users/-nu N - --results-dir/-rd PATH - Optional: - --cache-dir PATH - --kvcache-bin-path PATH + = KV_RUN_CLOSED plus: + --model/-m {tiny-1b,mistral-7b,llama2-7b,llama3.1-8b,llama3.1-70b-instruct} + (default: tiny-1b) + --num-users/-nu N Concurrent users to simulate (default: 100) --npernode/--num-processes-per-client N (default: 1) - --exec-type/-et {mpi,docker} (default: mpi) - --num-processes/-np N - --hosts/-s HOST... (default: 127.0.0.1) --gpu-mem-gb FLOAT (default: 16.0) --cpu-mem-gb FLOAT (default: 32.0) --duration/-d N Seconds (default: 60) @@ -485,13 +580,14 @@ --enable-rag --rag-num-docs N (default: 10) --enable-autoscaling - --autoscaler-mode {qos,predictive} (default: qos) + --autoscaler-mode {qos,capacity} (default: qos) --seed N --trials N --inter-option-delay N --config PATH - + MPI_ARGS - + TIMESERIES + --max-concurrent-allocs N Cap on concurrent in-flight cache allocations + --enable-latency-tracing bpftrace block-layer device latency tracing + (requires root) + OPEN_STD KV_RUN_WHATIF @@ -501,27 +597,19 @@ RP_REPORTGEN Required: - --results-dir/-rd PATH - Optional: - --config-file/-c PATH - --debug - --verbose + --results-dir/-rd PATH (or MLPERF_RESULTS_DIR) + + CORE_STD (every standard argument is accepted) ────────────────────────────────────────────────────────────────── -HI_LIST +HI_SHOW Optional: - --limit/-n N Show N most recent entries - --id/-i N Show specific entry by ID - --debug - --verbose + --limit/-n N Show the N most recent entries + --id/-i N Show a specific entry by ID -HI_REPLAY +HI_RERUN Required: - ID (positional) History entry ID to re-run - Optional: - --debug - --verbose + rerun_id (positional) History entry ID to re-run ────────────────────────────────────────────────────────────────── @@ -533,6 +621,7 @@ --python-version VERSION --pyproject PATH (default: pyproject.toml) --all Generate both requirements.txt and requirements-full.txt + + CORE_STD (--results-dir required — or MLPERF_RESULTS_DIR) LF_VERIFY Optional: @@ -540,6 +629,23 @@ --skip PKG Package to skip (repeatable) --allow-missing --strict + + CORE_STD (--results-dir required — or MLPERF_RESULTS_DIR) + +────────────────────────────────────────────────────────────────── + +VALIDATE + Required: + input (positional) Submission directory to check + Optional: + --submitters CSV Comma-separated submitter allowlist (default: all) + --mlperf-version VERSION Spec version (default: v3.0) + --csv PATH Summary CSV path (default: summary.csv) + --skip-output-file Suppress per-submission output file + --reference-checksum MD5 Override REFERENCE_CHECKSUMS for code/ MD5 check + +RULES_COVERAGE + Optional: + --rules-md PATH Path to Rules.md (default: project-root Rules.md) ────────────────────────────────────────────────────────────────── diff --git a/mlpstorage_py/cli_parser.py b/mlpstorage_py/cli_parser.py index 593ae580..86e23698 100755 --- a/mlpstorage_py/cli_parser.py +++ b/mlpstorage_py/cli_parser.py @@ -78,40 +78,15 @@ def _build_mode_branch(mode_parser, mode): add_kvcache_arguments(kvcache_parser, mode) -def parse_arguments(): - """Parse command-line arguments for MLPerf Storage benchmarks. +def build_parser(): + """Construct the complete mlpstorage argparse tree. + + Exposed separately from parse_arguments() so tests and tooling can walk + the real parser tree programmatically (e.g. the --help_all parity test). Returns: - argparse.Namespace: Parsed and validated arguments. + argparse.ArgumentParser: The fully assembled parser. """ - _argv = sys.argv[1:] - - # HELP-01: --help_all — print full command tree and exit - if '--help_all' in _argv: - from mlpstorage_py.cli.help_formatter import HELP_ALL_TEXT - print(HELP_ALL_TEXT) - sys.exit(0) - - # HELP-02 / HELP-03: context-sensitive help — bare, --help, AND incomplete paths - # R-03-01 fix: call get_context_help_tokens unconditionally (not gated on --help presence). - # Strip help flags first so they don't appear as positionals. Then strip all remaining - # option-style tokens (anything starting with '-') so that flags like '-cm 64' interspersed - # between positionals don't confuse the path lookup. - _help_flags = {'-h', '--help'} - _stripped = [a for a in _argv if a not in _help_flags] - _positionals = [a for a in _stripped if not a.startswith('-')] - from mlpstorage_py.cli.help_formatter import get_context_help_tokens, SYNOPSIS_TEXT - _msg = get_context_help_tokens(_positionals) - if _msg is not None: - # Fire for: bare invocation, --help at any level, AND bare incomplete paths - # (e.g., 'mlpstorage closed training' with no --help still shows "next: unet3d | retinanet") - if _help_flags.intersection(_argv): - print(SYNOPSIS_TEXT) - print() - print(_msg + ' (or -h or --help_all for details)') - sys.exit(0) - # _msg is None → leaf level OR unrecognized token → fall through to argparse (HELP-03) - parser = argparse.ArgumentParser( prog="mlpstorage", description="Script to launch the MLPerf Storage benchmark" @@ -160,6 +135,44 @@ def parse_arguments(): add_rules_coverage_arguments(rules_coverage_parser) _apply_formatter(parser) + return parser + + +def parse_arguments(): + """Parse command-line arguments for MLPerf Storage benchmarks. + + Returns: + argparse.Namespace: Parsed and validated arguments. + """ + _argv = sys.argv[1:] + + # HELP-01: --help_all — print full command tree and exit + if '--help_all' in _argv: + from mlpstorage_py.cli.help_formatter import HELP_ALL_TEXT + print(HELP_ALL_TEXT) + sys.exit(0) + + # HELP-02 / HELP-03: context-sensitive help — bare, --help, AND incomplete paths + # R-03-01 fix: call get_context_help_tokens unconditionally (not gated on --help presence). + # Strip help flags first so they don't appear as positionals. Then strip all remaining + # option-style tokens (anything starting with '-') so that flags like '-cm 64' interspersed + # between positionals don't confuse the path lookup. + _help_flags = {'-h', '--help'} + _stripped = [a for a in _argv if a not in _help_flags] + _positionals = [a for a in _stripped if not a.startswith('-')] + from mlpstorage_py.cli.help_formatter import get_context_help_tokens, SYNOPSIS_TEXT + _msg = get_context_help_tokens(_positionals) + if _msg is not None: + # Fire for: bare invocation, --help at any level, AND bare incomplete paths + # (e.g., 'mlpstorage closed training' with no --help still shows "next: unet3d | retinanet") + if _help_flags.intersection(_argv): + print(SYNOPSIS_TEXT) + print() + print(_msg + ' (or -h or --help_all for details)') + sys.exit(0) + # _msg is None → leaf level OR unrecognized token → fall through to argparse (HELP-03) + + parser = build_parser() parsed_args = parser.parse_args() diff --git a/tests/unit/test_help_all_parity.py b/tests/unit/test_help_all_parity.py new file mode 100644 index 00000000..f4370c3e --- /dev/null +++ b/tests/unit/test_help_all_parity.py @@ -0,0 +1,320 @@ +""" +HELP-04: the hand-curated --help_all reference must stay in lockstep with argparse. + +The COMPLETE COMMAND REFERENCE in mlpstorage_py/cli/help_formatter.py is +hand-maintained, not generated from the parser, so a parser-only change +silently drifts the reference (that is how --checkpoint-subset went missing, +storage#844). These tests walk the real tree from build_parser() and diff it +against the reference blocks, resolving the block notation: + + CK_RUN_OPEN + = CK_RUN_CLOSED plus: <- inherits every flag of the parent block + + MPI_ARGS <- pulls in a common argument group + +Four invariants per leaf command: + 1. every long option the parser accepts appears in the leaf's resolved block + 2. every flag the resolved block declares exists on that leaf's parser + 3. every "--long/-short" pair the block writes matches the parser's aliases + 4. choice sets written as {a,b,c} in a block's own text match the parser + +Plus one global invariant: every parser action carries a help string. +""" + +import argparse +import re + +import pytest + +from mlpstorage_py.cli.help_formatter import HELP_ALL_TEXT +from mlpstorage_py.cli_parser import build_parser + + +# ===================================================================== +# Reference-side: extract and resolve the documentation blocks +# ===================================================================== + +# Block headers sit at column 0: "TR_RUN_CLOSED", "CORE_STD — Standard ..." +_BLOCK_HEADER_RE = re.compile(r'^([A-Z][A-Z0-9_]{2,})(?:\s+—.*)?$') +_SECTION_HEADER_PREFIXES = ('Placeholder definitions', 'Common argument groups') + +# A flag *declaration* line: indented, starting with a --flag token +# (optionally --long/-short or --long/--alias). Prose mentions of flags +# ("defaults to the --hosts count") deliberately do not match. +_DECLARATION_RE = re.compile(r'^\s+(--[A-Za-z][\w-]*(?:/-{1,2}[A-Za-z][\w-]*)*)') + +_CHOICES_RE = re.compile(r'\{([^{}]+)\}') + +# Inheritance ("= PARENT plus:") and group references ("+ MPI_ARGS") +_REF_RE = re.compile(r'^\s*[=+]\s+([A-Z][A-Z0-9_]{2,})', re.M) + + +def _extract_blocks(text): + blocks = {} + current = None + for line in text.split('\n'): + m = _BLOCK_HEADER_RE.match(line) + if m: + current = m.group(1) + blocks[current] = [] + continue + if line.startswith(_SECTION_HEADER_PREFIXES): + current = None + continue + if current is not None: + blocks[current].append(line) + return {name: '\n'.join(body) for name, body in blocks.items()} + + +BLOCKS = _extract_blocks(HELP_ALL_TEXT) + + +def _resolve(name, _seen=None): + """Block text plus everything it inherits or includes, transitively.""" + if _seen is None: + _seen = set() + if name in _seen or name not in BLOCKS: + return '' + _seen.add(name) + text = BLOCKS[name] + parts = [text] + for ref in _REF_RE.findall(text): + parts.append(_resolve(ref, _seen)) + return '\n'.join(parts) + + +# ===================================================================== +# Parser-side: walk every leaf command of the real tree +# ===================================================================== + +def _walk(parser, path, out): + sub = None + options, positionals = [], [] + for action in parser._actions: + if isinstance(action, argparse._SubParsersAction): + sub = action + elif isinstance(action, (argparse._HelpAction, argparse._VersionAction)): + continue + elif action.option_strings: + options.append(action) + else: + positionals.append(action) + if sub is None: + out.append((tuple(path), options, positionals)) + else: + for name, subparser in sub.choices.items(): + _walk(subparser, path + [name], out) + + +_LEAVES = [] +_walk(build_parser(), [], _LEAVES) + +_BENCH_PREFIX = {'training': 'TR', 'checkpointing': 'CK', 'vectordb': 'VDB', 'kvcache': 'KV'} +_SPECIAL_BLOCKS = { + ('reports', 'reportgen'): 'RP_REPORTGEN', + ('history', 'show'): 'HI_SHOW', + ('history', 'rerun'): 'HI_RERUN', + ('lockfile', 'generate'): 'LF_GENERATE', + ('lockfile', 'verify'): 'LF_VERIFY', + ('init',): 'INIT', + ('version',): 'VERSION', + ('validate',): 'VALIDATE', + ('rules-coverage',): 'RULES_COVERAGE', +} + + +def _block_for(path): + if path in _SPECIAL_BLOCKS: + return _SPECIAL_BLOCKS[path] + if len(path) == 4 and path[1] == 'training': + # ('closed', 'training', '', 'run') — flags are identical across + # models; the reference documents one block per (command, mode). + mode, bench, _model, cmd = path + return f'{_BENCH_PREFIX[bench]}_{cmd.upper()}_{mode.upper()}' + if len(path) == 3 and path[0] in ('closed', 'open', 'whatif'): + mode, bench, cmd = path + return f'{_BENCH_PREFIX[bench]}_{cmd.upper()}_{mode.upper()}' + raise AssertionError(f'no --help_all block mapping for parser path {path!r}') + + +# Flags deliberately left out of the --help_all reference. Keep this empty +# unless there is a stated reason a flag must stay undocumented. +UNDOCUMENTED_OK = set() # e.g. {('closed/training/run', '--some-flag')} + +_LEAF_IDS = ['/'.join(p) for p, _, _ in _LEAVES] + + +# ===================================================================== +# 1. Every parser flag is documented in its resolved block +# ===================================================================== + +@pytest.mark.parametrize('path, options, positionals', _LEAVES, ids=_LEAF_IDS) +def test_every_parser_flag_documented(path, options, positionals): + block = _block_for(path) + text = _resolve(block) + assert text.strip(), \ + f'--help_all has no block {block} for command {"/".join(path)}' + missing = [] + for action in options: + longs = [o for o in action.option_strings if o.startswith('--')] + if not longs: + continue + if ('/'.join(path), longs[0]) in UNDOCUMENTED_OK: + continue + if not any(l in text for l in longs): + missing.append(longs[0]) + assert not missing, ( + f'{"/".join(path)}: flags accepted by the parser but absent from ' + f'--help_all block {block} (or its inherited/included blocks): {missing}' + ) + + +# ===================================================================== +# 2. Every flag the block declares exists on the leaf's parser +# ===================================================================== + +@pytest.mark.parametrize('path, options, positionals', _LEAVES, ids=_LEAF_IDS) +def test_every_documented_flag_exists(path, options, positionals): + block = _block_for(path) + text = _resolve(block) + parser_flags = set() + for action in options: + parser_flags.update(action.option_strings) + stale = [] + for line in text.split('\n'): + m = _DECLARATION_RE.match(line) + if not m: + continue + declared = m.group(1).split('/') + if not any(alias in parser_flags for alias in declared): + stale.append(m.group(1)) + assert not stale, ( + f'{"/".join(path)}: --help_all block {block} declares flags the ' + f'parser does not accept (renamed or removed?): {stale}' + ) + + +# ===================================================================== +# 3. Documented --long/-short alias pairs match the parser +# ===================================================================== + +@pytest.mark.parametrize('path, options, positionals', _LEAVES, ids=_LEAF_IDS) +def test_documented_alias_pairs_match(path, options, positionals): + block = _block_for(path) + text = _resolve(block) + by_flag = {} + for action in options: + for opt in action.option_strings: + by_flag[opt] = set(action.option_strings) + mismatched = [] + for line in text.split('\n'): + m = _DECLARATION_RE.match(line) + if not m: + continue + declared = m.group(1).split('/') + if len(declared) < 2: + continue + anchor = next((a for a in declared if a in by_flag), None) + if anchor is None: + continue # stale flag — test 2 reports it + wrong = [a for a in declared if a not in by_flag[anchor]] + if wrong: + mismatched.append((m.group(1), sorted(by_flag[anchor]))) + assert not mismatched, ( + f'{"/".join(path)}: --help_all block {block} writes alias pairs that ' + f'do not match the parser (documented, actual): {mismatched}' + ) + + +# ===================================================================== +# 4. Choice sets written as {a,b,c} in a block's OWN text match the parser +# ===================================================================== + +@pytest.mark.parametrize('path, options, positionals', _LEAVES, ids=_LEAF_IDS) +def test_documented_choices_match(path, options, positionals): + block = _block_for(path) + text = BLOCKS.get(block, '') # own text only — inherited blocks may + # legitimately show a different mode's choice set + by_flag = {} + for action in options: + for opt in action.option_strings: + by_flag[opt] = action + wrong = [] + for line in text.split('\n'): + m = _DECLARATION_RE.match(line) + if not m: + continue + anchor = next((a for a in m.group(1).split('/') if a in by_flag), None) + if anchor is None: + continue + cm = _CHOICES_RE.search(line) + if not cm: + continue + documented = {c.strip() for c in cm.group(1).split(',')} + actual = by_flag[anchor].choices + if actual is None: + wrong.append((anchor, sorted(documented), None)) + elif documented != {str(c) for c in actual}: + wrong.append((anchor, sorted(documented), sorted(str(c) for c in actual))) + assert not wrong, ( + f'{"/".join(path)}: --help_all block {block} documents choice sets ' + f'that do not match the parser (flag, documented, actual): {wrong}' + ) + + +# ===================================================================== +# 5. Storage positional and user positionals are documented +# ===================================================================== + +@pytest.mark.parametrize('path, options, positionals', _LEAVES, ids=_LEAF_IDS) +def test_positionals_documented(path, options, positionals): + block = _block_for(path) + text = _resolve(block) + for action in positionals: + if action.dest == 'data_access_protocol': + assert 'file | object' in text, ( + f'{"/".join(path)}: takes the file|object storage positional ' + f'but block {block} never mentions "file | object"' + ) + else: + assert action.dest in text, ( + f'{"/".join(path)}: positional {action.dest!r} missing from ' + f'block {block}' + ) + + +# ===================================================================== +# 6. Every parser action carries a help string (argparse --help surface) +# ===================================================================== + +def test_every_action_has_help_string(): + bare = [] + for path, options, positionals in _LEAVES: + for action in options + positionals: + if not (action.help or '').strip(): + bare.append(('/'.join(path), action.option_strings or action.dest)) + # Collapse duplicates (universal args repeat across every leaf) + unique = sorted({(p.split('/')[-1], str(f)) for p, f in bare}) + assert not bare, ( + f'parser actions with no help= string (shown as (command, flag), ' + f'deduplicated): {unique}' + ) + + +# ===================================================================== +# 7. The command tree in --help_all reflects the real shape +# ===================================================================== + +def test_tree_shows_training_model_choices(): + """Training is the only benchmark with a model positional; the tree rows + must carry the real per-mode choice sets.""" + assert 'unet3d | retinanet' in HELP_ALL_TEXT + assert 'cosmoflow | resnet50 | unet3d | dlrm | retinanet | flux' in HELP_ALL_TEXT + + +def test_tree_history_subcommands_are_show_and_rerun(): + """The history subcommands are `show` and `rerun` — the reference must not + keep documenting the old `list`/`replay` names.""" + assert 'HI_SHOW' in HELP_ALL_TEXT + assert 'HI_RERUN' in HELP_ALL_TEXT + assert 'HI_LIST' not in HELP_ALL_TEXT + assert 'HI_REPLAY' not in HELP_ALL_TEXT