diff --git a/Makefile b/Makefile index 56fcacae2..5904512a3 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ -MY_ISV_DOMAINS := bare_metal control-plane iam image-registry network observability security storage vm -DEMO_TARGETS := $(addprefix demo-,$(MY_ISV_DOMAINS)) +MY_ISV_SUITES := bare_metal control-plane iam image-registry network observability security storage vm +DEMO_TARGETS := $(addprefix demo-,$(MY_ISV_SUITES)) .PHONY: help pre-commit build test coverage clean lint format install bump-patch bump-fix bump-minor bump-feat bump-major bump bump-check \ security-trivy security-trivy-detail security-trufflehog ci-security demo-test demo-all $(DEMO_TARGETS) plan plan-coverage validate-suites \ @@ -84,29 +84,46 @@ test: ## Run tests for all packages @echo "" @echo "✅ All tests passed!" +# run_demo,[,] - a plain suite with no capability runs only +# its core checks, so suites holding gated checks name one to exercise those +# stubs. Unlisted suites are core-only or platform suites. +DEMO_CAP_network := vm +DEMO_CAP_observability := vm +DEMO_CAP_security := vm +DEMO_CAP_storage := vm + define run_demo @echo "" @echo "==========================================" - @echo "Demo test: $(1)" + @echo "Demo test: $(1)$(if $(2), --capability $(2),)" @echo "==========================================" - @echo "Running cmd: ISVCTL_DEMO_MODE=1 uv run isvctl test run -f isvctl/configs/providers/my-isv/config/$(1).yaml" + @echo "Running cmd: ISVCTL_DEMO_MODE=1 uv run isvctl test run -f isvctl/configs/providers/my-isv/config/$(1).yaml$(if $(2), --capability $(2),)" @ISVCTL_DEMO_MODE=1 uv run isvctl test run \ - -f isvctl/configs/providers/my-isv/config/$(1).yaml + -f isvctl/configs/providers/my-isv/config/$(1).yaml \ + $(if $(2),--capability $(2),) endef demo-test: demo-all ## Alias for demo-all (backward compat) -demo-all: ## Run all my-isv living examples (or demo- for one, e.g. demo-security) +demo-all: ## Run all my-isv living examples (or demo- for one, e.g. demo-security) @echo "Running my-isv living examples in demo mode..." - @for domain in $(MY_ISV_DOMAINS); do \ - $(MAKE) --no-print-directory demo-$$domain || exit 1; \ + @for suite in $(MY_ISV_SUITES); do \ + $(MAKE) --no-print-directory demo-$$suite || exit 1; \ done @echo "" @echo "✅ All my-isv living examples passed in demo mode!" - @echo "Domains: $(MY_ISV_DOMAINS)" - -$(DEMO_TARGETS): demo-%: - $(call run_demo,$*) + @echo "Suites: $(MY_ISV_SUITES)" + +# image-registry is excluded here because it has its own rule below; leaving it +# in would make every make invocation warn about an overridden recipe. +$(filter-out demo-image-registry,$(DEMO_TARGETS)): demo-%: + $(call run_demo,$*,$(DEMO_CAP_$*)) + +# image-registry splits its gated checks between vm and bare_metal, so one run +# cannot reach both. +demo-image-registry: + $(call run_demo,image-registry,vm) + $(call run_demo,image-registry,bare_metal) coverage: ## Run tests with coverage and generate combined report @echo "Running tests with coverage..." diff --git a/docs/guides/configuration.md b/docs/guides/configuration.md index 5c0f9bc2a..2912e150f 100644 --- a/docs/guides/configuration.md +++ b/docs/guides/configuration.md @@ -124,7 +124,6 @@ commands: timeout: 300 tests: - platform: network cluster_name: "aws-network-test" settings: @@ -195,6 +194,28 @@ Each step defines a command to execute: | `skip` | No | Skip this step | | `continue_on_failure` | No | Continue even if this step fails | | `output_schema` | No | Schema name for output validation | +| `requires` | No | Capability contexts this step runs in (see [Capabilities](#capabilities-and-requires)) | + +#### Gating a step with `requires` + +A step is skipped automatically when **every** validation bound to it is +filtered out by the run's capability. That inference covers most steps, but it +cannot reach a step no validation binds to — typically a teardown step: + +```yaml +# Runs only under --capability kubernetes. Without the gate, a core run would +# try to tear down a cluster it never created. +- name: teardown_cluster + phase: teardown + command: "./scripts/teardown_cluster.sh" + requires: [kubernetes] +``` + +Rule of thumb: **if a step builds or destroys a fixture that only some contexts +need, give it an explicit `requires:`** — and give both halves of the fixture the +same one, so setup and teardown always move together. A step that survives the +gate must not reference a gated-off step's output; use `default(...)` if it +legitimately might be absent. ### Validation Configuration @@ -267,6 +288,64 @@ The part before the dash must match an existing validation class name (e.g., `K8 - The suffix is free-form: `K8sNimHelmWorkload-small`, `SlurmPartition-cpu`, `SlurmGpuAllocation-1gpu` are all valid. - Each variant is a distinct test entry in coverage tracking. +## Capabilities and `requires` + +An ISV declares which **capabilities** it supports. There are exactly four, and +they are mutually exclusive execution environments — you run on one at a time, +never a combination: + +`vm` · `bare_metal` · `kubernetes` · `slurm` + +Each has a **platform suite** (`suites/vm.yaml`, `suites/k8s.yaml`, ...) carrying +the checks you owe by declaring it. Its `tests.platform:` key names the +capability, and its checks declare no `requires:` — they all run. + +Everything else is a **plain suite** (`storage`, `network`, `iam`, ...), named by +its filename. A plain suite mixes checks that need no particular infrastructure +with checks that presuppose some. Each check says which: + +```yaml +requires: [] # core - runs in every context +requires: [kubernetes] # runs only under --capability kubernetes +requires: [vm, bare_metal] # any-match: either context satisfies it +``` + +`requires` is **any-match**, not a set to satisfy simultaneously: a check runs +when its list is empty, or when the run's capability appears in it. There is +deliberately no way to express "needs vm AND kubernetes" — no check needs it, +and the mutual exclusivity above means such a check could never run. + +### What runs, and when + +One rule, and it does not depend on how you named the config — `--suite`, `-f`, +and `--label` discovery behave identically: + +> **A plain suite with no `--capability` runs its core checks.** Name a +> capability to add the checks gated on it. + +```bash +isvctl test run --provider acme --suite storage # core only +isvctl test run --provider acme --suite storage --capability kubernetes # core + k8s checks +isvctl test run --provider acme --suite kubernetes # the platform suite +``` + +There is no "run everything" context: a plain suite always carries exactly one. +Passing a capability no check in the suite requires is allowed but warns, since +a flag that silently does nothing is usually a typo. + +Two consequences worth internalising: + +- **Nothing is mandatory.** A check is in scope only if you declared the suite + that contains it. 100% is always relative to what you declared, so declaring + a subset legitimately yields zero checks from the suites you left out. +- **A capability and a plain suite compose.** The 15 CSI checks in `storage` + need `storage` *and* `kubernetes`. Declaring `kubernetes` alone runs the + Kubernetes platform suite but no storage CSI checks. + +Capability names and plain-suite names share one namespace, so a plain suite may +not be named after a capability. `catalog_document` and +`scripts/validate_suite_wiring.py` both reject the collision. + ## Import and Override Provider configs can import a canonical test suite and override command definitions while inheriting validations (unless explicitly overridden): diff --git a/docs/guides/external-validation-guide.md b/docs/guides/external-validation-guide.md index 6d5dd3045..ff2726d28 100644 --- a/docs/guides/external-validation-guide.md +++ b/docs/guides/external-validation-guide.md @@ -126,7 +126,6 @@ commands: timeout: 300 tests: - platform: myplatform cluster_name: "my-validation" settings: @@ -186,9 +185,64 @@ For validation timing and phase control, see the [Configuration Guide](configura --- +## Capabilities and check requirements + +You declare which **capabilities** you support. There are four, and they are +mutually exclusive execution environments — you run on one at a time: + +`vm` · `bare_metal` · `kubernetes` · `slurm` + +Each has a **platform suite** you owe by declaring it (`vm`, `bare_metal`, +`kubernetes`, `slurm`). Everything else is a **plain suite** (`storage`, +`network`, `iam`, ...) that mixes checks needing no particular infrastructure +with checks that presuppose some. Each check declares which: + +```yaml +requires: [] # core - runs in every context +requires: [kubernetes] # runs only under --capability kubernetes +requires: [vm, bare_metal] # any-match: either context satisfies it +``` + +**Nothing is mandatory.** A check is in scope only if you declared the suite +containing it, so 100% is always relative to what you declared. Declaring a +subset legitimately yields zero checks from the suites you left out — that is +the design, not a gap. + +One rule decides what runs, and it does not depend on how you named the config +(`--suite`, `-f` and `--label` behave identically): + +> **A plain suite with no `--capability` runs its core checks.** Name a +> capability to add the checks gated on it. + +If a step builds or destroys a fixture only some contexts need, gate it the same +way so a core run neither provisions nor leaks it: + +```yaml +- name: teardown_cluster + phase: teardown + command: "./scripts/teardown_cluster.sh" + requires: [kubernetes] +``` + +See the [Configuration Guide](configuration.md#capabilities-and-requires) for the +full model. + +--- + ## Running Validations ```bash +# One suite for your provider - the form the UI emits +isvctl test run --provider acme --suite storage # core checks +isvctl test run --provider acme --suite storage --capability kubernetes # + k8s checks +isvctl test run --provider acme --suite kubernetes # a platform suite + +# See what would run, without executing anything +isvctl test run --provider acme --suite storage --capability vm --dry-run + +# Point at a config file directly (same capability rule applies) +isvctl test run -f config.yaml + # Run all phases isvctl test run -f config.yaml @@ -209,10 +263,20 @@ isvctl test run -f config.yaml -- -k "ConnectivityCheck" isvctl test run -f config.yaml --label gpu isvctl test run -f config.yaml -- -m "not slow" +# Labels compose with suite selection +isvctl test run --provider acme --suite storage --capability vm --label min_req + +# Re-run one failed check in its lifecycle context (pytest passthrough) +isvctl test run --provider acme --suite storage --capability kubernetes -- -k K8sCsiPvcExpandCheck + # Debug: full output on failure isvctl test run -f config.yaml -v -- -s --tb=long ``` +Re-running a single failed check is pytest passthrough after `--`; there are no +dedicated rerun flags. Setup steps re-run, which is the deliberate trade for not +maintaining a dependency graph. + > **Teardown behavior:** By default, teardown runs even when setup or test validations fail, ensuring cloud resources are cleaned up. Individual teardown step failures don't block remaining teardown steps (best-effort execution). --- @@ -266,9 +330,13 @@ Preview the whole pipeline with no cloud: ```bash make demo-test # sets ISVCTL_DEMO_MODE=1 and runs all my-isv configs (~10s) -# Domains are listed in the Makefile MY_ISV_DOMAINS variable. +# Suites are listed in the Makefile MY_ISV_SUITES variable; DEMO_CAP_ +# names the capability a suite runs under so its gated checks are exercised. ``` +The k8s and Slurm examples are excluded: they drive a real cluster, so a +dummy-success stub has nothing to return for them. + --- ## Related Documentation diff --git a/docs/guides/local-development.md b/docs/guides/local-development.md index ef6d80ef6..c81d9ef17 100644 --- a/docs/guides/local-development.md +++ b/docs/guides/local-development.md @@ -149,7 +149,6 @@ commands: timeout: 60 tests: - platform: network cluster_name: "local-test" settings: region: "us-west-2" diff --git a/docs/test-plan.yaml b/docs/test-plan.yaml index 5913c35d4..78f4b6aa8 100644 --- a/docs/test-plan.yaml +++ b/docs/test-plan.yaml @@ -3243,6 +3243,7 @@ domains: labels: - kubernetes - min_req + - storage priority: P0 milestone: M5 notes: "" @@ -3255,6 +3256,7 @@ domains: labels: - kubernetes - min_req + - storage priority: P0 milestone: M5 notes: "" @@ -3267,6 +3269,7 @@ domains: labels: - kubernetes - min_req + - storage priority: P0 milestone: M5 notes: "" @@ -3279,6 +3282,7 @@ domains: labels: - kubernetes - min_req + - storage priority: P0 milestone: M5 notes: "" diff --git a/isvctl/configs/providers/aws/config/image-registry.yaml b/isvctl/configs/providers/aws/config/image-registry.yaml index b6b5dcf9a..1092d4e35 100644 --- a/isvctl/configs/providers/aws/config/image-registry.yaml +++ b/isvctl/configs/providers/aws/config/image-registry.yaml @@ -102,18 +102,18 @@ commands: timeout: 120 # Step 5: Cleanup all resources - - name: teardown + # Teardown is split by owner so each half can be gated independently. + # This half cleans up what launch_instance created, so it carries the + # same `requires: [vm]` gate and is skipped whole in other contexts. + # Declared first: terminate the instance before deregistering the AMI it + # was launched from. + - name: teardown_instance phase: teardown command: "python3 ../scripts/image-registry/teardown.py" + requires: [vm] args: - "--instance-id" - "{{steps.launch_instance.instance_id}}" - - "--ami-id" - - "{{steps.upload_image.image_id}}" - - "--snapshot-ids" - - "{{steps.upload_image.disk_ids | join(',')}}" - - "--bucket-name" - - "{{steps.upload_image.storage_bucket}}" - "--key-name" - "{{steps.launch_instance.key_name}}" - "--security-group-id" @@ -125,6 +125,23 @@ commands: - "{{teardown_flag}}" timeout: 1800 + # This half cleans up what the core upload_image step created, so it is + # ungated and runs in every capability context. + - name: teardown_image + phase: teardown + command: "python3 ../scripts/image-registry/teardown.py" + args: + - "--ami-id" + - "{{steps.upload_image.image_id}}" + - "--snapshot-ids" + - "{{steps.upload_image.disk_ids | join(',')}}" + - "--bucket-name" + - "{{steps.upload_image.storage_bucket}}" + - "--region" + - "{{region}}" + - "{{teardown_flag}}" + timeout: 1800 + tests: cluster_name: "aws-image-registry-validation" description: "AWS VM Import ISO/VMDK validation tests" diff --git a/isvctl/configs/providers/aws/config/observability.yaml b/isvctl/configs/providers/aws/config/observability.yaml index 7aeda1534..2ab9ae9f8 100644 --- a/isvctl/configs/providers/aws/config/observability.yaml +++ b/isvctl/configs/providers/aws/config/observability.yaml @@ -61,9 +61,13 @@ commands: - "isv-observability" timeout: 300 + # The host fixture only serves the vm/bare_metal telemetry checks; no + # bound validation can infer that (nothing validates the launch itself), + # so the gate is explicit. Core runs skip the metal instance entirely. - name: launch_host phase: setup command: "python3 ../scripts/bare_metal/launch_instance.py" + requires: [vm, bare_metal] args: - "--name" - "isv-observability-host" @@ -340,9 +344,12 @@ commands: - "switch_kernel_logs" timeout: 60 + # Same gate as launch_host: both sides of the fixture move together, so + # a core run never tears down a host it never launched. - name: teardown_host phase: teardown command: "python3 ../scripts/bare_metal/teardown.py" + requires: [vm, bare_metal] args: - "--instance-id" - "{{steps.launch_host.instance_id}}" @@ -382,7 +389,6 @@ commands: timeout: 900 tests: - platform: observability cluster_name: "aws-observability-validation" description: "AWS observability validation for VPC Flow Logs and host syslogs" diff --git a/isvctl/configs/providers/aws/config/security.yaml b/isvctl/configs/providers/aws/config/security.yaml index 901fd5057..c773d42c7 100644 --- a/isvctl/configs/providers/aws/config/security.yaml +++ b/isvctl/configs/providers/aws/config/security.yaml @@ -334,9 +334,13 @@ commands: # its finally block; this teardown step reclaims resources left behind # when AWS_CAPACITY_SKIP_DESTROY=true (a no-op while that flag is set, so # a later standalone `--phase teardown` without it does the cleanup). + # Mirrors the gate inferred on topology_block_atomic_allocation (its only + # check is vm/bare_metal): a core run allocates no block, so it must not + # try to release one. - name: topology_block_teardown phase: teardown command: "python3 ../scripts/capacity/topology_block_atomic_allocation.py" + requires: [vm, bare_metal] requires_available_validations: - CapacityTopologyBlockAtomicAllocationCheck args: @@ -365,6 +369,7 @@ commands: - name: capacity_teardown phase: teardown command: "python3 ../scripts/capacity/reservation_grouping.py" + requires: [vm, bare_metal] requires_available_validations: - CapacityReservationGroupingCheck args: diff --git a/isvctl/configs/providers/aws/config/storage.yaml b/isvctl/configs/providers/aws/config/storage.yaml index 80e549324..7cfab39df 100644 --- a/isvctl/configs/providers/aws/config/storage.yaml +++ b/isvctl/configs/providers/aws/config/storage.yaml @@ -16,6 +16,7 @@ # AWS Storage Validation Configuration # # Imports suites/storage.yaml and wires AWS implementations for: +# - Kubernetes CSI (EKS): setup_cluster / teardown_cluster via eks scripts # - Block volume (EBS): launch/create/snapshot/resize/persistence/teardown # - Home directory (FSx for OpenZFS): NFSv4, volume quota, uid/gid accounting # - High-speed storage (FSx for Lustre): provisioning, multiple filesystems, @@ -25,6 +26,9 @@ # Block-volume steps reuse the VM scripts for launch/teardown; volume steps # live in scripts/storage/. FSx steps are self-contained (create → assert → # cleanup) and create real PERSISTENT_2 filesystems (~10-15 min each, cost). +# The EKS fixture reuses Terraform state when present but will provision a +# cluster when none exists; teardown_cluster always runs so that path cannot +# leak. Set AWS_SKIP_TEARDOWN=true to preserve the cluster across runs. # HSS checks ship unreleased; run with ISVTEST_INCLUDE_UNRELEASED=1. # # Usage: @@ -51,6 +55,24 @@ commands: storage: phases: ["setup", "test", "teardown"] steps: + # Kubernetes CSI fixture: provisions (or reuses) the shared EKS Terraform + # stack and emits kubeconfig + CSI StorageClass names. Paired with + # teardown_cluster below so a standalone storage run does not leak the + # cluster. Honors AWS_SKIP_TEARDOWN like eks.yaml. + - name: setup_cluster + phase: setup + command: "../scripts/eks/setup.sh" + requires: [kubernetes] + # Same contract as my-isv's stub: kubeconfig + StorageClass names. The + # EKS script happens to emit full cluster inventory too, but storage + # must not require that of an ISV, so both providers validate the + # fixture against its own contract, not the platform `cluster` schema. + output_schema: generic + timeout: 5400 + env: + TF_AUTO_APPROVE: "true" + TF_VAR_region: "{{region}}" + # ─── Block volume (EBS) ──────────────────────────────────────── - name: launch_instance phase: setup @@ -191,6 +213,7 @@ commands: - name: teardown_volume phase: teardown command: "python3 ../scripts/storage/teardown_volume.py" + requires: [vm, bare_metal] args: - "--region" - "{{region}}" @@ -202,6 +225,7 @@ commands: - name: teardown phase: teardown command: "python3 ../scripts/vm/teardown.py" + requires: [vm, bare_metal] args: - "--instance-id" - "{{steps.launch_instance.instance_id}}" @@ -212,6 +236,17 @@ commands: - "{{teardown_flag}}" timeout: 600 + # Destroy the EKS stack from setup_cluster (including the out-of-band + # static-CSI EBS volume tagged by setup.sh). Keep this last so CSI + # checks finish before the cluster disappears. + - name: teardown_cluster + phase: teardown + command: "../scripts/eks/teardown.sh" + requires: [kubernetes] + timeout: 1800 + env: + TF_AUTO_APPROVE: "true" + tests: cluster_name: "aws-storage-validation" description: "AWS storage validation (EBS block volume + FSx for Lustre)" diff --git a/isvctl/configs/providers/aws/scripts/control-plane/docs/aws-control-plane.md b/isvctl/configs/providers/aws/scripts/control-plane/docs/aws-control-plane.md index e1323eab7..ca8378039 100644 --- a/isvctl/configs/providers/aws/scripts/control-plane/docs/aws-control-plane.md +++ b/isvctl/configs/providers/aws/scripts/control-plane/docs/aws-control-plane.md @@ -247,7 +247,6 @@ commands: # ... more steps ... tests: - platform: control_plane settings: region: "us-west-2" services: "ec2,s3,iam,sts" diff --git a/isvctl/configs/providers/my-isv/config/image-registry.yaml b/isvctl/configs/providers/my-isv/config/image-registry.yaml index 98594a3b9..371000696 100644 --- a/isvctl/configs/providers/my-isv/config/image-registry.yaml +++ b/isvctl/configs/providers/my-isv/config/image-registry.yaml @@ -111,18 +111,18 @@ commands: - "{{region}}" timeout: 60 - - name: teardown + # Teardown is split by owner so each half can be gated independently. + # This half cleans up what launch_instance created, so it carries the + # same `requires: [vm]` gate and is skipped whole in other contexts. + # Declared first: terminate the instance before deregistering the image + # it was launched from. + - name: teardown_instance phase: teardown command: "python ../scripts/image-registry/teardown.py" + requires: [vm] args: - "--instance-id" - "{{steps.launch_instance.instance_id}}" - - "--image-id" - - "{{steps.upload_image.image_id}}" - - "--disk-ids" - - "{{steps.upload_image.disk_ids | join(',')}}" - - "--bucket-name" - - "{{steps.upload_image.storage_bucket}}" - "--key-name" - "{{steps.launch_instance.key_name}}" - "--security-group-id" @@ -134,6 +134,23 @@ commands: - "{{teardown_flag}}" timeout: 60 + # This half cleans up what the core upload_image step created, so it is + # ungated and runs in every capability context. + - name: teardown_image + phase: teardown + command: "python ../scripts/image-registry/teardown.py" + args: + - "--image-id" + - "{{steps.upload_image.image_id}}" + - "--disk-ids" + - "{{steps.upload_image.disk_ids | join(',')}}" + - "--bucket-name" + - "{{steps.upload_image.storage_bucket}}" + - "--region" + - "{{region}}" + - "{{teardown_flag}}" + timeout: 60 + tests: cluster_name: "my-isv-image-registry-validation" description: "my-isv image registry lifecycle validation (generic stubs, dummy overrides)" diff --git a/isvctl/configs/providers/my-isv/config/k8s.yaml b/isvctl/configs/providers/my-isv/config/k8s.yaml new file mode 100644 index 000000000..d1edf638d --- /dev/null +++ b/isvctl/configs/providers/my-isv/config/k8s.yaml @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# my-isv Kubernetes Platform Suite - Living Example +# +# This is the obligation attached to declaring the `kubernetes` capability. +# It imports the provider-agnostic Kubernetes validation contract and wires +# the lifecycle to the generic scripts in scripts/k8s/*. +# +# Usage: +# uv run isvctl test run --provider my-isv --suite kubernetes +# +# Unlike the other my-isv examples this one talks to a REAL cluster: the +# validations run kubectl directly, so there is nothing for a dummy-success +# stub to return. That is why it is not part of `make demo-test`. +# +# ACTION: point KUBECTL at your cluster (or make `kubectl` resolve to it), +# then replace scripts/k8s/setup.sh with your own provisioning if you want +# isvctl to create the cluster rather than reuse an existing one. +# +# Environment Variables: +# - KUBECTL: kubectl-compatible CLI prefix (defaults to kubectl, then microk8s kubectl) +# - NGC_API_KEY: NGC API key for NIM workloads + +import: ../../../suites/k8s.yaml + +version: "1.0" + +# Steps are overridden (not inherited) so the script paths resolve from this +# provider's config directory. Lists are replaced wholesale on merge. +commands: + kubernetes: + phases: ["setup", "test", "teardown"] + steps: + - name: setup + phase: setup + command: "../scripts/k8s/setup.sh" + timeout: 120 + + - name: teardown + phase: teardown + command: "../scripts/k8s/teardown.sh" + timeout: 30 + +tests: + description: "my-isv Kubernetes platform suite (real cluster required)" diff --git a/isvctl/configs/providers/my-isv/config/observability.yaml b/isvctl/configs/providers/my-isv/config/observability.yaml index 50a8bf095..f5325c036 100644 --- a/isvctl/configs/providers/my-isv/config/observability.yaml +++ b/isvctl/configs/providers/my-isv/config/observability.yaml @@ -255,7 +255,6 @@ commands: timeout: 60 tests: - platform: observability cluster_name: "my-isv-observability-validation" description: "my-isv observability validation (generic stubs, dummy overrides)" diff --git a/isvctl/configs/providers/my-isv/config/slurm.yaml b/isvctl/configs/providers/my-isv/config/slurm.yaml new file mode 100644 index 000000000..d65cc5e62 --- /dev/null +++ b/isvctl/configs/providers/my-isv/config/slurm.yaml @@ -0,0 +1,55 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# my-isv Slurm Platform Suite - Living Example +# +# This is the obligation attached to declaring the `slurm` capability. It +# imports the provider-agnostic Slurm validation contract and wires the +# lifecycle to the generic scripts in scripts/slurm/*. +# +# Usage: +# uv run isvctl test run --provider my-isv --suite slurm +# +# Unlike the other my-isv examples this one talks to a REAL cluster: the +# validations run sinfo/srun directly, so there is nothing for a dummy-success +# stub to return. That is why it is not part of `make demo-test`. +# +# ACTION: make the Slurm client commands (sinfo, scontrol, srun) resolve to +# your cluster, then replace scripts/slurm/setup.sh with your own +# provisioning if you want isvctl to create the cluster rather than reuse an +# existing one. + +import: ../../../suites/slurm.yaml + +version: "1.0" + +# Steps are overridden (not inherited) so the script paths resolve from this +# provider's config directory. Lists are replaced wholesale on merge. +commands: + slurm: + phases: ["setup", "test", "teardown"] + steps: + - name: setup + phase: setup + command: "../scripts/slurm/setup.sh" + timeout: 120 + + - name: teardown + phase: teardown + command: "../scripts/slurm/teardown.sh" + timeout: 30 + +tests: + description: "my-isv Slurm platform suite (real cluster required)" diff --git a/isvctl/configs/providers/my-isv/config/storage.yaml b/isvctl/configs/providers/my-isv/config/storage.yaml index e311a70c9..c5cdbb9a3 100644 --- a/isvctl/configs/providers/my-isv/config/storage.yaml +++ b/isvctl/configs/providers/my-isv/config/storage.yaml @@ -49,6 +49,21 @@ commands: storage: phases: ["setup", "test", "teardown"] steps: + # Only the kubernetes CSI checks need a cluster, so both sides of the + # fixture carry the same gate: a `--capability vm` run never provisions + # one. Acquire is idempotent - reuse an existing cluster if you have one. + - name: setup_cluster + phase: setup + command: "python ../scripts/storage/setup_cluster.py" + requires: [kubernetes] + # The step name auto-detects the platform-suite `cluster` schema, which + # demands cluster_name/node_count. This fixture has its own, smaller + # contract - kubeconfig + StorageClass names, the only fields the + # k8s_storage checks read - so opt out rather than make every ISV + # report inventory no check consumes. + output_schema: generic + timeout: 60 + - name: launch_instance phase: setup command: "python ../scripts/vm/launch_instance.py" @@ -235,6 +250,7 @@ commands: - name: teardown_volume phase: teardown command: "python ../scripts/storage/teardown_volume.py" + requires: [vm, bare_metal] args: - "--region" - "{{region}}" @@ -246,6 +262,7 @@ commands: - name: teardown phase: teardown command: "python ../scripts/vm/teardown.py" + requires: [vm, bare_metal] args: - "--instance-id" - "{{steps.launch_instance.instance_id}}" @@ -255,6 +272,20 @@ commands: - "--delete-security-group" timeout: 60 + # The other half of the setup_cluster fixture. Teardown may be a no-op if + # you reuse a long-lived cluster, but the step must exist so a standalone + # storage run does not leak a cluster it provisioned. STORAGE_SKIP_TEARDOWN + # keeps it alive for cheap reruns (mirrors AWS_SKIP_TEARDOWN in aws/). + - name: teardown_cluster + phase: teardown + command: "python ../scripts/storage/teardown_cluster.py" + requires: [kubernetes] + args: + - "--kubeconfig" + - "{{steps.setup_cluster.kubeconfig_path}}" + - "{{cluster_teardown_flag}}" + timeout: 60 + tests: cluster_name: "my-isv-storage-validation" description: "my-isv storage validation (block volume + high-speed stubs, dummy overrides)" @@ -264,3 +295,4 @@ tests: instance_type: "my-isv.standard.1x" volume_size_gib: "10" teardown_flag: "{{(env.BLOCK_STORAGE_SKIP_TEARDOWN == 'true') | ternary('--skip-destroy', '')}}" + cluster_teardown_flag: "{{(env.STORAGE_SKIP_TEARDOWN == 'true') | ternary('--skip-destroy', '')}}" diff --git a/isvctl/configs/providers/my-isv/scripts/README.md b/isvctl/configs/providers/my-isv/scripts/README.md index 3d7b89dba..d15e34e57 100644 --- a/isvctl/configs/providers/my-isv/scripts/README.md +++ b/isvctl/configs/providers/my-isv/scripts/README.md @@ -28,16 +28,20 @@ template, then fill in the TODOs. | Domain | Scripts | Contract | Provider YAML | AWS reference | |--------|---------|----------|---------------|---------------| | `iam/` | 3 | [`suites/iam.yaml`](../../../suites/iam.yaml) | [`config/iam.yaml`](../config/iam.yaml) | [`providers/aws/scripts/iam/`](../../aws/scripts/iam/) | -| `control-plane/` | 10 | [`suites/control-plane.yaml`](../../../suites/control-plane.yaml) | [`config/control-plane.yaml`](../config/control-plane.yaml) | [`providers/aws/scripts/control-plane/`](../../aws/scripts/control-plane/) | -| `vm/` | 10 | [`suites/vm.yaml`](../../../suites/vm.yaml) | [`config/vm.yaml`](../config/vm.yaml) | [`providers/aws/scripts/vm/`](../../aws/scripts/vm/) | -| `bare_metal/` | 12 | [`suites/bare_metal.yaml`](../../../suites/bare_metal.yaml) | [`config/bare_metal.yaml`](../config/bare_metal.yaml) | [`providers/aws/scripts/bare_metal/`](../../aws/scripts/bare_metal/) | -| `storage/` | 17 | [`suites/storage.yaml`](../../../suites/storage.yaml) | [`config/storage.yaml`](../config/storage.yaml) | [`providers/aws/scripts/storage/`](../../aws/scripts/storage/) | -| `network/` | 18 | [`suites/network.yaml`](../../../suites/network.yaml) | [`config/network.yaml`](../config/network.yaml) | [`providers/aws/scripts/network/`](../../aws/scripts/network/) | -| `observability/` | 1 | [`suites/observability.yaml`](../../../suites/observability.yaml) | [`config/observability.yaml`](../config/observability.yaml) | [`providers/aws/scripts/observability/`](../../aws/scripts/observability/) | +| `control-plane/` | 11 | [`suites/control-plane.yaml`](../../../suites/control-plane.yaml) | [`config/control-plane.yaml`](../config/control-plane.yaml) | [`providers/aws/scripts/control-plane/`](../../aws/scripts/control-plane/) | +| `vm/` | 12 | [`suites/vm.yaml`](../../../suites/vm.yaml) | [`config/vm.yaml`](../config/vm.yaml) | [`providers/aws/scripts/vm/`](../../aws/scripts/vm/) | +| `bare_metal/` | 14 | [`suites/bare_metal.yaml`](../../../suites/bare_metal.yaml) | [`config/bare_metal.yaml`](../config/bare_metal.yaml) | [`providers/aws/scripts/bare_metal/`](../../aws/scripts/bare_metal/) | +| `storage/` | 20 | [`suites/storage.yaml`](../../../suites/storage.yaml) | [`config/storage.yaml`](../config/storage.yaml) | [`providers/aws/scripts/storage/`](../../aws/scripts/storage/) | +| `network/` | 24 | [`suites/network.yaml`](../../../suites/network.yaml) | [`config/network.yaml`](../config/network.yaml) | [`providers/aws/scripts/network/`](../../aws/scripts/network/) | +| `observability/` | 5 | [`suites/observability.yaml`](../../../suites/observability.yaml) | [`config/observability.yaml`](../config/observability.yaml) | [`providers/aws/scripts/observability/`](../../aws/scripts/observability/) | | `image-registry/` | 7 | [`suites/image-registry.yaml`](../../../suites/image-registry.yaml) | [`config/image-registry.yaml`](../config/image-registry.yaml) | [`providers/aws/scripts/image-registry/`](../../aws/scripts/image-registry/) | -| `security/` | 19 | [`suites/security.yaml`](../../../suites/security.yaml) | [`config/security.yaml`](../config/security.yaml) | [`providers/aws/scripts/security/`](../../aws/scripts/security/), [`providers/aws/scripts/capacity/`](../../aws/scripts/capacity/) | -| `k8s/` | 9 shell | [`suites/k8s.yaml`](../../../suites/k8s.yaml) | - | [`providers/aws/scripts/eks/`](../../aws/scripts/eks/) | -| `slurm/` | 2 shell | [`suites/slurm.yaml`](../../../suites/slurm.yaml) | - | - | +| `security/` | 17 | [`suites/security.yaml`](../../../suites/security.yaml) | [`config/security.yaml`](../config/security.yaml) | [`providers/aws/scripts/security/`](../../aws/scripts/security/), [`providers/aws/scripts/capacity/`](../../aws/scripts/capacity/) | +| `k8s/` | 9 shell | [`suites/k8s.yaml`](../../../suites/k8s.yaml) | [`config/k8s.yaml`](../config/k8s.yaml) | [`providers/aws/scripts/eks/`](../../aws/scripts/eks/) | +| `slurm/` | 2 shell | [`suites/slurm.yaml`](../../../suites/slurm.yaml) | [`config/slurm.yaml`](../config/slurm.yaml) | - | + +The `k8s/` and `slurm/` examples drive a **real** cluster (validations shell out +to `kubectl` / `sinfo`), so they are not part of `make demo-test` — a +dummy-success stub has nothing to return for them. See [`suites/README.md`](../../../suites/README.md) for the per-step / per-field breakdown. @@ -63,9 +67,35 @@ to generate outside `isvctl/configs/providers/`. **4. Run for real (no demo flag):** ```bash +# A platform suite - the obligation attached to declaring that capability +uv run isvctl test run --provider acme --suite vm + +# A plain suite: core checks by default, capability-gated checks when you name one +uv run isvctl test run --provider acme --suite storage +uv run isvctl test run --provider acme --suite storage --capability vm + +# Or point at the config file directly - the same capability rule applies uv run isvctl test run -f isvctl/configs/providers/acme/config/vm.yaml ``` +Add `--dry-run` to any of these to list what would run and what would be +skipped, without executing a thing. + +### Which checks run + +Checks in a plain suite declare what they presuppose. `requires: []` (core) runs +in every context; `requires: [kubernetes]` runs only under +`--capability kubernetes`; `requires: [vm, bare_metal]` is any-match — either +context satisfies it. A plain suite with no `--capability` runs its core checks. + +The same applies to your steps: if a step builds or destroys a fixture only some +contexts need, gate it so a core run neither provisions nor leaks it. Both halves +of a fixture take the same gate — see `config/storage.yaml`, where +`setup_cluster` and `teardown_cluster` are both `requires: [kubernetes]`. + +Nothing is mandatory. A check is in scope only if you declared the suite holding +it, so 100% is always relative to what you declared. + ## Private provider repositories You do not need to contribute your provider scripts back to this repository, diff --git a/isvctl/configs/providers/my-isv/scripts/storage/setup_cluster.py b/isvctl/configs/providers/my-isv/scripts/storage/setup_cluster.py new file mode 100644 index 000000000..e1d0525cc --- /dev/null +++ b/isvctl/configs/providers/my-isv/scripts/storage/setup_cluster.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Set up a Kubernetes cluster with the provider's CSI drivers.""" + +import json +import os +import sys +from typing import Any + +DEMO_MODE = os.environ.get("ISVCTL_DEMO_MODE") == "1" + + +def main() -> int: + """Emit the provider-neutral cluster and StorageClass contract.""" + result: dict[str, Any] = { + "success": False, + "platform": "kubernetes", + "kubeconfig_path": "", + "csi": { + "block_storage_class": "", + "shared_fs_storage_class": "", + "nfs_storage_class": "", + "static_volume_handle": "", + "static_driver_name": "", + "static_volume_az": "", + }, + } + + # TODO: Provision (or reuse) a cluster with your CSI drivers, then return + # its kubeconfig and StorageClass names using this contract. + if DEMO_MODE: + result.update( + { + "success": True, + "kubeconfig_path": "/tmp/isvctl-demo-kubeconfig", + "csi": { + "block_storage_class": "demo-block", + "shared_fs_storage_class": "demo-shared", + "nfs_storage_class": "demo-nfs", + "static_volume_handle": "", + "static_driver_name": "", + "static_volume_az": "", + }, + } + ) + else: + result["error"] = "Not implemented - set up a Kubernetes cluster with CSI installed" + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/providers/my-isv/scripts/storage/teardown_cluster.py b/isvctl/configs/providers/my-isv/scripts/storage/teardown_cluster.py new file mode 100644 index 000000000..019cdf76f --- /dev/null +++ b/isvctl/configs/providers/my-isv/scripts/storage/teardown_cluster.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Release the Kubernetes cluster that setup_cluster acquired.""" + +import argparse +import json +import os +import sys +from typing import Any + +DEMO_MODE = os.environ.get("ISVCTL_DEMO_MODE") == "1" + + +def main() -> int: + """Emit the provider-neutral cluster teardown contract.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--kubeconfig", default="", help="Kubeconfig emitted by setup_cluster") + parser.add_argument("--skip-destroy", action="store_true", help="Keep the cluster for cheap reruns") + args = parser.parse_args() + + result: dict[str, Any] = { + "success": False, + "platform": "kubernetes", + "resources_deleted": [], + "message": "", + } + + if args.skip_destroy: + result.update({"success": True, "skipped": True, "message": "Teardown skipped (--skip-destroy)"}) + print(json.dumps(result, indent=2)) + return 0 + + # TODO: Release whatever setup_cluster acquired. This is deliberately the + # mirror of that step: if setup_cluster created a cluster, destroy it here; + # if it reused a long-lived one, a no-op success is the correct answer. The + # step must still exist either way, so a standalone storage run never leaks + # a cluster it provisioned. + if DEMO_MODE: + result.update( + { + "success": True, + "resources_deleted": ["demo-cluster"], + "message": "Cluster released", + } + ) + else: + result["error"] = "Not implemented - release the cluster setup_cluster acquired" + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/providers/nico/config/control-plane.yaml b/isvctl/configs/providers/nico/config/control-plane.yaml index 6ae155635..5300ea95a 100644 --- a/isvctl/configs/providers/nico/config/control-plane.yaml +++ b/isvctl/configs/providers/nico/config/control-plane.yaml @@ -45,7 +45,6 @@ commands: timeout: 120 tests: - platform: control_plane cluster_name: "nico-control-plane-validation" description: "Verify NICo API health" diff --git a/isvctl/configs/providers/nico/config/iam.yaml b/isvctl/configs/providers/nico/config/iam.yaml index 2738919c3..4711568ee 100644 --- a/isvctl/configs/providers/nico/config/iam.yaml +++ b/isvctl/configs/providers/nico/config/iam.yaml @@ -44,7 +44,6 @@ commands: timeout: 120 tests: - platform: iam cluster_name: "nico-iam-validation" description: "Verify NICo credential readiness" diff --git a/isvctl/configs/providers/nico/config/network.yaml b/isvctl/configs/providers/nico/config/network.yaml index f9764a3a8..80a345ad4 100644 --- a/isvctl/configs/providers/nico/config/network.yaml +++ b/isvctl/configs/providers/nico/config/network.yaml @@ -92,7 +92,6 @@ commands: timeout: 120 tests: - platform: network cluster_name: "nico-network-validation" description: "Verify NICo VPC and subnet inventory" diff --git a/isvctl/configs/providers/nico/config/observability.yaml b/isvctl/configs/providers/nico/config/observability.yaml index 07ed70878..c19b1562e 100644 --- a/isvctl/configs/providers/nico/config/observability.yaml +++ b/isvctl/configs/providers/nico/config/observability.yaml @@ -84,7 +84,6 @@ commands: timeout: 60 tests: - platform: observability cluster_name: "nico-observability-validation" description: "NICo observability validation for UFM event logs and switch log evidence" diff --git a/isvctl/configs/suites/README.md b/isvctl/configs/suites/README.md index 09c19cafe..e801dd0c3 100644 --- a/isvctl/configs/suites/README.md +++ b/isvctl/configs/suites/README.md @@ -9,6 +9,38 @@ commands (steps + scripts) that produce JSON for the validations to check. - **New to the framework?** See the [External Validation Guide](../../../docs/guides/external-validation-guide.md). - **Try it without cloud credentials:** `make demo-test`. +## What runs, and when + +A **platform suite** (`tests.platform: ` — `vm`, `bare_metal`, +`kubernetes`, `slurm`) is the obligation attached to declaring that capability. Its checks +declare no `requires:`; they all run. + +A **plain suite** (everything else — `storage`, `network`, ...) mixes checks +that need no infrastructure with checks that do. Each declares what it +presupposes: + +```yaml +requires: [] # core - runs in every context +requires: [kubernetes] # runs only under --capability kubernetes +requires: [vm, bare_metal] # any-match: either context satisfies it +``` + +One rule decides what runs, and it does not depend on how you named the +config — `--suite`, `-f`, and `--label` discovery all behave identically: + +> **A plain suite with no `--capability` runs its core checks.** Name a +> capability to add the checks gated on it. + +```bash +isvctl test run --provider aws --suite storage # core only +isvctl test run --provider aws --suite storage --capability kubernetes # core + k8s checks +``` + +There is no "run everything" context: no ISV runs on `vm` and `kubernetes` +at once, so a run always carries exactly one context. Steps follow the same +rule — give a step `requires:` when it builds or tears down a fixture only +some contexts need, so a core run neither provisions nor leaks it. + Suites: [`iam`](iam.yaml), [`network`](network.yaml), @@ -141,13 +173,15 @@ volume. The three test-phase steps all reuse that fixture. | Step | Phase | Script | Key JSON Fields | |------|-------|--------|-----------------| -| `launch_instance` | setup | `providers/my-isv/scripts/vm/launch_instance.py` | `instance_id`, `state`, `public_ip`, `key_file` (reuses VM script) | +| `setup_cluster` | setup | `providers/my-isv/scripts/storage/setup_cluster.py` | `kubeconfig_path`, `csi.{block,shared_fs,nfs}_storage_class` (only under `kubernetes`) | +| `launch_instance` | setup | `providers/my-isv/scripts/vm/launch_instance.py` | `instance_id`, `state`, `public_ip`, `key_file` (reuses VM script; only under `vm`/`bare_metal`) | | `create_volume` | setup | `providers/my-isv/scripts/storage/create_volume.py` | `volume_id`, `mount_point`, `sentinel_content`, `operations.{create,attach,format,mount,write_sentinel}` | | `snapshot_lifecycle` | test | `providers/my-isv/scripts/storage/snapshot_lifecycle.py` | `volume_id`, `snapshot_id`, `operations.{create_snapshot,restore_volume,verify_data}` (verify_data includes `content_matches`) | | `volume_resize` | test | `providers/my-isv/scripts/storage/volume_resize.py` | `volume_id`, `operations.{modify_volume,grow_partition,resize_filesystem,verify_size}` | | `volume_persistence` | test | `providers/my-isv/scripts/storage/volume_persistence.py` | `volume_id`, `operations.{stop,start,verify_attached,verify_data}` (verify_data includes `content_matches`) | | `teardown_volume` | teardown | `providers/my-isv/scripts/storage/teardown_volume.py` | `resources_deleted`, `message` | | `teardown` | teardown | `providers/my-isv/scripts/vm/teardown.py` | `resources_deleted`, `message` (reuses VM script) | +| `teardown_cluster` | teardown | `providers/my-isv/scripts/storage/teardown_cluster.py` | `resources_deleted`, `message` (only under `kubernetes`; may be a no-op if you reuse a cluster) | ### Kubernetes (`k8s.yaml`) @@ -193,7 +227,8 @@ Validations use `sinfo`/`srun` directly: partitions, GPU allocation, job schedul | `crud_install_config` | test | `providers/my-isv/scripts/image-registry/crud_install_config.py` | `config_id`, `config_name`, `operations` | | `install_image_bm` | test | `providers/my-isv/scripts/image-registry/install_image_bm.py` | `instance_id`, `image_id`, `instance_state` | | `install_config_bm` | test | `providers/my-isv/scripts/image-registry/install_config_bm.py` | `instance_id`, `config_id`, `instance_state`, `state` | -| `teardown` | teardown | `providers/my-isv/scripts/image-registry/teardown.py` | `resources_deleted`, `message` | +| `teardown_instance` | teardown | `providers/my-isv/scripts/image-registry/teardown.py` | `resources_deleted`, `message` (instance, key pair, security group, instance profile — only under `vm`) | +| `teardown_image` | teardown | `providers/my-isv/scripts/image-registry/teardown.py` | `resources_deleted`, `message` (image, disks, bucket — always) | ### Security (`security.yaml`) diff --git a/isvctl/configs/suites/control-plane.yaml b/isvctl/configs/suites/control-plane.yaml index 53a6ef492..bf8e15e53 100644 --- a/isvctl/configs/suites/control-plane.yaml +++ b/isvctl/configs/suites/control-plane.yaml @@ -42,7 +42,6 @@ version: "1.0" tests: - platform: control_plane cluster_name: "control-plane-validation" description: "Control plane: API health, access key lifecycle, tenant lifecycle" @@ -60,6 +59,7 @@ tests: FieldExistsCheck: test_id: "N/A" labels: ["control_plane"] + requires: [] fields: ["account_id", "tests"] # DATASVC-XX-01 (S3-compatible API w/ authenticated endpoints): # authenticated-endpoint half - a successful authenticated check_api @@ -68,6 +68,7 @@ tests: FieldValueCheck: test_id: "DATASVC01-01" labels: ["control_plane", "min_req"] + requires: [] field: "success" expected: true @@ -76,10 +77,12 @@ tests: AccessKeyCreatedCheck: test_id: "CP05-01" labels: ["control_plane", "iam"] + requires: [] step: create_access_key TenantCreatedCheck: test_id: "CP07-01" labels: ["control_plane", "iam"] + requires: [] step: create_tenant access_key_lifecycle: @@ -87,14 +90,17 @@ tests: AccessKeyAuthenticatedCheck: test_id: "CP05-01" labels: ["control_plane", "iam"] + requires: [] step: test_access_key AccessKeyDisabledCheck: test_id: "CP06-01" labels: ["control_plane", "iam", "min_req"] + requires: [] step: disable_access_key AccessKeyRejectedCheck: test_id: "CP06-01" labels: ["control_plane", "iam", "min_req"] + requires: [] step: verify_key_rejected tenant_lifecycle: @@ -102,10 +108,12 @@ tests: TenantListedCheck: test_id: "CP08-01" labels: ["control_plane", "iam"] + requires: [] step: list_tenants TenantInfoCheck: test_id: "CP09-01" labels: ["control_plane", "iam"] + requires: [] step: get_tenant s3_object_lifecycle: @@ -116,13 +124,16 @@ tests: StepSuccessCheck: test_id: "N/A" labels: ["control_plane"] + requires: [] FieldExistsCheck: test_id: "N/A" labels: ["control_plane"] + requires: [] fields: ["bucket_name", "object_key", "operations"] CrudOperationsCheck: test_id: "DATASVC01-01" labels: ["control_plane", "min_req"] + requires: [] operations: ["put", "get", "delete"] teardown_checks: @@ -132,10 +143,12 @@ tests: # CP05-01 (create) and CP06-01 (disable). test_id: "N/A" labels: ["control_plane"] + requires: [] step: delete_access_key StepSuccessCheck-delete_tenant: test_id: "CP10-01" labels: ["control_plane"] + requires: [] step: delete_tenant exclude: diff --git a/isvctl/configs/suites/iam.yaml b/isvctl/configs/suites/iam.yaml index 4ac3be7e1..bd8442fd5 100644 --- a/isvctl/configs/suites/iam.yaml +++ b/isvctl/configs/suites/iam.yaml @@ -37,7 +37,6 @@ version: "1.0" tests: cluster_name: "iam-validation" description: "IAM user lifecycle validation" - platform: iam settings: # Provider-specific - override in your provider config @@ -54,10 +53,12 @@ tests: FieldExistsCheck: test_id: "IAM01-01" labels: ["iam", "min_req"] + requires: [] fields: ["username", "access_key_id"] StepSuccessCheck: test_id: "N/A" labels: ["iam"] + requires: [] credentials: step: test_credentials @@ -65,9 +66,11 @@ tests: IamCredentialAccessCheck: test_id: "IAM03-01" labels: ["iam"] + requires: [] StepSuccessCheck: test_id: "N/A" labels: ["iam"] + requires: [] teardown_checks: step: teardown @@ -75,6 +78,7 @@ tests: StepSuccessCheck: test_id: "N/A" labels: ["iam"] + requires: [] exclude: labels: [] diff --git a/isvctl/configs/suites/image-registry.yaml b/isvctl/configs/suites/image-registry.yaml index c400cfb3d..07a184dc9 100644 --- a/isvctl/configs/suites/image-registry.yaml +++ b/isvctl/configs/suites/image-registry.yaml @@ -37,7 +37,6 @@ version: "1.0" tests: cluster_name: "image-registry-validation" description: "Image registry validation tests" - platform: image_registry settings: # Provider-specific - override in your provider config @@ -57,9 +56,11 @@ tests: StepSuccessCheck: test_id: "N/A" labels: ["image_registry"] + requires: [] FieldExistsCheck: test_id: "BOOT01-01" labels: ["image_registry", "min_req"] + requires: [] fields: ["image_id", "storage_bucket", "disk_ids"] # --- Image CRUD --- @@ -69,13 +70,16 @@ tests: StepSuccessCheck: test_id: "N/A" labels: ["image_registry"] + requires: [] FieldExistsCheck: test_id: "N/A" labels: ["image_registry"] + requires: [] fields: ["image_id", "operations"] CrudOperationsCheck: test_id: "BOOT03-02" labels: ["image_registry", "min_req"] + requires: [] operations: ["get", "list", "create", "delete"] # --- VM Launch from Image --- @@ -85,13 +89,16 @@ tests: StepSuccessCheck: test_id: "BOOT01-05" labels: ["image_registry", "min_req"] + requires: [vm] FieldExistsCheck: test_id: "N/A" labels: ["image_registry"] + requires: [vm] fields: ["instance_id", "public_ip", "key_path"] InstanceStateCheck: test_id: "N/A" labels: ["image_registry"] + requires: [vm] expected_state: "running" vm_ssh: @@ -100,9 +107,11 @@ tests: ConnectivityCheck: test_id: "N/A" labels: ["image_registry", "ssh"] + requires: [vm] OsCheck: test_id: "N/A" labels: ["image_registry", "ssh"] + requires: [vm] expected_os: "ubuntu" # --- OS Install Config CRUD --- @@ -112,9 +121,11 @@ tests: StepSuccessCheck: test_id: "N/A" labels: ["image_registry"] + requires: [] FieldExistsCheck: test_id: "BOOT01-02" labels: ["image_registry"] + requires: [] fields: ["config_id", "config_name", "operations"] # --- OS Image on BMaaS --- @@ -124,13 +135,16 @@ tests: StepSuccessCheck: test_id: "BOOT01-03" labels: ["image_registry", "min_req"] + requires: [bare_metal] FieldExistsCheck: test_id: "N/A" labels: ["image_registry"] + requires: [bare_metal] fields: ["instance_id", "image_id", "instance_state"] InstanceStateCheck: test_id: "N/A" labels: ["image_registry"] + requires: [bare_metal] expected_state: "running" # --- OS Install Config on BMaaS --- @@ -140,21 +154,28 @@ tests: StepSuccessCheck: test_id: "BOOT01-04" labels: ["image_registry", "min_req"] + requires: [bare_metal] FieldExistsCheck: test_id: "N/A" labels: ["image_registry"] + requires: [bare_metal] fields: ["instance_id", "config_id", "instance_state"] InstanceStateCheck: test_id: "N/A" labels: ["image_registry"] + requires: [bare_metal] expected_state: "running" + # Bound to the image half of teardown: it is ungated, so this stays a core + # check. The instance half (teardown_instance) only runs under vm and owns + # nothing an ISV must prove, so it carries no check of its own. teardown_checks: - step: teardown + step: teardown_image checks: StepSuccessCheck: test_id: "N/A" labels: ["image_registry"] + requires: [] exclude: labels: [] diff --git a/isvctl/configs/suites/k8s.yaml b/isvctl/configs/suites/k8s.yaml index ea87d5f38..489456533 100644 --- a/isvctl/configs/suites/k8s.yaml +++ b/isvctl/configs/suites/k8s.yaml @@ -239,207 +239,6 @@ tests: wait_timeout: 180 poll_interval: 2 - k8s_storage: - checks: - K8sCsiStorageTypesCheck: - test_id: "K8S23-04" - labels: ["kubernetes", "min_req"] - # StorageClass names by type; omit (or leave empty) to skip that type. - # Env overrides: K8S_CSI_BLOCK_SC, K8S_CSI_SHARED_FS_SC, K8S_CSI_NFS_SC. - block_storage_class: "{{ steps.setup.csi.block_storage_class | default('', true) }}" - shared_fs_storage_class: "{{ steps.setup.csi.shared_fs_storage_class | default('', true) }}" - nfs_storage_class: "{{ steps.setup.csi.nfs_storage_class | default('', true) }}" - pvc_size: "1Gi" - bind_timeout_s: 120 - namespace_prefix: "isvtest-csi-types" - K8sCsiStorageQuotaApiCheck: - test_id: "K8S23-07" - labels: ["kubernetes", "min_req"] - # Uses the block StorageClass by default; skips entirely when unset. - # per_sc_quota must exceed pvc_request so the usage PVC fits, and - # over_quota_request must exceed per_sc_quota so admission rejects it. - storage_class: "{{ steps.setup.csi.block_storage_class | default('', true) }}" - total_quota: "10Gi" - per_sc_quota: "5Gi" - pvc_request: "1Gi" - over_quota_request: "100Gi" - bind_timeout_s: 120 - quota_settle_s: 30 - namespace_prefix: "isvtest-csi-quota" - K8sCsiTenantScopedCredentialsCheck: - test_id: "K8S23-06" - labels: ["kubernetes", "min_req"] - # Pure read-only check: verifies CSI credentials are tenant-scoped - # by construction (Secret namespaces, labels/annotations, RBAC, - # and node-plugin volume topology). Skipped entirely when no - # CSIDriver objects exist. - csi_driver_namespaces: - - "kube-system" - allowed_workload_namespaces: [] - forbidden_labels: - - "shared-across-clusters=true" - K8sCsiProvisioningModesCheck: - test_id: "K8S23-05" - labels: ["kubernetes", "min_req"] - # Exercises dynamic and static CSI provisioning end-to-end with a - # BusyBox mount + canary-file write/read. Skipped entirely when - # dynamic_storage_class is unset. The static subtest is skipped - # (not failed) when static_pv.volume_handle / csi_driver are - # unset, so providers that don't pre-provision a volume can still - # run the dynamic probe. - dynamic_storage_class: "{{ steps.setup.csi.block_storage_class | default('', true) }}" - dynamic_pvc_size: "1Gi" - static_pv: - volume_handle: "{{ steps.setup.csi.static_volume_handle | default('', true) }}" - csi_driver: "{{ steps.setup.csi.static_driver_name | default('', true) }}" - # Zonal block volumes (EBS/PD/Disk) must pin their consumer pod to - # the volume's AZ; empty for zone-agnostic backends. - zone: "{{ steps.setup.csi.static_volume_az | default('', true) }}" - fs_type: "ext4" - capacity: "1Gi" - access_mode: "ReadWriteOnce" - bind_timeout_s: 180 - namespace_prefix: "isvtest-csi-prov" - K8sCsiDriverHealthCheck: - test_id: "N/A" - labels: ["kubernetes", "min_req"] - # Verifies each configured CSI driver is installed and healthy. - # Controller/node health (Deployment + DaemonSet) runs only - # when a spec supplies a `workloads` block; otherwise those subtests are - # skipped (driver registration is still checked). - # - storage_classes: ["..."] - # workloads: - # namespace: kube-system - # controller: { deployment: } - # node: { daemonset: } - drivers: - - storage_classes: - - "{{ steps.setup.csi.block_storage_class | default('', true) }}" - workloads: {} - - storage_classes: - - "{{ steps.setup.csi.shared_fs_storage_class | default('', true) }}" - workloads: {} - - storage_classes: - - "{{ steps.setup.csi.nfs_storage_class | default('', true) }}" - workloads: {} - min_controller_replicas: 1 - K8sCsiConcurrentPvcCheck: - test_id: "N/A" - labels: ["kubernetes", "min_req"] - storage_class: "{{ steps.setup.csi.block_storage_class | default('', true) }}" - pvc_count: 2 - pvc_size: "1Gi" - bind_timeout_s: 120 - namespace_prefix: "isvtest-csi-concurrent" - K8sCsiPvcExpandCheck: - test_id: "N/A" - labels: ["kubernetes", "min_req"] - storage_class: "{{ steps.setup.csi.block_storage_class | default('', true) }}" - initial_size: "1Gi" - expanded_size: "2Gi" - bind_timeout_s: 120 - expand_timeout_s: 180 - namespace_prefix: "isvtest-csi-expand" - K8sNodeKernelModulesCheck: - test_id: "HSS11-04" - labels: ["kubernetes", "storage"] - kernel_modules: [] - node_selector: {} - image: "busybox:1.36" - bind_timeout_s: 60 - namespace_prefix: "isvtest-kmod" - K8sNfsMountOptionsCheck: - test_id: "HSS11-01" - labels: ["kubernetes", "storage"] - # Env overrides: K8S_CSI_SHARED_FS_SC, K8S_CSI_NFS_SC. - shared_fs_storage_class: "{{ steps.setup.csi.shared_fs_storage_class | default('', true) }}" - nfs_storage_class: "{{ steps.setup.csi.nfs_storage_class | default('', true) }}" - image: "busybox:1.36" - node_selector: {} - pvc_size: "1Gi" - bind_timeout_s: 180 - namespace_prefix: "isvtest-fs-nfs" - # if expected values are not set, the check will be skipped - expected_version: "" - expected_nconnect: "" - expected_proto: "" - expected_read_ahead_kb: "" - - k8s_filesystem: - checks: - # Shared-filesystem POSIX-semantics checks (multi-pod, RWX PVC). - K8sFileLockingCheck: - test_id: "HSS14-02" - labels: ["kubernetes", "storage"] - shared_fs_storage_class: "{{ steps.setup.csi.shared_fs_storage_class | default('', true) }}" - nfs_storage_class: "{{ steps.setup.csi.nfs_storage_class | default('', true) }}" - image: "busybox:1.36" - node_selector: {} - pvc_size: "1Gi" - bind_timeout_s: 120 - release_timeout_s: 60 - namespace_prefix: "isvtest-fs-lock" - K8sCrossNodeWriteVisibilityCheck: - test_id: "HSS17-01" - labels: ["kubernetes", "storage"] - shared_fs_storage_class: "{{ steps.setup.csi.shared_fs_storage_class | default('', true) }}" - nfs_storage_class: "{{ steps.setup.csi.nfs_storage_class | default('', true) }}" - image: "busybox:1.36" - node_selector: {} - pvc_size: "1Gi" - bind_timeout_s: 120 - visibility_window_s: 5.0 - namespace_prefix: "isvtest-fs-vis" - K8sCrossNodeAttrConsistencyCheck: - test_id: "HSS17-02" - labels: ["kubernetes", "storage"] - shared_fs_storage_class: "{{ steps.setup.csi.shared_fs_storage_class | default('', true) }}" - nfs_storage_class: "{{ steps.setup.csi.nfs_storage_class | default('', true) }}" - image: "busybox:1.36" - node_selector: {} - pvc_size: "1Gi" - bind_timeout_s: 120 - attr_cache_window_s: 30.0 - extend_bytes: 1024 - namespace_prefix: "isvtest-fs-attr" - K8sLargeDirListingFilesCheck: - test_id: "HSS07-02" - labels: ["kubernetes", "storage"] - shared_fs_storage_class: "{{ steps.setup.csi.shared_fs_storage_class | default('', true) }}" - nfs_storage_class: "{{ steps.setup.csi.nfs_storage_class | default('', true) }}" - image: "busybox:1.36" - node_selector: {} - files_count: 10000 - pvc_size: "10Gi" - bind_timeout_s: 120 - namespace_prefix: "isvtest-fs-bigdir" - timeout: 3600 - K8sLargeDirListingDirsCheck: - test_id: "HSS07-02" - labels: ["kubernetes", "storage"] - shared_fs_storage_class: "{{ steps.setup.csi.shared_fs_storage_class | default('', true) }}" - nfs_storage_class: "{{ steps.setup.csi.nfs_storage_class | default('', true) }}" - image: "busybox:1.36" - node_selector: {} - dirs_count: 500 - pvc_size: "10Gi" - bind_timeout_s: 120 - namespace_prefix: "isvtest-fs-bigdir" - timeout: 3600 - K8sPosixComplianceCheck: - test_id: "DIR02-02" - labels: ["kubernetes", "slow", "storage"] - shared_fs_storage_class: "{{ steps.setup.csi.shared_fs_storage_class | default('', true) }}" - nfs_storage_class: "{{ steps.setup.csi.nfs_storage_class | default('', true) }}" - image: "gcc:12" - node_selector: {} - pvc_size: "5Gi" - bind_timeout_s: 120 - build_timeout_s: 600 - # tests_subdir: "" # optional: scope to a single tests/ subdir (e.g. "chmod") - namespace_prefix: "isvtest-fs-posix" - timeout: 3600 - k8s_identity: checks: K8sOidcIssuerCheck: diff --git a/isvctl/configs/suites/network.yaml b/isvctl/configs/suites/network.yaml index 181f1405b..e9b19f11d 100644 --- a/isvctl/configs/suites/network.yaml +++ b/isvctl/configs/suites/network.yaml @@ -37,7 +37,6 @@ version: "1.0" tests: cluster_name: "network-validation" description: "Network validation tests" - platform: network settings: # Provider-specific - override in your provider config @@ -53,6 +52,7 @@ tests: NetworkProvisionedCheck: test_id: "N/A" labels: ["network"] + requires: [] require_subnets: true min_subnets: 2 @@ -63,27 +63,32 @@ tests: VpcCrudCheck-create: test_id: "SDN01-01" labels: ["min_req", "network"] + requires: [] step: vpc_crud operations: ["create_vpc"] VpcCrudCheck-read: test_id: "SDN01-02" labels: ["min_req", "network"] + requires: [] step: vpc_crud operations: ["read_vpc"] VpcCrudCheck-update: test_id: "SDN01-03" labels: ["min_req", "network"] + requires: [] step: vpc_crud operations: ["update_tags", "update_dns"] VpcCrudCheck-delete: test_id: "SDN01-04" labels: ["min_req", "network"] + requires: [] step: vpc_crud operations: ["delete_vpc"] # no test id for subnet count / multi-AZ layout (SDN04-01 covers exclusive-subnet isolation, not subnet topology). SubnetConfigCheck: test_id: "N/A" labels: ["network"] + requires: [] step: subnet_config min_subnets: 4 require_multi_az: true @@ -91,25 +96,30 @@ tests: # SDN04-03 (E/W isolation) is exercised by the same check but is not separately attributable under a single test_id. test_id: "SDN04-02" labels: ["min_req", "network", "security"] + requires: [] step: vpc_isolation SgCrudCheck-create: test_id: "SDN02-01" labels: ["min_req", "network", "security"] + requires: [] step: sg_crud operations: ["create_vpc", "create_sg"] SgCrudCheck-read: test_id: "SDN02-02" labels: ["min_req", "network", "security"] + requires: [] step: sg_crud operations: ["read_sg"] SgCrudCheck-update: test_id: "SDN02-03" labels: ["min_req", "network", "security"] + requires: [] step: sg_crud operations: ["update_sg_add_rule", "update_sg_modify_rule", "update_sg_remove_rule"] SgCrudCheck-delete: test_id: "SDN02-04" labels: ["min_req", "network", "security"] + requires: [] step: sg_crud operations: ["delete_sg", "verify_deleted"] # SG/NACL enforcement + connectivity/traffic behavior have no dedicated @@ -117,22 +127,27 @@ tests: SecurityBlockingCheck: test_id: "N/A" labels: ["network", "security"] + requires: [] step: security_blocking NetworkConnectivityCheck: test_id: "N/A" labels: ["network"] + requires: [vm, bare_metal] step: connectivity_test TrafficFlowCheck: test_id: "N/A" labels: ["network"] + requires: [vm, bare_metal] step: traffic_validation VpcIpConfigCheck: test_id: "IPAM02-01" labels: ["network"] + requires: [] step: vpc_ip_config DhcpIpManagementCheck: test_id: "IPAM01-01" labels: ["network", "ssh"] + requires: [vm, bare_metal] step: dhcp_ip_test sg_scoping: @@ -140,22 +155,27 @@ tests: SgWorkloadScopingCheck: test_id: "SDN02-05" labels: ["min_req", "network", "security"] + requires: [] step: sg_workload_scoping SgNodeScopingCheck: test_id: "SDN02-06" labels: ["min_req", "network", "security"] + requires: [] step: sg_node_scoping SgSubnetScopingCheck: test_id: "SDN02-07" labels: ["min_req", "network", "security"] + requires: [] step: sg_subnet_scoping SgServiceScopingCheck: test_id: "SDN02-09" labels: ["min_req", "network", "security"] + requires: [] step: sg_service_scoping SgPortSecurityPolicyCheck: test_id: "SDN02-10" labels: ["min_req", "network", "security"] + requires: [] step: sg_port_security_policy sdn_policy: @@ -163,6 +183,7 @@ tests: SgPolicyPropagationTimingCheck: test_id: "SDN02-08" labels: ["min_req", "network", "security"] + requires: [] step: sg_policy_propagation max_propagation_seconds: 10 @@ -171,14 +192,17 @@ tests: SdnHardwareFaultLoggingCheck: test_id: "SDN09-01" labels: ["min_req", "network"] + requires: [] step: sdn_hardware_fault_logging SdnLatencyPerfLoggingCheck: test_id: "SDN09-02" labels: ["min_req", "network"] + requires: [] step: sdn_latency_perf_logging SdnFilterAuditTrailCheck: test_id: "SDN09-03" labels: ["min_req", "network", "security"] + requires: [] step: sdn_filter_audit_trail sdn: @@ -186,31 +210,38 @@ tests: ByoipCheck: test_id: "NET03-01" labels: ["min_req", "network"] + requires: [] step: byoip_test StablePrivateIpCheck: test_id: "SDN11-01" labels: ["min_req", "network"] + requires: [vm, bare_metal] step: stable_ip_test StableEgressIpCheck: test_id: "DMS05-01" labels: ["min_req", "network"] + requires: [vm, bare_metal] step: stable_egress_ip_test FloatingIpCheck: test_id: "SDN05-01" labels: ["min_req", "network"] + requires: [vm, bare_metal] step: floating_ip_test max_switch_seconds: 10 LocalizedDnsCheck: test_id: "SDN06-01" labels: ["min_req", "network"] + requires: [] step: dns_test VpcPeeringCheck: test_id: "SDN07-01" labels: ["min_req", "network"] + requires: [] step: peering_test StorageL3RoutingCheck: test_id: "SDN08-01" labels: ["min_req", "network"] + requires: [vm, bare_metal] step: storage_l3_routing # Read-only metadata/API checks for provider-reported backend fabric @@ -220,10 +251,12 @@ tests: BackendSwitchFabricCheck: test_id: "NET01-01" labels: ["min_req", "network"] + requires: [vm, bare_metal] step: backend_switch_fabric NvlinkDomainCheck: test_id: "NET02-02" labels: ["min_req", "network"] + requires: [vm, bare_metal] step: nvlink_domain teardown_checks: @@ -232,6 +265,7 @@ tests: StepSuccessCheck: test_id: "N/A" labels: ["network"] + requires: [] exclude: labels: [] diff --git a/isvctl/configs/suites/observability.yaml b/isvctl/configs/suites/observability.yaml index db36a6f8f..650339130 100644 --- a/isvctl/configs/suites/observability.yaml +++ b/isvctl/configs/suites/observability.yaml @@ -23,7 +23,6 @@ version: "1.0" tests: cluster_name: "observability-validation" description: "Observability collector validation tests" - platform: observability settings: region: "" @@ -40,6 +39,7 @@ tests: TelemetryDeliveryLatencyCheck: test_id: "OBS05-01" labels: ["min_req", "observability"] + requires: [vm, bare_metal] step: telemetry_delivery_latency max_delivery_seconds: "{{max_delivery_seconds}}" @@ -48,22 +48,27 @@ tests: NorthSouthNetworkTelemetryCheck: test_id: "OBS06-01" labels: ["min_req", "network", "observability"] + requires: [vm, bare_metal] step: north_south_network_telemetry EastWestNetworkTelemetryCheck: test_id: "OBS07-01" labels: ["min_req", "network", "observability"] + requires: [] step: east_west_network_telemetry ManagementNetworkTelemetryCheck: test_id: "OBS08-01" labels: ["min_req", "network", "observability"] + requires: [] step: management_network_telemetry NvswitchFabricTelemetryCheck: test_id: "OBS09-01" labels: ["min_req", "network", "observability"] + requires: [] step: nvswitch_fabric_telemetry HostNicNetworkTelemetryCheck: test_id: "OBS10-01" labels: ["min_req", "network", "observability"] + requires: [vm, bare_metal] step: host_nic_network_telemetry network_logs: @@ -71,6 +76,7 @@ tests: VpcFlowLogsCheck: test_id: "OBS19-01" labels: ["min_req", "network", "observability"] + requires: [] step: vpc_flow_logs host_logs: @@ -78,6 +84,7 @@ tests: HostSyslogCheck: test_id: "OBS18-01" labels: ["bare_metal", "min_req", "observability"] + requires: [vm, bare_metal] step: host_syslogs bmc_logs: @@ -85,6 +92,7 @@ tests: BmcSelLogsCheck: test_id: "OBS17-01" labels: ["min_req", "observability", "security"] + requires: [] step: bmc_sel_logs bmc_telemetry: @@ -92,6 +100,7 @@ tests: BmcGpuTelemetryCheck: test_id: "TELEM04-01" labels: ["gpu", "min_req", "observability", "security"] + requires: [] step: bmc_gpu_telemetry storage_capacity_telemetry: @@ -99,6 +108,11 @@ tests: StorageCapacityTelemetryCheck: test_id: "TELEM05-01" labels: ["min_req", "observability"] + # Probes the volumes attached to a running instance, exactly like its + # StoragePerformanceTelemetryCheck twin (same script, different + # --aspect). Without one it reports "no volumes attached", so it can + # never pass as a core check. + requires: [vm, bare_metal] step: storage_capacity_telemetry storage_performance_telemetry: @@ -106,6 +120,7 @@ tests: StoragePerformanceTelemetryCheck: test_id: "TELEM06-01" labels: ["min_req", "observability"] + requires: [vm, bare_metal] step: storage_performance_telemetry gpu_nvlink_telemetry: @@ -113,6 +128,7 @@ tests: GpuNvlinkTelemetryCheck: test_id: "TELEM07-01" labels: ["min_req", "observability"] + requires: [] step: gpu_nvlink_telemetry switch_nvlink_telemetry: @@ -120,6 +136,7 @@ tests: SwitchNvlinkTelemetryCheck: test_id: "TELEM08-01" labels: ["min_req", "observability"] + requires: [] step: switch_nvlink_telemetry fabric_logs: @@ -127,10 +144,12 @@ tests: FabricManagerLogsCheck: test_id: "OBS11-01" labels: ["min_req", "observability"] + requires: [] step: fabric_manager_logs UfmEventLogsCheck: test_id: "OBS13-01" labels: ["min_req", "observability"] + requires: [] step: ufm_event_logs subnet_manager_logs: @@ -138,6 +157,7 @@ tests: SubnetManagerLogsCheck: test_id: "OBS12-01" labels: ["min_req", "observability"] + requires: [] step: subnet_manager_logs switch_logs: @@ -145,14 +165,17 @@ tests: GeneralSwitchLogsCheck: test_id: "OBS14-01" labels: ["min_req", "network", "observability"] + requires: [] step: general_switch_logs SwitchSyslogCheck: test_id: "OBS15-01" labels: ["min_req", "network", "observability"] + requires: [] step: switch_syslogs SwitchKernelLogsCheck: test_id: "OBS16-01" labels: ["min_req", "network", "observability"] + requires: [] step: switch_kernel_logs exclude: diff --git a/isvctl/configs/suites/security.yaml b/isvctl/configs/suites/security.yaml index e08fe66ad..fdfc1cff5 100644 --- a/isvctl/configs/suites/security.yaml +++ b/isvctl/configs/suites/security.yaml @@ -36,7 +36,6 @@ version: "1.0" tests: cluster_name: "security-validation" description: "Infrastructure security hardening validation" - platform: security settings: region: "" @@ -55,6 +54,7 @@ tests: BmcManagementNetworkCheck: test_id: "SEC12-01" labels: ["min_req", "network", "security"] + requires: [] step: bmc_management_network bmc_isolation: @@ -62,6 +62,7 @@ tests: BmcTenantIsolationCheck: test_id: "SEC12-02" labels: ["min_req", "network", "security"] + requires: [] step: bmc_tenant_isolation bmc_protocol_security: @@ -69,6 +70,7 @@ tests: BmcProtocolSecurityCheck: test_id: "CNP10-01" labels: ["min_req", "network", "security"] + requires: [] step: bmc_protocol_security bmc_bastion_access: @@ -76,6 +78,7 @@ tests: BmcBastionAccessCheck: test_id: "SEC12-03" labels: ["min_req", "network", "security"] + requires: [] step: bmc_bastion_access api_security: @@ -83,6 +86,7 @@ tests: ApiEndpointIsolationCheck: test_id: "SEC14-01" labels: ["min_req", "network", "security"] + requires: [] step: api_endpoint_isolation mutual_tls: @@ -90,6 +94,7 @@ tests: MutualTlsCheck: test_id: "SEC13-01" labels: ["min_req", "network", "security"] + requires: [] step: mutual_tls_test insecure_protocols: @@ -97,6 +102,7 @@ tests: InsecureProtocolsCheck: test_id: "SEC13-02" labels: ["min_req", "network", "security"] + requires: [] step: insecure_protocols_test mfa_enforcement: @@ -104,6 +110,7 @@ tests: MfaEnforcedCheck: test_id: "SEC07-01" labels: ["iam", "min_req", "security"] + requires: [] step: mfa_enforcement cert_rotation: @@ -111,6 +118,7 @@ tests: CertRotationCycleCheck: test_id: "SEC09-01" labels: ["min_req", "security"] + requires: [] step: cert_rotation_test kms_encryption_options: @@ -118,6 +126,7 @@ tests: KmsEncryptionOptionCheck: test_id: "SEC09-02" labels: ["min_req", "security"] + requires: [] step: kms_encryption_options_test centralized_kms: @@ -125,6 +134,7 @@ tests: CentralizedKmsCheck: test_id: "SEC09-03" labels: ["min_req", "security"] + requires: [] step: centralized_kms_test customer_managed_key: @@ -132,6 +142,7 @@ tests: CustomerManagedKeyCheck: test_id: "SEC09-04" labels: ["min_req", "security", "slow", "workload"] + requires: [] step: customer_managed_key_test service_account_auth: @@ -139,6 +150,7 @@ tests: ServiceAccountCredentialCheck: test_id: "SEC03-01" labels: ["iam", "min_req", "security"] + requires: [] step: sa_credential_test user_auth_oidc: @@ -146,6 +158,7 @@ tests: OidcUserAuthCheck: test_id: "SEC01-01" labels: ["iam", "min_req", "security"] + requires: [] step: oidc_user_auth_test short_lived_credentials: @@ -153,6 +166,7 @@ tests: ShortLivedCredentialsCheck: test_id: "SEC06-01" labels: ["iam", "min_req", "security"] + requires: [] step: short_lived_credentials_test least_privilege_policy: @@ -160,6 +174,7 @@ tests: LeastPrivilegePolicyCheck: test_id: "SEC04-01" labels: ["iam", "min_req", "security"] + requires: [] step: least_privilege_test minimal_role_enforcement: @@ -167,6 +182,7 @@ tests: MinimalRoleEnforcementCheck: test_id: "SEC04-02" labels: ["iam", "min_req", "security"] + requires: [] step: least_privilege_test audit_log_entry: @@ -174,6 +190,7 @@ tests: AuditLogEntryCheck: test_id: "SEC08-01" labels: ["min_req", "security"] + requires: [] step: audit_logging_test audit_log_retention: @@ -181,6 +198,7 @@ tests: AuditLogRetentionCheck: test_id: "SEC08-02" labels: ["min_req", "security"] + requires: [] step: audit_logging_test tenant_isolation: @@ -188,6 +206,7 @@ tests: TenantIsolationCheck: test_id: "SEC11-01" labels: ["iam", "min_req", "network", "security"] + requires: [vm, bare_metal] step: tenant_isolation_test # Verify capacity is logically grouped and pinned to one tenant/account @@ -197,6 +216,7 @@ tests: CapacityReservationGroupingCheck: test_id: "CAP04-01" labels: ["bare_metal", "capacity", "min_req", "security"] + requires: [vm, bare_metal] step: capacity_reservation_grouping min_resources: "{{min_resources}}" @@ -208,6 +228,7 @@ tests: CapacityTopologyBlockAtomicAllocationCheck: test_id: "CAP04-02" labels: ["bare_metal", "capacity", "min_req", "security"] + requires: [vm, bare_metal] min_resources: 2 teardown_checks: @@ -216,6 +237,7 @@ tests: StepSuccessCheck: test_id: "N/A" labels: ["security"] + requires: [] exclude: labels: [] diff --git a/isvctl/configs/suites/storage.yaml b/isvctl/configs/suites/storage.yaml index ef43c9464..e5ee24537 100644 --- a/isvctl/configs/suites/storage.yaml +++ b/isvctl/configs/suites/storage.yaml @@ -53,7 +53,6 @@ version: "1.0" tests: - platform: storage cluster_name: "storage-validation" description: "Storage: block volumes and high-speed / parallel filesystem services" @@ -74,6 +73,7 @@ tests: InstanceStateCheck: test_id: "N/A" labels: ["storage"] + requires: [vm, bare_metal] expected_state: "running" fixture_volume: @@ -82,13 +82,16 @@ tests: StepSuccessCheck: test_id: "N/A" labels: ["storage"] + requires: [vm, bare_metal] FieldExistsCheck: test_id: "N/A" labels: ["storage"] + requires: [vm, bare_metal] fields: ["volume_id", "mount_point", "operations"] CrudOperationsCheck: test_id: "N/A" labels: ["storage"] + requires: [vm, bare_metal] operations: ["create", "attach", "format", "mount", "write_sentinel"] # ─── DATASVC-XX-02: volume snapshots (issue #321) ──────────────── @@ -98,13 +101,16 @@ tests: StepSuccessCheck: test_id: "N/A" labels: ["storage"] + requires: [vm, bare_metal] FieldExistsCheck: test_id: "N/A" labels: ["storage"] + requires: [vm, bare_metal] fields: ["volume_id", "snapshot_id", "operations"] CrudOperationsCheck: test_id: "DATASVC02-01" labels: ["min_req", "storage"] + requires: [vm, bare_metal] operations: ["create_snapshot", "restore_volume", "verify_data"] # ─── DATASVC-XX-03: volume resizing (issue #322) ───────────────── @@ -114,13 +120,16 @@ tests: StepSuccessCheck: test_id: "N/A" labels: ["storage"] + requires: [vm, bare_metal] FieldExistsCheck: test_id: "N/A" labels: ["storage"] + requires: [vm, bare_metal] fields: ["volume_id", "operations"] CrudOperationsCheck: test_id: "DATASVC03-01" labels: ["min_req", "storage"] + requires: [vm, bare_metal] operations: ["modify_volume", "grow_partition", "resize_filesystem", "verify_size"] # ─── DATASVC-XX-04: persistence across restarts (issue #323) ───── @@ -130,13 +139,16 @@ tests: StepSuccessCheck: test_id: "N/A" labels: ["storage"] + requires: [vm, bare_metal] FieldExistsCheck: test_id: "N/A" labels: ["storage"] + requires: [vm, bare_metal] fields: ["volume_id", "operations"] CrudOperationsCheck: test_id: "DATASVC04-01" labels: ["min_req", "storage"] + requires: [vm, bare_metal] operations: ["stop", "start", "verify_attached", "verify_data"] # ─── Home directory storage ────────────────────────────────────── @@ -146,12 +158,15 @@ tests: DirectoryFilesystemQuotaCheck: test_id: "DIR01-01" labels: ["min_req", "storage"] + requires: [vm, bare_metal] DirectoryUsageAccountingCheck: test_id: "DIR01-02" labels: ["min_req", "storage"] + requires: [vm, bare_metal] DirectoryNfsAvailabilityCheck: test_id: "DIR02-01" labels: ["min_req", "storage"] + requires: [vm, bare_metal] # ─── High-speed storage: SDS controller ────────────────────────── sds_controller: @@ -159,18 +174,22 @@ tests: HssStorageProvisioningCheck: test_id: "HSS01-01" labels: ["min_req", "storage"] + requires: [] step: provision_storage HssQosThroughputCheck: test_id: "HSS02-01" labels: ["min_req", "storage"] + requires: [vm, bare_metal] step: qos_throughput HssNonDisruptiveUpgradeCheck: test_id: "HSS05-01" labels: ["min_req", "storage"] + requires: [] step: non_disruptive_upgrade HssRdmaMemoryProtectionCheck: test_id: "HSS06-01" labels: ["min_req", "storage"] + requires: [vm, bare_metal] step: rdma_memory_protection # ─── High-speed storage: parallel file system services ─────────── @@ -179,43 +198,271 @@ tests: HssParallelFsProvisioningCheck: test_id: "HSS07-01" labels: ["min_req", "storage"] + requires: [vm, bare_metal] step: provision_parallel_fs HssMultipleFilesystemsCheck: test_id: "HSS09-01" labels: ["min_req", "storage"] + requires: [] step: multiple_filesystems HssLiveExpansionCheck: test_id: "HSS10-01" labels: ["min_req", "storage"] + requires: [] step: live_expansion HssQuotaEnforcementCheck: test_id: "HSS12-01" labels: ["min_req", "storage"] + requires: [vm, bare_metal] step: quota_enforcement HssRootSquashCheck: test_id: "HSS13-01" labels: ["min_req", "storage"] + requires: [] step: root_squash HssFlockMountCheck: test_id: "HSS14-01" labels: ["min_req", "storage"] + requires: [vm, bare_metal] step: flock_mount HssChangelogAuditCheck: test_id: "HSS15-01" labels: ["min_req", "storage"] + requires: [vm, bare_metal] step: changelog_audit HssMultipathCheck: test_id: "HSS18-01" labels: ["min_req", "storage"] + requires: [vm, bare_metal] step: multipath - # ─── Teardown ──────────────────────────────────────────────────── + # Kubernetes storage checks use a normal provider-owned cluster fixture. + k8s_storage: + step: setup_cluster + checks: + K8sCsiStorageTypesCheck: + test_id: "K8S23-04" + labels: ["kubernetes", "min_req", "storage"] + requires: [kubernetes] + # StorageClass names by type; omit (or leave empty) to skip that type. + # Env overrides: K8S_CSI_BLOCK_SC, K8S_CSI_SHARED_FS_SC, K8S_CSI_NFS_SC. + block_storage_class: "{{ steps.setup_cluster.csi.block_storage_class | default('', true) }}" + shared_fs_storage_class: "{{ steps.setup_cluster.csi.shared_fs_storage_class | default('', true) }}" + nfs_storage_class: "{{ steps.setup_cluster.csi.nfs_storage_class | default('', true) }}" + pvc_size: "1Gi" + bind_timeout_s: 120 + namespace_prefix: "isvtest-csi-types" + K8sCsiStorageQuotaApiCheck: + test_id: "K8S23-07" + labels: ["kubernetes", "min_req", "storage"] + requires: [kubernetes] + # Uses the block StorageClass by default; skips entirely when unset. + # per_sc_quota must exceed pvc_request so the usage PVC fits, and + # over_quota_request must exceed per_sc_quota so admission rejects it. + storage_class: "{{ steps.setup_cluster.csi.block_storage_class | default('', true) }}" + total_quota: "10Gi" + per_sc_quota: "5Gi" + pvc_request: "1Gi" + over_quota_request: "100Gi" + bind_timeout_s: 120 + quota_settle_s: 30 + namespace_prefix: "isvtest-csi-quota" + K8sCsiTenantScopedCredentialsCheck: + test_id: "K8S23-06" + labels: ["kubernetes", "min_req", "storage"] + requires: [kubernetes] + # Pure read-only check: verifies CSI credentials are tenant-scoped + # by construction (Secret namespaces, labels/annotations, RBAC, + # and node-plugin volume topology). Skipped entirely when no + # CSIDriver objects exist. + csi_driver_namespaces: + - "kube-system" + allowed_workload_namespaces: [] + forbidden_labels: + - "shared-across-clusters=true" + K8sCsiProvisioningModesCheck: + test_id: "K8S23-05" + labels: ["kubernetes", "min_req", "storage"] + requires: [kubernetes] + # Exercises dynamic and static CSI provisioning end-to-end with a + # BusyBox mount + canary-file write/read. Skipped entirely when + # dynamic_storage_class is unset. The static subtest is skipped + # (not failed) when static_pv.volume_handle / csi_driver are + # unset, so providers that don't pre-provision a volume can still + # run the dynamic probe. + dynamic_storage_class: "{{ steps.setup_cluster.csi.block_storage_class | default('', true) }}" + dynamic_pvc_size: "1Gi" + static_pv: + volume_handle: "{{ steps.setup_cluster.csi.static_volume_handle | default('', true) }}" + csi_driver: "{{ steps.setup_cluster.csi.static_driver_name | default('', true) }}" + # Zonal block volumes (EBS/PD/Disk) must pin their consumer pod to + # the volume's AZ; empty for zone-agnostic backends. + zone: "{{ steps.setup_cluster.csi.static_volume_az | default('', true) }}" + fs_type: "ext4" + capacity: "1Gi" + access_mode: "ReadWriteOnce" + bind_timeout_s: 180 + namespace_prefix: "isvtest-csi-prov" + K8sCsiDriverHealthCheck: + test_id: "N/A" + labels: ["kubernetes", "min_req", "storage"] + requires: [kubernetes] + # Verifies each configured CSI driver is installed and healthy. + # Controller/node health (Deployment + DaemonSet) runs only + # when a spec supplies a `workloads` block; otherwise those subtests are + # skipped (driver registration is still checked). + # - storage_classes: ["..."] + # workloads: + # namespace: kube-system + # controller: { deployment: } + # node: { daemonset: } + drivers: + - storage_classes: + - "{{ steps.setup_cluster.csi.block_storage_class | default('', true) }}" + workloads: {} + - storage_classes: + - "{{ steps.setup_cluster.csi.shared_fs_storage_class | default('', true) }}" + workloads: {} + - storage_classes: + - "{{ steps.setup_cluster.csi.nfs_storage_class | default('', true) }}" + workloads: {} + min_controller_replicas: 1 + K8sCsiConcurrentPvcCheck: + test_id: "N/A" + labels: ["kubernetes", "min_req", "storage"] + requires: [kubernetes] + storage_class: "{{ steps.setup_cluster.csi.block_storage_class | default('', true) }}" + pvc_count: 2 + pvc_size: "1Gi" + bind_timeout_s: 120 + namespace_prefix: "isvtest-csi-concurrent" + K8sCsiPvcExpandCheck: + test_id: "N/A" + labels: ["kubernetes", "min_req", "storage"] + requires: [kubernetes] + storage_class: "{{ steps.setup_cluster.csi.block_storage_class | default('', true) }}" + initial_size: "1Gi" + expanded_size: "2Gi" + bind_timeout_s: 120 + expand_timeout_s: 180 + namespace_prefix: "isvtest-csi-expand" + K8sNodeKernelModulesCheck: + test_id: "HSS11-04" + labels: ["kubernetes", "storage"] + requires: [kubernetes] + kernel_modules: [] + node_selector: {} + image: "busybox:1.36" + bind_timeout_s: 60 + namespace_prefix: "isvtest-kmod" + K8sNfsMountOptionsCheck: + test_id: "HSS11-01" + labels: ["kubernetes", "storage"] + requires: [kubernetes] + # Env overrides: K8S_CSI_SHARED_FS_SC, K8S_CSI_NFS_SC. + shared_fs_storage_class: "{{ steps.setup_cluster.csi.shared_fs_storage_class | default('', true) }}" + nfs_storage_class: "{{ steps.setup_cluster.csi.nfs_storage_class | default('', true) }}" + image: "busybox:1.36" + node_selector: {} + pvc_size: "1Gi" + bind_timeout_s: 180 + namespace_prefix: "isvtest-fs-nfs" + # if expected values are not set, the check will be skipped + expected_version: "" + expected_nconnect: "" + expected_proto: "" + expected_read_ahead_kb: "" + + k8s_filesystem: + step: setup_cluster + checks: + # Shared-filesystem POSIX-semantics checks (multi-pod, RWX PVC). + K8sFileLockingCheck: + test_id: "HSS14-02" + labels: ["kubernetes", "storage"] + requires: [kubernetes] + shared_fs_storage_class: "{{ steps.setup_cluster.csi.shared_fs_storage_class | default('', true) }}" + nfs_storage_class: "{{ steps.setup_cluster.csi.nfs_storage_class | default('', true) }}" + image: "busybox:1.36" + node_selector: {} + pvc_size: "1Gi" + bind_timeout_s: 120 + release_timeout_s: 60 + namespace_prefix: "isvtest-fs-lock" + K8sCrossNodeWriteVisibilityCheck: + test_id: "HSS17-01" + labels: ["kubernetes", "storage"] + requires: [kubernetes] + shared_fs_storage_class: "{{ steps.setup_cluster.csi.shared_fs_storage_class | default('', true) }}" + nfs_storage_class: "{{ steps.setup_cluster.csi.nfs_storage_class | default('', true) }}" + image: "busybox:1.36" + node_selector: {} + pvc_size: "1Gi" + bind_timeout_s: 120 + visibility_window_s: 5.0 + namespace_prefix: "isvtest-fs-vis" + K8sCrossNodeAttrConsistencyCheck: + test_id: "HSS17-02" + labels: ["kubernetes", "storage"] + requires: [kubernetes] + shared_fs_storage_class: "{{ steps.setup_cluster.csi.shared_fs_storage_class | default('', true) }}" + nfs_storage_class: "{{ steps.setup_cluster.csi.nfs_storage_class | default('', true) }}" + image: "busybox:1.36" + node_selector: {} + pvc_size: "1Gi" + bind_timeout_s: 120 + attr_cache_window_s: 30.0 + extend_bytes: 1024 + namespace_prefix: "isvtest-fs-attr" + K8sLargeDirListingFilesCheck: + test_id: "HSS07-02" + labels: ["kubernetes", "storage"] + requires: [kubernetes] + shared_fs_storage_class: "{{ steps.setup_cluster.csi.shared_fs_storage_class | default('', true) }}" + nfs_storage_class: "{{ steps.setup_cluster.csi.nfs_storage_class | default('', true) }}" + image: "busybox:1.36" + node_selector: {} + files_count: 10000 + pvc_size: "10Gi" + bind_timeout_s: 120 + namespace_prefix: "isvtest-fs-bigdir" + timeout: 3600 + K8sLargeDirListingDirsCheck: + test_id: "HSS07-02" + labels: ["kubernetes", "storage"] + requires: [kubernetes] + shared_fs_storage_class: "{{ steps.setup_cluster.csi.shared_fs_storage_class | default('', true) }}" + nfs_storage_class: "{{ steps.setup_cluster.csi.nfs_storage_class | default('', true) }}" + image: "busybox:1.36" + node_selector: {} + dirs_count: 500 + pvc_size: "10Gi" + bind_timeout_s: 120 + namespace_prefix: "isvtest-fs-bigdir" + timeout: 3600 + K8sPosixComplianceCheck: + test_id: "DIR02-02" + labels: ["kubernetes", "slow", "storage"] + requires: [kubernetes] + shared_fs_storage_class: "{{ steps.setup_cluster.csi.shared_fs_storage_class | default('', true) }}" + nfs_storage_class: "{{ steps.setup_cluster.csi.nfs_storage_class | default('', true) }}" + image: "gcc:12" + node_selector: {} + pvc_size: "5Gi" + bind_timeout_s: 120 + build_timeout_s: 600 + # tests_subdir: "" # optional: scope to a single tests/ subdir (e.g. "chmod") + namespace_prefix: "isvtest-fs-posix" + timeout: 3600 + + # ─── Teardown tests ────────────────────────────────────────────── teardown_checks: step: teardown_volume checks: StepSuccessCheck: test_id: "N/A" labels: ["storage"] + requires: [vm, bare_metal] exclude: labels: [] diff --git a/isvctl/schemas/config.schema.json b/isvctl/schemas/config.schema.json index 9cba8929f..b48861131 100644 --- a/isvctl/schemas/config.schema.json +++ b/isvctl/schemas/config.schema.json @@ -62,7 +62,7 @@ }, "PlatformCommands": { "additionalProperties": true, - "description": "Lifecycle commands for a specific platform.\n\nGroups commands for a platform (kubernetes, slurm, bare_metal, network, vm, iam, image_registry, security).\nSupports skip at both platform level (skips all phases) and phase level.\n\nThe `phases` field defines the execution order. Steps are grouped by their `phase`\nfield and executed in the order defined by `phases`. Validations run after each phase.\n\nExample:\n ```yaml\n network:\n phases: [\"setup\", \"teardown\"] # Defines execution order\n steps:\n - name: create_vpc\n phase: setup\n command: \"./create_vpc.py\"\n - name: cleanup\n phase: teardown\n command: \"./teardown.py\"\n ```\n\nIf a step's phase is not in the phases list, an error is raised.\n\nSupports skip at both platform level (skips all phases) and step level.", + "description": "Lifecycle commands for a specific platform.\n\nGroups commands for a platform (kubernetes, slurm, bare_metal, network, vm, iam, image_registry, security, observability).\nSupports skip at both platform level (skips all phases) and phase level.\n\nThe `phases` field defines the execution order. Steps are grouped by their `phase`\nfield and executed in the order defined by `phases`. Validations run after each phase.\n\nExample:\n ```yaml\n network:\n phases: [\"setup\", \"test\", \"teardown\"] # Defines execution order\n steps:\n - name: create_vpc\n phase: setup\n command: \"./create_vpc.py\"\n - name: cleanup\n phase: teardown\n command: \"./teardown.py\"\n ```\n\nIf a step's phase is not in the phases list, an error is raised.\n\nSupports skip at both platform level (skips all phases) and step level.", "properties": { "skip": { "default": false, @@ -145,6 +145,22 @@ "title": "Skip", "type": "boolean" }, + "requires": { + "description": "Capability contexts allowed to run this step. Empty delegates capability gating to bound validations.", + "items": { + "type": "string" + }, + "title": "Requires", + "type": "array" + }, + "requires_available_validations": { + "description": "Validation names that must be available after release filtering for this step to run. Unreleased validations are available only when ISVTEST_INCLUDE_UNRELEASED=1.", + "items": { + "type": "string" + }, + "title": "Requires Available Validations", + "type": "array" + }, "continue_on_failure": { "default": false, "description": "Continue to next step even if this step fails", @@ -188,7 +204,7 @@ }, "ValidationConfig": { "additionalProperties": true, - "description": "Test configuration section.\n\nValidations are grouped by meaningful category names (e.g., 'network', 'ssh', 'gpu').\nEach validation can have a `phase` field to control execution timing:\n\n- No `phase`: Runs after setup steps complete (default)\n- `phase: teardown`: Runs after teardown steps complete\n- `phase: test`: Runs after test steps (if any exist)\n\nTwo formats are supported:\n\n1. List format (each validation specifies its own step/phase):\n validations:\n network:\n - VpcCrudCheck:\n step: vpc_crud\n - NetworkProvisionedCheck:\n step: create_network\n\n2. Group defaults format (step/phase apply to all checks):\n validations:\n credentials:\n step: test_credentials\n phase: test\n checks:\n - StepSuccessCheck: {}\n - FieldExistsCheck:\n field: \"account_id\"", + "description": "Test configuration section.\n\nValidations are grouped by meaningful category names (e.g., 'network', 'ssh', 'gpu').\nEach validation can have a `phase` field to control execution timing:\n\n- No `phase`: Runs after setup steps complete (default)\n- `phase: teardown`: Runs after teardown steps complete\n- `phase: test`: Runs after test steps (if any exist)\n\nThree formats are supported:\n\n1. List format (each validation specifies its own step/phase):\n validations:\n network:\n - VpcCrudCheck:\n step: vpc_crud\n - NetworkProvisionedCheck:\n step: create_network\n\n2. Group defaults with list checks (step/phase apply to all checks):\n validations:\n credentials:\n step: test_credentials\n phase: test\n checks:\n - StepSuccessCheck: {}\n - FieldExistsCheck:\n field: \"account_id\"\n\n3. Group defaults with dict checks (deep-merge friendly):\n validations:\n credentials:\n step: test_credentials\n checks:\n StepSuccessCheck: {}\n FieldExistsCheck:\n field: \"account_id\"", "properties": { "cluster_name": { "anyOf": [ @@ -226,7 +242,7 @@ } ], "default": null, - "description": "Platform: kubernetes, slurm, bare_metal, network, vm, iam, image_registry, security", + "description": "Capability a platform suite declares: vm, bare_metal, kubernetes, slurm. Plain suites omit it and gate each check with requires.", "title": "Platform" }, "settings": { @@ -291,7 +307,7 @@ "additionalProperties": { "$ref": "#/$defs/PlatformCommands" }, - "description": "Lifecycle commands by platform (kubernetes, slurm, bare_metal, network, vm, iam, image_registry, security)", + "description": "Lifecycle commands by platform (kubernetes, slurm, bare_metal, network, vm, iam, image_registry, security, observability)", "title": "Commands", "type": "object" }, diff --git a/isvctl/src/isvctl/cli/catalog.py b/isvctl/src/isvctl/cli/catalog.py index 9df35bf88..2d0790a44 100644 --- a/isvctl/src/isvctl/cli/catalog.py +++ b/isvctl/src/isvctl/cli/catalog.py @@ -86,20 +86,17 @@ def list_cmd( ) table.add_column("Test", style="green", no_wrap=True) table.add_column("Test IDs", style="magenta", max_width=32) - table.add_column("Labels (Platforms)", style="dim", max_width=40) + table.add_column("Suite / Requirements", style="dim", max_width=40) table.add_column("Description") for entry in sorted(catalog_entries, key=lambda e: e["name"]): - labels = ", ".join(entry.get("labels") or []) - platforms = ", ".join(entry.get("platforms") or []) - if labels and platforms: - labels_platforms = f"{labels} ({platforms})" - else: - labels_platforms = labels or platforms + suite = entry.get("suite") or "-" + requirement = entry.get("platform") or ", ".join(entry.get("requires") or []) or "core" + suite_requirement = f"{suite} / {requirement}" table.add_row( entry["name"], ", ".join(entry.get("test_ids") or []) or "-", - labels_platforms or "-", + suite_requirement, entry.get("description") or "-", ) @@ -172,9 +169,13 @@ def push( bool, typer.Option("--verbose", "-v", help="Enable verbose logging"), ] = False, - no_upload: Annotated[ + dry_run: Annotated[ bool, - typer.Option("--no-upload", help="Build and save locally without uploading"), + typer.Option( + "--dry-run", + "--no-upload", + help="Build and save locally without uploading", + ), ] = False, ) -> None: """Build the test catalog and upload it to ISV Lab Service. @@ -186,7 +187,7 @@ def push( Examples: isvctl catalog push - isvctl catalog push --no-upload + isvctl catalog push --dry-run """ setup_logging(verbose) @@ -201,8 +202,8 @@ def push( catalog_path.write_text(json.dumps(document, indent=2)) print_progress(f" Saved to: {catalog_path}") - if no_upload: - print_progress("Skipping upload (--no-upload)") + if dry_run: + print_progress("Dry run: saved catalog locally (upload skipped)") return from isvctl.reporting import check_upload_credentials, get_environment_config @@ -228,6 +229,7 @@ def push( entries=catalog_entries, schema_version=document["schemaVersion"], platforms=document["platforms"], + suites=document["suites"], ): print_progress(typer.style("[OK]", fg=typer.colors.GREEN) + " Catalog push complete") else: diff --git a/isvctl/src/isvctl/cli/deploy.py b/isvctl/src/isvctl/cli/deploy.py index 4d1ee8f6c..53552a2a5 100644 --- a/isvctl/src/isvctl/cli/deploy.py +++ b/isvctl/src/isvctl/cli/deploy.py @@ -32,6 +32,7 @@ from isvctl.cli import setup_logging from isvctl.cli.common import get_output_dir, print_error, print_progress, print_step, print_warning +from isvctl.config.suite_resolution import CONFIGS_ROOT, resolve_suite_name from isvctl.orchestrator.loop import Phase from isvctl.remote import SCPTransfer, SSHClient, TarArchive from isvctl.remote.archive import DEFAULT_EXCLUDES as DEFAULT_ARCHIVE_EXCLUDES @@ -396,6 +397,9 @@ def run( executed_by="isvctl deploy", ci_reference="local-deployment", isv_software_version=isv_software_version, + # deploy has no --capability of its own; the remote `test run` + # it invokes is core-only unless the config is a platform suite. + suite=resolve_suite_name(list(config_files), CONFIGS_ROOT), ) if not test_run_id: print_warning("Failed to create test run, continuing without upload") diff --git a/isvctl/src/isvctl/cli/test.py b/isvctl/src/isvctl/cli/test.py index 5b721c79c..ccd67d57b 100644 --- a/isvctl/src/isvctl/cli/test.py +++ b/isvctl/src/isvctl/cli/test.py @@ -27,7 +27,8 @@ import typer import yaml -from isvtest.catalog import build_catalog, get_catalog_version +from isvtest.catalog import build_catalog, catalog_document, get_catalog_version +from isvtest.core.resolution import parse_validations, requirements_satisfied from isvtest.release_manifest import load_released_test_filter from isvctl.cli import setup_logging @@ -47,12 +48,18 @@ ) from isvctl.config.merger import merge_yaml_files from isvctl.config.schema import RunConfig +from isvctl.config.suite_resolution import ( + CONFIGS_ROOT, + SuiteResolutionError, + parse_capability, + resolve_suite, + resolve_suite_name, +) from isvctl.orchestrator.loop import Orchestrator, Phase -from isvctl.redaction import redact_dict from isvctl.reporting import check_upload_credentials, create_test_run, get_environment_config, update_test_run logger = logging.getLogger(__name__) -CONFIGS_ROOT = Path(__file__).resolve().parents[3] / "configs" +CORE_REQUIREMENT_CONTEXT = "core" class TeeWriter: @@ -115,7 +122,90 @@ def _junitxml_for_discovered_config(junitxml: Path, match: ProviderConfigMatch, return junitxml.with_name(f"{junitxml.stem}-{match.config_path.stem}{junitxml.suffix}") -@app.command("run", context_settings={"allow_extra_args": True, "ignore_unknown_options": True}) +def _resolve_capability_context(config: RunConfig, capability: str | None, suite_label: str) -> str | None: + """Return the requirement context a run should execute under. + + One rule for every entry path: a plain suite with no ``--capability`` runs + its core checks. "Unfiltered" corresponds to no real ISV situation - nobody + runs on vm and kubernetes at once - so a plain suite always has a context. + Platform suites declare no ``requires:``, so they are left alone. + """ + if config.tests and config.tests.platform: + return capability + + if capability is None: + print_progress(f"No capability selected; running {suite_label!r} core checks.") + return CORE_REQUIREMENT_CONTEXT + + entries = parse_validations(config.tests.validations if config.tests else {}) + if not any(capability in entry.requires for entry in entries): + print_warning(f"No check in {suite_label!r} requires {capability}; running core checks only.") + return capability + + +def _reported_capability(config: RunConfig, capability_context: str | None) -> str | None: + """Return the capability to record on the uploaded test run. + + Two client-side spellings have to be translated before they leave the process. + ``CORE_REQUIREMENT_CONTEXT`` is this module's word for "no capability", which + the service records as NULL. A platform suite is left with no explicit context + because its own platform *is* the capability it runs under, so report that + rather than losing the axis for every platform-suite run. + """ + if capability_context == CORE_REQUIREMENT_CONTEXT: + return None + if capability_context is None and config.tests and config.tests.platform: + return config.tests.platform + return capability_context + + +def _human_readable_dry_run( + config: RunConfig, + capability: str | None, + include_labels: list[str] | None = None, + exclude_labels: list[str] | None = None, +) -> str: + """Render the validation requirement plan without executing lifecycle steps.""" + platform = config.tests.platform if config.tests and config.tests.platform else None + suite_type = f"platform ({platform})" if platform else "plain" + context = "not filtered" if capability is None else capability + selected_labels = set(include_labels or []) + rejected_labels = set(exclude_labels or []) + validations = config.tests.validations if config.tests else {} + entries = parse_validations(validations) + + lines = [ + "Dry-run plan", + f" Suite type: {suite_type}", + f" Capability: {context}", + f" Checks: {len(entries)}", + ] + if selected_labels: + lines.append(f" Labels: {', '.join(sorted(selected_labels))} (all required)") + if rejected_labels: + lines.append(f" Excluded labels: {', '.join(sorted(rejected_labels))}") + + for entry in entries: + if capability is not None and not requirements_satisfied(entry.requires, capability): + requirement = ", ".join(entry.requires) + lines.append(f" [SKIP] {entry.name}: requires {requirement} (context: {capability})") + elif missing_labels := sorted(selected_labels.difference(entry.labels)): + lines.append(f" [SKIP] {entry.name}: does not match all selected labels: {', '.join(missing_labels)}") + elif matched_labels := sorted(rejected_labels.intersection(entry.labels)): + lines.append(f" [SKIP] {entry.name}: excluded by label: {', '.join(matched_labels)}") + elif entry.requires: + lines.append(f" [RUN] {entry.name}: requires {', '.join(entry.requires)}") + else: + lines.append(f" [RUN] {entry.name}") + return "\n".join(lines) + + +@app.command( + "run", + # Allow pytest args after `--`, but reject unknown options before it so + # stale flags like `--platform` fail loudly instead of being forwarded. + context_settings={"allow_extra_args": True, "ignore_unknown_options": False}, +) def run( ctx: typer.Context, config_files: Annotated[ @@ -134,7 +224,24 @@ def run( str | None, typer.Option( "--provider", - help="Provider name for label discovery when no --config/-f files are supplied.", + help="Provider name for --suite selection or --label discovery when no --config/-f is supplied.", + ), + ] = None, + suite: Annotated[ + str | None, + typer.Option( + "--suite", + help="Run one platform or plain suite from the selected provider.", + ), + ] = None, + capability: Annotated[ + str | None, + typer.Option( + "--capability", + help=( + "Capability context used to filter check requirements (one of the platform suites). " + "Omit it and a plain suite runs its core checks -- the same rule for --suite, -f and --label." + ), ), ] = None, set_values: Annotated[ @@ -160,6 +267,13 @@ def run( help="Label to filter validations (can be repeated; all selected labels must match)", ), ] = None, + exclude_labels: Annotated[ + list[str] | None, + typer.Option( + "--exclude-label", + help="Exclude validations carrying this label (can be repeated).", + ), + ] = None, dry_run: Annotated[ bool, typer.Option( @@ -243,6 +357,9 @@ def run( Use -- to pass additional arguments to pytest/isvtest. Examples: + isvctl test run --provider aws --suite k8s + isvctl test run --provider aws --suite storage --label min_req + isvctl test run --provider aws --label network isvctl test run -f lab.yaml -f commands.yaml -f suites/k8s.yaml isvctl test run -f config.yaml --set context.node_count=8 isvctl test run -f config.yaml --phase setup @@ -252,12 +369,37 @@ def run( setup_logging(verbose) apply_user_config(no_user_config) + try: + capability_context = parse_capability(capability, CONFIGS_ROOT) + except SuiteResolutionError as exc: + print_error(str(exc)) + raise typer.Exit(code=1) + + suite_label: str | None = None + + if suite: + if not provider: + print_error("--suite requires --provider.") + raise typer.Exit(code=1) + if config_files: + print_error("--suite cannot be combined with --config/-f.") + raise typer.Exit(code=1) + try: + selected_suite = resolve_suite(provider, suite, configs_root=CONFIGS_ROOT) + except SuiteResolutionError as exc: + print_error(str(exc)) + raise typer.Exit(code=1) + print_progress(f"Selected {selected_suite.name!r} suite for provider {provider!r}.") + suite_label = selected_suite.name + config_files = [selected_suite.config_path] + provider = None + if provider: if config_files: print_error("--provider discovery cannot be combined with --config/-f.") raise typer.Exit(code=1) if not labels: - print_error("--provider requires at least one --label/-l for discovery.") + print_error("--provider requires either --suite NAME or at least one --label/-l.") raise typer.Exit(code=1) known_providers = list_providers(CONFIGS_ROOT) @@ -289,9 +431,12 @@ def run( ctx, config_files=[match.config_path], provider=None, + suite=None, + capability=capability, set_values=set_values, phase=phase, labels=labels, + exclude_labels=exclude_labels, dry_run=False, working_dir=working_dir, verbose=verbose, @@ -350,12 +495,19 @@ def run( print_error(f"Configuration validation failed: {e}") raise typer.Exit(code=1) + # Resolve the requirement context here rather than per entry path, so the + # same config behaves identically whether it was reached via --suite, -f, + # or --label discovery. Platform suites carry no `requires:`, so they keep + # the unfiltered context (filtering there would be a no-op anyway). + suite_name = suite_label or resolve_suite_name(config_files, CONFIGS_ROOT) + capability_context = _resolve_capability_context( + config, + capability_context, + suite_name or config_files[0].stem, + ) + if dry_run: - # Keep stdout as pure JSON so it can be consumed with `json.loads`; - # decorative headers and extra args go to stderr. - print_progress("\n--- Dry Run: Configuration ---") - redacted_config = redact_dict(config.model_dump(mode="json")) - typer.echo(json.dumps(redacted_config, indent=2)) + typer.echo(_human_readable_dry_run(config, capability_context, labels, exclude_labels)) if extra_pytest_args: print_progress(f"\n--- Extra pytest args ---\n{extra_pytest_args}") return @@ -400,13 +552,17 @@ def run( # Create test run before running tests if upload_results and lab_id: print_progress("Creating test run in ISV Lab Service...") - platform = config.tests.platform if config.tests and config.tests.platform else "kubernetes" + platform = ( + config.tests.platform if config.tests and config.tests.platform else next(iter(config.commands), "unknown") + ) test_run_id = create_test_run( lab_id=lab_id, platform=platform, tags=tags or ["validation-test", "isvctl"], start_time=start_time, isv_software_version=isv_software_version, + suite=suite_name, + capability=_reported_capability(config, capability_context), ) if not test_run_id: print_warning("Failed to create test run, continuing without upload") @@ -419,8 +575,7 @@ def run( # Build test catalog early so it runs inside the TeeWriter context # (avoids logging errors from stale stream references after the log file closes) - catalog_entries: list[dict] | None = None - catalog_version: str | None = None + test_catalog_document: dict[str, Any] | None = None # Always capture output to log file while still displaying (like `tee`) with open(log_file_path, "w") as log_file: @@ -432,6 +587,8 @@ def run( phases=phases, extra_pytest_args=extra_pytest_args, include_labels=labels, + exclude_labels=exclude_labels, + capability=capability_context, verbose=verbose, junitxml=str(junitxml), ) @@ -439,11 +596,10 @@ def run( try: catalog_entries = build_catalog() catalog_version = get_catalog_version() + test_catalog_document = catalog_document(catalog_entries, catalog_version) print_progress(f"Built test catalog: {len(catalog_entries)} tests (version: {catalog_version})") catalog_path = output_dir / "test_catalog.json" - catalog_path.write_text( - json.dumps({"isvTestVersion": catalog_version, "entries": catalog_entries}, indent=2) - ) + catalog_path.write_text(json.dumps(test_catalog_document, indent=2)) print_progress(f" Saved test catalog to: {catalog_path}") except Exception as e: logger.warning("Failed to build test catalog: %s", e) @@ -471,8 +627,7 @@ def run( junit_xml=junit_path if junit_path.exists() else None, log_file=log_file_path if log_file_path.exists() else None, isv_software_version=isv_software_version, - catalog_entries=catalog_entries, - catalog_version=catalog_version, + catalog_document=test_catalog_document, ): print_progress(typer.style("[OK]", fg=typer.colors.GREEN) + " Test results uploaded successfully") else: diff --git a/isvctl/src/isvctl/config/schema.py b/isvctl/src/isvctl/config/schema.py index 7ef1d56ed..bb8eb747d 100644 --- a/isvctl/src/isvctl/config/schema.py +++ b/isvctl/src/isvctl/config/schema.py @@ -24,7 +24,8 @@ from typing import Any -from pydantic import BaseModel, ConfigDict, Field +from isvtest.core.resolution import DECLARABLE_CAPABILITIES, parse_validations, requires_error +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator class LabConfig(BaseModel): @@ -86,6 +87,12 @@ class StepConfig(BaseModel): env: dict[str, str] = Field(default_factory=dict, description="Additional environment variables") working_dir: str | None = Field(default=None, description="Working directory for command execution") skip: bool = Field(default=False, description="Skip this step") + requires: list[str] = Field( + default_factory=list, + description=( + "Capability contexts allowed to run this step. Empty delegates capability gating to bound validations." + ), + ) requires_available_validations: list[str] = Field( default_factory=list, description=( @@ -107,6 +114,15 @@ class StepConfig(BaseModel): description="Additional argument patterns to mask in logs (e.g., ['--my-secret'])", ) + @field_validator("requires") + @classmethod + def validate_requires(cls, requires: list[str]) -> list[str]: + """Keep step requirements in the same vocabulary as validation requirements.""" + message = requires_error(requires) + if message: + raise ValueError(message) + return requires + class PlatformCommands(BaseModel): """Lifecycle commands for a specific platform. @@ -361,8 +377,8 @@ class ValidationConfig(BaseModel): platform: str | None = Field( default=None, description=( - "Platform type: KUBERNETES, SLURM, BARE_METAL, CONTROL_PLANE, IAM, NETWORK, " - "SECURITY, VM, IMAGE_REGISTRY, OBSERVABILITY, STORAGE" + "Capability a platform suite declares: vm, bare_metal, kubernetes, slurm. " + "Plain suites omit it and gate each check with requires." ), ) settings: dict[str, Any] = Field(default_factory=dict, description="Test settings") @@ -375,6 +391,28 @@ class ValidationConfig(BaseModel): description="Exclusion rules (keys: labels, platforms, tests, files)", ) + @model_validator(mode="after") + def validate_suite_shape(self) -> "ValidationConfig": + """Reject legacy axes and requirements on platform suite checks.""" + if self.model_extra and "module" in self.model_extra: + raise ValueError("tests.module is no longer supported; plain suites have no axis key") + if self.platform and self.platform not in DECLARABLE_CAPABILITIES: + raise ValueError(f"tests.platform must be one of: {', '.join(sorted(DECLARABLE_CAPABILITIES))}") + entries = parse_validations(self.validations) + if self.platform and any("requires" in entry.params_template for entry in entries): + raise ValueError("requires is not allowed in platform suites") + if any("platforms" in entry.params_template for entry in entries): + raise ValueError("per-check platforms is no longer supported; use requires in plain suites") + if not self.platform: + for entry in entries: + raw_requires = entry.params_template.get("requires") + if raw_requires is None: + continue + message = requires_error(raw_requires) + if message: + raise ValueError(message) + return self + class RunConfig(BaseModel): """Unified configuration schema for a complete test run. diff --git a/isvctl/src/isvctl/config/suite_resolution.py b/isvctl/src/isvctl/config/suite_resolution.py new file mode 100644 index 000000000..c45208f00 --- /dev/null +++ b/isvctl/src/isvctl/config/suite_resolution.py @@ -0,0 +1,170 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Resolve one platform or plain suite to a provider configuration.""" + +from dataclasses import dataclass +from functools import cache +from pathlib import Path +from typing import Any + +import yaml +from isvtest.core.resolution import canonical_suite_name as _normalize_name + +from isvctl.config.merger import merge_yaml_files + +# The canonical suite/provider config tree shipped with isvctl. Lives here +# rather than on a CLI command module so every entry point (test, deploy, ...) +# sources it from the config layer that consumes it. +CONFIGS_ROOT = Path(__file__).resolve().parents[3] / "configs" + + +class SuiteResolutionError(Exception): + """Raised when a suite selection cannot be resolved unambiguously.""" + + +@dataclass(frozen=True) +class ResolvedSuite: + """A provider configuration selected for one logical suite.""" + + config_path: Path + name: str + platform: str | None + + +@cache +def platform_vocabulary(configs_root: Path) -> frozenset[str]: + """Return declarable capabilities from canonical platform suite YAML. + + Cached: the suite directory is fixed for the life of a CLI invocation, and + several entry points ask for the vocabulary two or three times per run. + """ + platforms: set[str] = set() + for path in (configs_root / "suites").glob("*.yaml"): + try: + data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + except (OSError, yaml.YAMLError): + continue + platform = (data.get("tests") or {}).get("platform") if isinstance(data, dict) else None + if isinstance(platform, str) and platform: + platforms.add(_normalize_name(platform)) + return frozenset(platforms) + + +@cache +def suite_vocabulary(configs_root: Path) -> frozenset[str]: + """Return plain suite names declared by canonical suite YAML.""" + declarable = platform_vocabulary(configs_root) + names = {_normalize_name(path.stem) for path in (configs_root / "suites").glob("*.yaml")} + return frozenset(names - declarable) + + +def parse_capability(value: str | None, configs_root: Path) -> str | None: + """Parse the single capability context (one platform suite name). + + The four capabilities are mutually exclusive execution environments (you run + on kubernetes OR slurm OR vm OR bare_metal, never a combination), so this + takes exactly one value. + """ + if value is None: + return None + if "," in value: + raise SuiteResolutionError( + f"--capability takes a single platform (got {value!r}); the capabilities " + "are mutually exclusive execution environments, so only one runs at a time." + ) + capability = _normalize_name(value) + allowed = platform_vocabulary(configs_root) + if capability not in allowed: + raise SuiteResolutionError( + f"Unknown or non-declarable capability: {value}. " + f"Available capabilities: {', '.join(sorted(allowed)) or '(none)'}." + ) + return capability + + +def _raw_imports(config_path: Path) -> list[str]: + """Return import paths declared directly by a provider config.""" + try: + data: Any = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {} + except (OSError, yaml.YAMLError) as exc: + raise SuiteResolutionError(f"Failed to read {config_path}: {exc}") from exc + value = data.get("import") if isinstance(data, dict) else None + if isinstance(value, str): + return [value] + if isinstance(value, list): + return [item for item in value if isinstance(item, str)] + return [] + + +@cache +def _suite_name(config_path: Path, declarable: frozenset[str]) -> tuple[str, str | None]: + """Return the logical suite name and optional platform key for a config. + + Cached: classifying a config merges it with the suite it imports, and a + single ``isvctl test run`` asks for the same config's identity from both + ``resolve_suite`` and ``resolve_suite_name``. + """ + merged = merge_yaml_files([config_path]) + tests = merged.get("tests") or {} + platform = tests.get("platform") if isinstance(tests, dict) else None + if isinstance(platform, str) and _normalize_name(platform) in declarable: + normalized = _normalize_name(platform) + return normalized, normalized + + suite_imports = [Path(value).stem for value in _raw_imports(config_path) if "suites" in Path(value).parts] + if len(suite_imports) > 1: + raise SuiteResolutionError( + f"Config {config_path} imports multiple suites ({', '.join(suite_imports)}); use --config/-f." + ) + name = suite_imports[0] if suite_imports else config_path.stem + return _normalize_name(name), None + + +def resolve_suite_name(config_paths: list[Path], configs_root: Path) -> str | None: + """Return the suite name a set of ``-f`` configs resolves to. + + A run's identity is (suite, capability), so the suite has to be recoverable + from every entry path -- including ``-f lab.yaml -f commands.yaml + -f suites/k8s.yaml``, where the first config is not the suite. Classify each + config in order and take the first that names a known platform or plain + suite; fall back to the first config's stem when nothing matches, which is + the best available label for an ad-hoc config. + """ + if not config_paths: + return None + + declarable = platform_vocabulary(configs_root) + known = declarable | suite_vocabulary(configs_root) + for path in config_paths: + try: + name, _ = _suite_name(path, declarable) + except SuiteResolutionError: + continue + if name in known: + return name + return _normalize_name(config_paths[0].stem) + + +def resolve_suite(provider: str, suite: str, *, configs_root: Path) -> ResolvedSuite: + """Resolve exactly one provider config for a platform or plain suite.""" + config_dir = configs_root / "providers" / provider / "config" + if not config_dir.is_dir(): + raise SuiteResolutionError(f"Provider {provider!r} has no config directory at {config_dir}.") + + requested = _normalize_name(suite) + declarable = platform_vocabulary(configs_root) + classified = [(path, *_suite_name(path, declarable)) for path in sorted(config_dir.glob("*.yaml"))] + matches = [item for item in classified if item[1] == requested] + if not matches: + available = sorted({name for _, name, _ in classified}) + raise SuiteResolutionError( + f"Provider {provider!r} has no {requested!r} suite. Available suites: {', '.join(available) or '(none)'}." + ) + if len(matches) > 1: + paths = ", ".join(str(path) for path, _, _ in matches) + raise SuiteResolutionError( + f"Provider {provider!r} has multiple {requested!r} suite configs ({paths}). Disambiguate with --config/-f." + ) + path, name, platform = matches[0] + return ResolvedSuite(config_path=path, name=name, platform=platform) diff --git a/isvctl/src/isvctl/orchestrator/loop.py b/isvctl/src/isvctl/orchestrator/loop.py index eb36db07a..24c4d9e81 100644 --- a/isvctl/src/isvctl/orchestrator/loop.py +++ b/isvctl/src/isvctl/orchestrator/loop.py @@ -37,6 +37,7 @@ ValidationEntry, get_entry_phase, parse_validations, + requirements_satisfied, resolve_class_key, resolve_entries, ) @@ -334,6 +335,34 @@ def _apply_step_validation_gates(steps: list[Any], released_tests: set[str] | No return gated_steps +def _apply_capability_step_gates( + steps: list[Any], + validation_entries: list[ValidationEntry], + capability: str | None, +) -> list[Any]: + """Skip a step when its explicit or validation-derived requirements do not match.""" + if capability is None: + return steps + gated_steps: list[Any] = [] + for step in steps: + explicit_requires = getattr(step, "requires", []) + if explicit_requires and not requirements_satisfied(explicit_requires, capability): + logger.info( + "Skipping step '%s' because it requires capability: %s", + step.name, + ", ".join(explicit_requires), + ) + gated_steps.append(step.model_copy(update={"skip": True})) + continue + bound_entries = [entry for entry in validation_entries if entry.step == step.name] + if bound_entries and all(not requirements_satisfied(entry.requires, capability) for entry in bound_entries): + logger.info("Skipping step '%s' because all bound validations are capability-filtered", step.name) + gated_steps.append(step.model_copy(update={"skip": True})) + else: + gated_steps.append(step) + return gated_steps + + class Orchestrator: """Orchestrates the full test lifecycle using step-based execution. @@ -361,6 +390,8 @@ def __init__( self._results: list[PhaseResult] = [] self._extra_pytest_args: list[str] | None = None self._include_labels: list[str] = [] + self._exclude_labels: list[str] = [] + self._capability: str | None = None self._verbose: bool = False self._junitxml: str | None = None @@ -370,6 +401,8 @@ def run( teardown_on_failure: bool = True, extra_pytest_args: list[str] | None = None, include_labels: list[str] | None = None, + exclude_labels: list[str] | None = None, + capability: str | None = None, verbose: bool = False, junitxml: str | None = None, ) -> OrchestratorResult: @@ -383,6 +416,8 @@ def run( - `-m kubernetes`: Run only validations whose labels include "kubernetes" (labels are mirrored as pytest marks) include_labels: Labels that selected validations must all contain. + exclude_labels: Labels that selected validations must not contain. + capability: Single capability context used to filter validation requirements. verbose: Enable verbose output for validations junitxml: Path to write JUnit XML report for validations @@ -395,6 +430,8 @@ def run( self._results = [] self._extra_pytest_args = extra_pytest_args self._include_labels = include_labels or [] + self._exclude_labels = exclude_labels or [] + self._capability = capability self._verbose = verbose self._junitxml = junitxml @@ -472,6 +509,11 @@ def _run_steps_mode( logger.info(f"Including unreleased validations because {INCLUDE_UNRELEASED_ENV} is enabled") steps = _apply_step_validation_gates(steps, released_tests) + all_validations = {} + if self.config.tests and self.config.tests.validations: + all_validations = self.config.tests.validations + validation_entries = parse_validations(all_validations) + steps = _apply_capability_step_gates(steps, validation_entries, self._capability) logger.info(f"Configured phases: {config_phases}") @@ -503,10 +545,6 @@ def _run_steps_mode( step_phase = (step.phase or "setup").lower() self.context.set_step_phase(step.name, step_phase) - all_validations = {} - if self.config.tests and self.config.tests.validations: - all_validations = self.config.tests.validations - validation_entries = parse_validations(all_validations) resolved_validations_by_index: dict[int, ResolvedEntry] = {} exclude_labels: list[str] = [] @@ -517,7 +555,9 @@ def _run_steps_mode( skip_config_label_exclusions = bool(self._include_labels) or _has_explicit_pytest_selection( self._extra_pytest_args ) - resolution_exclude_labels = [] if skip_config_label_exclusions else exclude_labels + resolution_exclude_labels = set(self._exclude_labels) + if not skip_config_label_exclusions: + resolution_exclude_labels.update(exclude_labels) phase_results: list[PhaseResult] = [] overall_success = True @@ -598,7 +638,7 @@ def _run_steps_mode( phase_entries, requested_phase_names if Phase.ALL not in requested_phases else set(config_phases), set(self._include_labels), - set(resolution_exclude_labels), + resolution_exclude_labels, set(exclude_tests), released_tests, ) @@ -667,7 +707,7 @@ def _run_steps_mode( remaining_entries, requested_phase_names if Phase.ALL not in requested_phases else set(config_phases), set(self._include_labels), - set(resolution_exclude_labels), + resolution_exclude_labels, set(exclude_tests), released_tests, config_phases, @@ -792,6 +832,7 @@ def _resolve_validation_entries( exclude_tests=exclude_tests, released_tests=released_tests, render_context=self.context.get_accumulated_context(), + capability=self._capability, ) def _resolve_remaining_validation_entries( @@ -860,9 +901,10 @@ def _append_resolution_only_phase_results( def _detect_platform(self) -> str | None: """Detect platform from configuration. - Checks multiple locations for platform: + Checks execution identity without requiring a plain-suite axis key: 1. tests.platform (isvctl schema) 2. Root-level platform (legacy isvtest schema) + 3. The sole commands mapping key (plain suites) Returns: Platform string (e.g., 'kubernetes', 'slurm', 'bare_metal') or None @@ -875,6 +917,8 @@ def _detect_platform(self) -> str | None: # Fall back to root-level platform (legacy isvtest configs) elif hasattr(self.config, "model_extra") and self.config.model_extra: platform = self.config.model_extra.get("platform") + if not platform and len(self.config.commands) == 1: + platform = next(iter(self.config.commands)) if platform: # Normalize 'k8s' to 'kubernetes' diff --git a/isvctl/src/isvctl/reporting.py b/isvctl/src/isvctl/reporting.py index 6480b5f3b..bb64161d3 100644 --- a/isvctl/src/isvctl/reporting.py +++ b/isvctl/src/isvctl/reporting.py @@ -23,6 +23,7 @@ import os from datetime import UTC, datetime from pathlib import Path +from typing import Any from isvctl.redaction import redact_text @@ -79,6 +80,8 @@ def create_test_run( executed_by: str = "isvctl", ci_reference: str = "local-run", isv_software_version: str | None = None, + suite: str | None = None, + capability: str | None = None, ) -> str | None: """Create a test run in ISV Lab Service. @@ -90,6 +93,9 @@ def create_test_run( executed_by: Tool that executed the test ci_reference: CI/CD reference identifier isv_software_version: ISV software stack version (opaque string from ISV) + suite: Suite that was run (e.g. "network", "vm") + capability: Capability context resolved from ``--capability``; None for + a core-only run Returns: Test run ID if successful, None otherwise @@ -124,6 +130,8 @@ def create_test_run( start_time=start_time, isv_software_version=isv_software_version, isv_test_version=isv_test_version, + suite=suite, + capability=capability, ) return result.get("data", {}).get("testRunId") except SystemExit: @@ -144,8 +152,7 @@ def update_test_run( junit_xml: Path | None = None, log_content: str | None = None, isv_software_version: str | None = None, - catalog_entries: list[dict] | None = None, - catalog_version: str | None = None, + catalog_document: dict[str, Any] | None = None, ) -> bool: """Update a test run in ISV Lab Service. @@ -158,8 +165,7 @@ def update_test_run( junit_xml: Path to JUnit XML file (optional) log_content: Direct log content string (optional, alternative to log_file) isv_software_version: ISV software stack version (opaque string from ISV) - catalog_entries: Test catalog entries for coverage tracking (optional) - catalog_version: Test suite version for the catalog (optional) + catalog_document: Complete test catalog document for coverage tracking (optional) Returns: True if successful, False otherwise @@ -209,15 +215,18 @@ def update_test_run( jwt_token = get_jwt_token(ssa_issuer, client_id, client_secret) # Upload test catalog for coverage tracking (if provided) - if catalog_entries and catalog_version: + if catalog_document: try: from isvreporter.client import upload_test_catalog as client_upload_catalog client_upload_catalog( endpoint=endpoint, jwt_token=jwt_token, - isv_test_version=catalog_version, - entries=catalog_entries, + isv_test_version=catalog_document["isvTestVersion"], + entries=catalog_document["entries"], + schema_version=catalog_document["schemaVersion"], + platforms=catalog_document["platforms"], + suites=catalog_document["suites"], ) except SystemExit: logger.warning("Failed to upload test catalog to ISV Lab Service") diff --git a/isvctl/tests/providers/nico/test_nico_provider.py b/isvctl/tests/providers/nico/test_nico_provider.py index 06d7aa21c..cf9870d6c 100644 --- a/isvctl/tests/providers/nico/test_nico_provider.py +++ b/isvctl/tests/providers/nico/test_nico_provider.py @@ -461,11 +461,12 @@ def _assert_steps_use_nico_api_base(steps: dict[str, dict[str, Any]]) -> None: assert "{{nico_api_base}}" in step["args"] -def test_nico_control_plane_config_platform_matches_command_group() -> None: - """The orchestrator uses tests.platform to look up the control-plane commands group.""" +def test_nico_control_plane_plain_suite_has_one_command_group() -> None: + """A plain suite derives execution identity from its sole command group.""" merged, _steps = _merged_nico_config_steps("control-plane.yaml", "control_plane") - assert merged["tests"]["platform"] == "control_plane" + assert "platform" not in merged["tests"] + assert list(merged["commands"]) == ["control_plane"] def test_nico_control_plane_config_wires_api_health() -> None: @@ -551,11 +552,12 @@ def fake_forge_get( ] -def test_nico_iam_config_platform_matches_command_group() -> None: - """The orchestrator uses tests.platform to look up the IAM commands group.""" +def test_nico_iam_plain_suite_has_one_command_group() -> None: + """A plain suite derives execution identity from its sole command group.""" merged, _steps = _merged_nico_config_steps("iam.yaml", "iam") - assert merged["tests"]["platform"] == "iam" + assert "platform" not in merged["tests"] + assert list(merged["commands"]) == ["iam"] def test_nico_iam_config_wires_credential_readiness() -> None: @@ -836,11 +838,12 @@ def test_nico_instance_inventory_scripts_skip_when_site_has_no_instances( assert "No instances found" in describe_payload["skip_reason"] -def test_nico_network_config_platform_matches_command_group() -> None: - """The orchestrator uses tests.platform to look up the network commands group.""" +def test_nico_network_plain_suite_has_one_command_group() -> None: + """A plain suite derives execution identity from its sole command group.""" merged, _steps = _merged_nico_config_steps("network.yaml", "network") - assert merged["tests"]["platform"] == "network" + assert "platform" not in merged["tests"] + assert list(merged["commands"]) == ["network"] def test_nico_network_config_wires_network_inventory_probes() -> None: diff --git a/isvctl/tests/test_capability_step_gating.py b/isvctl/tests/test_capability_step_gating.py new file mode 100644 index 000000000..3adde0416 --- /dev/null +++ b/isvctl/tests/test_capability_step_gating.py @@ -0,0 +1,107 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Capability gating must leave every provider config internally consistent. + +Running a plain suite under one capability context skips two kinds of step: +those carrying an explicit ``requires:``, and those whose bound validations are +all requirement-filtered. Either way the step produces no output, so any step +that survives the gate must not depend on a skipped step's output without an +explicit ``default(...)`` - otherwise the orchestrator raises +``MissingStepRefError`` and silently abandons the cleanup that step owned. + +This walks every provider config rather than the three that regressed, so a new +suite inherits the guarantee instead of re-discovering it. +""" + +import re +from pathlib import Path +from typing import Any + +import pytest +from isvtest.core.resolution import ( + DECLARABLE_CAPABILITIES, + ValidationEntry, + parse_validations, +) + +from isvctl.cli.test import CORE_REQUIREMENT_CONTEXT +from isvctl.config.merger import merge_yaml_files +from isvctl.config.schema import RunConfig +from isvctl.orchestrator.loop import _apply_capability_step_gates + +CONFIGS_ROOT = Path(__file__).resolve().parents[1] / "configs" +PROVIDERS_ROOT = CONFIGS_ROOT / "providers" +# Every context a plain suite can be run under, including the core-only default +# that `--suite NAME` (no `--capability`) selects. +CONTEXTS = (CORE_REQUIREMENT_CONTEXT, *sorted(DECLARABLE_CAPABILITIES)) +STEP_REFERENCE = re.compile(r"steps\.([A-Za-z0-9_]+)") + + +def _plain_suite_configs() -> list[tuple[str, Path]]: + """Return (provider, config path) for every plain-suite provider config.""" + configs: list[tuple[str, Path]] = [] + # Discovered rather than listed so a new provider inherits the guarantee. + for config_dir in sorted(PROVIDERS_ROOT.glob("*/config")): + provider = config_dir.parent.name + for path in sorted(config_dir.glob("*.yaml")): + config = RunConfig.model_validate(merge_yaml_files([str(path)])) + # Platform suites carry no `requires:` on their checks, so nothing + # is ever gated inside them. + if config.tests and config.tests.platform: + continue + configs.append((provider, path)) + return configs + + +def _gated_step_names(steps: list[Any], entries: list[ValidationEntry], context: str) -> set[str]: + """Return the steps `_apply_capability_step_gates` skips in a context. + + Asks the real gate rather than restating its rules, so a change to what + gets skipped is exercised here instead of silently passing against a copy. + """ + gated = _apply_capability_step_gates(steps, entries, context) + return {after.name for before, after in zip(steps, gated, strict=True) if after.skip and not before.skip} + + +def _unguarded_references(value: Any) -> set[str]: + """Return step names referenced without a `default(...)` fallback.""" + referenced: set[str] = set() + if isinstance(value, str): + if "default(" in value: + return referenced + return set(STEP_REFERENCE.findall(value)) + if isinstance(value, dict): + for item in value.values(): + referenced |= _unguarded_references(item) + elif isinstance(value, list): + for item in value: + referenced |= _unguarded_references(item) + return referenced + + +@pytest.mark.parametrize(("provider", "config_path"), _plain_suite_configs(), ids=lambda v: getattr(v, "stem", v)) +@pytest.mark.parametrize("context", CONTEXTS) +def test_surviving_steps_never_depend_on_gated_steps(provider: str, config_path: Path, context: str) -> None: + """No step that survives capability gating may read a gated step's output.""" + config = RunConfig.model_validate(merge_yaml_files([str(config_path)])) + entries = parse_validations(config.tests.validations if config.tests else {}) + + violations: list[str] = [] + for platform_key in config.commands: + steps = config.get_steps(platform_key) + gated = _gated_step_names(steps, entries, context) + if not gated: + continue + for step in steps: + if step.name in gated: + continue + dangling = _unguarded_references(step.model_dump()) & gated + for missing in sorted(dangling): + violations.append(f"step '{step.name}' reads skipped step '{missing}'") + + assert not violations, ( + f"{provider}/{config_path.name} under context '{context}': " + + "; ".join(violations) + + ". Gate the step with `requires:` or give the reference a `default(...)`." + ) diff --git a/isvctl/tests/test_capacity_config.py b/isvctl/tests/test_capacity_config.py index b249804d9..8466513b8 100644 --- a/isvctl/tests/test_capacity_config.py +++ b/isvctl/tests/test_capacity_config.py @@ -24,6 +24,7 @@ def test_security_suite_defines_capacity_validations() -> None: "CapacityReservationGroupingCheck": { "test_id": "CAP04-01", "labels": ["bare_metal", "capacity", "min_req", "security"], + "requires": ["vm", "bare_metal"], "step": "capacity_reservation_grouping", "min_resources": "{{min_resources}}", } @@ -36,6 +37,7 @@ def test_security_suite_defines_capacity_validations() -> None: "CapacityTopologyBlockAtomicAllocationCheck": { "test_id": "CAP04-02", "labels": ["bare_metal", "capacity", "min_req", "security"], + "requires": ["vm", "bare_metal"], "min_resources": 2, } } diff --git a/isvctl/tests/test_catalog_cli.py b/isvctl/tests/test_catalog_cli.py index 1767819a2..9915ca1b1 100644 --- a/isvctl/tests/test_catalog_cli.py +++ b/isvctl/tests/test_catalog_cli.py @@ -16,8 +16,10 @@ """Unit tests for the catalog CLI subcommand.""" import json +from pathlib import Path from unittest.mock import patch +import pytest from typer.testing import CliRunner from isvctl.cli.catalog import app @@ -29,15 +31,19 @@ "name": "AlphaCheck", "description": "Alpha description", "labels": ["kubernetes"], - "module": "isvtest.validations.alpha", - "platforms": ["KUBERNETES"], + "source": "isvtest.validations.alpha", + "suite": "kubernetes", + "platform": "kubernetes", + "requires": [], }, { "name": "BetaCheck", "description": "", "labels": [], - "module": "isvtest.validations.beta", - "platforms": [], + "source": "isvtest.validations.beta", + "suite": "storage", + "platform": None, + "requires": ["vm", "bare_metal"], }, ] @@ -73,11 +79,11 @@ def test_catalog_list_json() -> None: assert result.exit_code == 0, result.output payload = json.loads(result.output) - assert payload["schemaVersion"] == 1 + assert payload["schemaVersion"] == 2 assert payload["isvTestVersion"] == "1.2.3" assert payload["entries"] == _FAKE_ENTRIES - # The platform axis is derived from the real configs and drives the UI matrix. - assert "KUBERNETES" in payload["platforms"] + assert "kubernetes" in payload["platforms"] + assert "storage" in payload["suites"] def test_catalog_labels_table() -> None: @@ -156,3 +162,23 @@ def test_catalog_list_unreleased_json() -> None: build_catalog.assert_called_once_with(released_only=False) payload = json.loads(result.output) assert payload["entries"] == [_FAKE_ENTRIES[1]] + + +@pytest.mark.parametrize("flag", ["--dry-run", "--no-upload"]) +def test_catalog_push_dry_run_saves_without_upload(flag: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """`catalog push --dry-run` / `--no-upload` saves locally and skips upload.""" + monkeypatch.setattr("isvctl.cli.catalog.get_output_dir", lambda: tmp_path) + with ( + patch("isvctl.cli.catalog.build_catalog", return_value=_FAKE_ENTRIES), + patch("isvctl.cli.catalog.get_catalog_version", return_value="1.2.3"), + patch("isvctl.reporting.check_upload_credentials") as check_creds, + ): + result = runner.invoke(app, ["push", flag]) + + assert result.exit_code == 0, result.output + check_creds.assert_not_called() + catalog_path = tmp_path / "test_catalog.json" + assert catalog_path.exists() + payload = json.loads(catalog_path.read_text(encoding="utf-8")) + assert payload["entries"] == _FAKE_ENTRIES + assert "Dry run: saved catalog locally" in result.output diff --git a/isvctl/tests/test_cli_streams.py b/isvctl/tests/test_cli_streams.py index 87fb93afe..a165f093d 100644 --- a/isvctl/tests/test_cli_streams.py +++ b/isvctl/tests/test_cli_streams.py @@ -38,8 +38,10 @@ "name": "AlphaCheck", "description": "Alpha description", "labels": ["kubernetes"], - "module": "isvtest.validations.alpha", - "platforms": ["KUBERNETES"], + "source": "isvtest.validations.alpha", + "suite": "kubernetes", + "platform": "kubernetes", + "requires": [], }, ] @@ -66,20 +68,18 @@ def _write_config(tmp_path: Path) -> Path: return config -def test_dry_run_stdout_is_pure_json(tmp_path: Path) -> None: - """`test run --dry-run` emits only JSON on stdout; progress goes to stderr.""" +def test_dry_run_stdout_is_human_readable(tmp_path: Path) -> None: + """`test run --dry-run` summarizes the execution plan for operators.""" config = _write_config(tmp_path) result = runner.invoke(test_cli.app, ["run", "-f", str(config), "--no-upload", "--dry-run"]) assert result.exit_code == 0, result.output - # stdout must be parseable JSON with nothing else mixed in. - payload = json.loads(result.stdout) - assert payload["tests"]["platform"] == "kubernetes" - # Progress lives on stderr, never on the machine-readable stdout stream. + assert "Dry-run plan" in result.stdout + assert "Suite type: platform (kubernetes)" in result.stdout + assert "Checks: 0" in result.stdout assert "Validating configuration" in result.stderr assert "Validating configuration" not in result.stdout - assert "--- Dry Run: Configuration ---" not in result.stdout def test_catalog_list_json_stdout_is_pure_json() -> None: diff --git a/isvctl/tests/test_merger.py b/isvctl/tests/test_merger.py index ce9bf919f..fd40701a6 100644 --- a/isvctl/tests/test_merger.py +++ b/isvctl/tests/test_merger.py @@ -437,23 +437,25 @@ def test_aws_iam_inherits_test_validations(self) -> None: assert "credentials" in validations assert "teardown_checks" in validations assert result["tests"]["cluster_name"] == "aws-iam-validation" - assert result["tests"]["platform"] == "iam" + assert "platform" not in result["tests"] + assert list(result["commands"]) == ["iam"] - def test_my_isv_observability_declares_raw_platform_for_report_upload(self) -> None: - """Raw observability config exposes platform for upload paths that skip imports.""" + def test_my_isv_observability_is_a_plain_suite(self) -> None: + """Plain-suite provider configs do not recreate a platform axis.""" config_path = self.CONFIGS_DIR / "providers" / "my-isv" / "config" / "observability.yaml" raw_config = yaml.safe_load(config_path.read_text()) or {} - assert raw_config.get("tests", {}).get("platform") == "observability" + assert "platform" not in raw_config.get("tests", {}) result = merge_yaml_files([config_path]) - assert result["tests"]["platform"] == "observability" + assert "platform" not in result["tests"] + assert list(result["commands"]) == ["observability"] def test_aws_observability_inherits_supported_validations(self) -> None: """AWS observability imports the canonical suite and wires supported steps.""" result = merge_yaml_files([self.CONFIGS_DIR / "providers" / "aws" / "config" / "observability.yaml"]) - assert result["tests"]["platform"] == "observability" + assert "platform" not in result["tests"] assert result["tests"]["cluster_name"] == "aws-observability-validation" steps = result["commands"]["observability"]["steps"] diff --git a/isvctl/tests/test_orchestrator_loop.py b/isvctl/tests/test_orchestrator_loop.py index 3dee004b9..edf691f9e 100644 --- a/isvctl/tests/test_orchestrator_loop.py +++ b/isvctl/tests/test_orchestrator_loop.py @@ -32,6 +32,7 @@ from isvctl.orchestrator.loop import ( Orchestrator, Phase, + _apply_capability_step_gates, _entries_missing_from_junit, _merge_junit_xmls, _write_terminal_junit_xml, @@ -62,6 +63,20 @@ """ +def test_explicit_step_requires_gate_unbound_lifecycle_steps() -> None: + """An unbound teardown step is skipped when its explicit capability does not match.""" + steps = [ + StepConfig(name="setup_cluster", command="setup", phase="setup", requires=["kubernetes"]), + StepConfig(name="teardown_cluster", command="teardown", phase="teardown", requires=["kubernetes"]), + ] + + vm_steps = _apply_capability_step_gates(steps, [], "vm") + kubernetes_steps = _apply_capability_step_gates(steps, [], "kubernetes") + + assert all(step.skip for step in vm_steps) + assert all(not step.skip for step in kubernetes_steps) + + def test_python_script_path_falls_back_to_current_working_directory( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, diff --git a/isvctl/tests/test_reporting.py b/isvctl/tests/test_reporting.py index 627254643..752202aab 100644 --- a/isvctl/tests/test_reporting.py +++ b/isvctl/tests/test_reporting.py @@ -16,12 +16,16 @@ """Tests for reporting module.""" import os +from typing import Any from unittest.mock import MagicMock, patch +from isvctl.cli.test import CORE_REQUIREMENT_CONTEXT, _reported_capability +from isvctl.config.schema import RunConfig from isvctl.reporting import ( check_upload_credentials, get_environment_config, get_isv_test_version, + update_test_run, ) @@ -124,3 +128,87 @@ def test_returns_none_on_import_error(self) -> None: result = get_isv_test_version() # Result depends on whether __version__ is available assert result is None or isinstance(result, str) + + +class TestUpdateTestRun: + """Tests for result and catalog upload orchestration.""" + + @patch("isvreporter.client.update_test_run") + @patch("isvreporter.client.upload_test_catalog") + @patch("isvreporter.auth.get_jwt_token", return_value="jwt-token") + @patch( + "isvctl.reporting.get_environment_config", return_value=("https://api.example.com", "https://ssa.example.com") + ) + @patch("isvctl.reporting.check_upload_credentials", return_value=(True, "client-id", "client-secret")) + def test_forwards_complete_catalog_document( + self, + _mock_credentials: MagicMock, + _mock_environment: MagicMock, + _mock_token: MagicMock, + mock_upload_catalog: MagicMock, + _mock_update_run: MagicMock, + ) -> None: + """Automatic result upload preserves all catalog envelope metadata.""" + document = { + "schemaVersion": 2, + "isvTestVersion": "1.2.3", + "platforms": ["kubernetes", "vm"], + "suites": ["storage", "iam"], + "entries": [{"name": "TestA"}], + } + + result = update_test_run( + lab_id=7, + test_run_id="run-123", + success=True, + start_time="2026-07-24T12:00:00Z", + catalog_document=document, + ) + + assert result is True + mock_upload_catalog.assert_called_once_with( + endpoint="https://api.example.com", + jwt_token="jwt-token", + isv_test_version="1.2.3", + entries=[{"name": "TestA"}], + schema_version=2, + platforms=["kubernetes", "vm"], + suites=["storage", "iam"], + ) + + +class TestReportedCapability: + """The capability recorded on a test run, translated out of client spellings. + + A run's signal is the (suite, capability) pair: `network` and + `network --capability vm` execute different checks, so the capability has to + survive the trip to the service in a form the service understands. + """ + + @staticmethod + def _config(platform: str | None) -> RunConfig: + """Return a minimal config, declaring `platform` only for a platform suite.""" + raw: dict[str, Any] = {"commands": {}, "tests": {"validations": {}}} + if platform: + raw["tests"]["platform"] = platform + return RunConfig.model_validate(raw) + + def test_core_context_is_reported_as_no_capability(self) -> None: + """`core` is this client's word for "no capability", not a fifth one.""" + config = self._config(None) + assert _reported_capability(config, CORE_REQUIREMENT_CONTEXT) is None + + def test_platform_suite_reports_its_own_platform(self) -> None: + """A platform suite gets no explicit context because its platform is one. + + Passing the resolved context straight through would record NULL for + every platform-suite run and lose the axis entirely. + """ + config = self._config("vm") + assert _reported_capability(config, None) == "vm" + + def test_explicit_capability_wins(self) -> None: + """An explicit `--capability` is reported as-is, whatever the suite declares.""" + config = self._config(None) + assert _reported_capability(config, "vm") == "vm" + assert _reported_capability(self._config("vm"), "kubernetes") == "kubernetes" diff --git a/isvctl/tests/test_schema.py b/isvctl/tests/test_schema.py index e462dd377..cac241ec1 100644 --- a/isvctl/tests/test_schema.py +++ b/isvctl/tests/test_schema.py @@ -75,6 +75,7 @@ def test_minimal_step(self) -> None: assert step.timeout == 300 assert step.phase == "setup" assert step.skip is False + assert step.requires == [] assert step.requires_available_validations == [] def test_full_step(self) -> None: @@ -88,6 +89,7 @@ def test_full_step(self) -> None: working_dir="/tmp", phase="setup", skip=False, + requires=["vm", "bare_metal"], requires_available_validations=["NewCheck"], continue_on_failure=True, output_schema="vpc", @@ -98,10 +100,18 @@ def test_full_step(self) -> None: assert step.timeout == 600 assert step.env == {"AWS_REGION": "us-west-2"} assert step.phase == "setup" + assert step.requires == ["vm", "bare_metal"] assert step.requires_available_validations == ["NewCheck"] assert step.continue_on_failure is True assert step.output_schema == "vpc" + def test_step_rejects_unknown_or_duplicate_requires(self) -> None: + """Step requirements use the declarable capability vocabulary.""" + with pytest.raises(ValidationError, match="requires must be a list containing only"): + StepConfig(name="setup", command="echo", requires=["compute"]) + with pytest.raises(ValidationError, match="requires must not contain duplicates"): + StepConfig(name="setup", command="echo", requires=["vm", "vm"]) + class TestCommandOutput: """Tests for CommandOutput model (setup command JSON output).""" diff --git a/isvctl/tests/test_suite_resolution.py b/isvctl/tests/test_suite_resolution.py new file mode 100644 index 000000000..4acee1990 --- /dev/null +++ b/isvctl/tests/test_suite_resolution.py @@ -0,0 +1,157 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for provider suite selection and capability parsing.""" + +from pathlib import Path + +import pytest +from isvtest.core.resolution import DECLARABLE_CAPABILITIES +from pydantic import ValidationError + +from isvctl.config.merger import merge_yaml_files +from isvctl.config.schema import RunConfig +from isvctl.config.suite_resolution import ( + SuiteResolutionError, + parse_capability, + resolve_suite, + resolve_suite_name, +) + +CONFIGS_ROOT = Path(__file__).resolve().parents[1] / "configs" + + +def _write_catalog(root: Path) -> None: + """Write one platform suite, one plain suite, and provider imports.""" + suites = root / "suites" + configs = root / "providers" / "acme" / "config" + suites.mkdir(parents=True) + configs.mkdir(parents=True) + (suites / "k8s.yaml").write_text("tests:\n platform: kubernetes\n validations: {}\n") + (suites / "storage.yaml").write_text("tests:\n validations: {}\n") + (configs / "eks.yaml").write_text("import: ../../../suites/k8s.yaml\ncommands: {}\n") + (configs / "storage.yaml").write_text("import: ../../../suites/storage.yaml\ncommands: {}\n") + + +def test_one_suite_flag_resolves_platform_and_plain_suites(tmp_path: Path) -> None: + """The same selector resolves both suite kinds by effective YAML identity.""" + _write_catalog(tmp_path) + + platform = resolve_suite("acme", "kubernetes", configs_root=tmp_path) + plain = resolve_suite("acme", "storage", configs_root=tmp_path) + + assert platform.config_path.name == "eks.yaml" + assert platform.platform == "kubernetes" + assert plain.config_path.name == "storage.yaml" + assert plain.platform is None + + +def test_capability_uses_catalog_vocabulary(tmp_path: Path) -> None: + """An unknown capability is rejected while omitted context disables filtering.""" + _write_catalog(tmp_path) + + assert parse_capability(None, tmp_path) is None + assert parse_capability("k8s", tmp_path) == "kubernetes" + with pytest.raises(SuiteResolutionError, match="non-declarable capability: compute"): + parse_capability("compute", tmp_path) + with pytest.raises(SuiteResolutionError, match="single platform"): + parse_capability("kubernetes,vm", tmp_path) + + +def test_platform_suites_reject_requires_and_unknown_platforms() -> None: + """Platform placement is its obligation; it cannot declare check requirements.""" + validation = {"sample": {"checks": {"PlainCheck": {"requires": []}}}} + + with pytest.raises(ValidationError, match="requires is not allowed in platform suites"): + RunConfig.model_validate({"tests": {"platform": "kubernetes", "validations": validation}}) + with pytest.raises(ValidationError, match=r"tests\.platform must be one of"): + RunConfig.model_validate({"tests": {"platform": "compute", "validations": {}}}) + + +def test_plain_suite_rejects_unknown_requires_vocabulary() -> None: + """`test validate` catches a bad `requires` value, not just a run does.""" + validation = {"sample": {"checks": {"PlainCheck": {"requires": ["compute"]}}}} + with pytest.raises(ValidationError, match="requires must be a list containing only"): + RunConfig.model_validate({"tests": {"validations": validation}}) + + +def test_plain_suite_rejects_duplicate_requires() -> None: + """Duplicate requirements are rejected at schema-validation time.""" + validation = {"sample": {"checks": {"PlainCheck": {"requires": ["vm", "vm"]}}}} + with pytest.raises(ValidationError, match="requires must not contain duplicates"): + RunConfig.model_validate({"tests": {"validations": validation}}) + + +def test_plain_suite_accepts_valid_requires() -> None: + """A well-formed requires list passes schema validation.""" + validation = {"sample": {"checks": {"PlainCheck": {"requires": ["vm", "bare_metal"]}}}} + RunConfig.model_validate({"tests": {"validations": validation}}) + + +@pytest.mark.parametrize("provider", ["aws", "my-isv"]) +def test_storage_cleanup_steps_have_explicit_capability_gates(provider: str) -> None: + """Destructive storage cleanup runs only in the context that owns its resources. + + Both providers carry the same gates: my-isv is the scaffold ISVs copy, so a + reference-only guarantee would teach the wrong lifecycle. + """ + config_path = CONFIGS_ROOT / "providers" / provider / "config" / "storage.yaml" + config = RunConfig.model_validate(merge_yaml_files([str(config_path)])) + steps = {step.name: step for step in config.get_steps("storage")} + + assert steps["teardown_volume"].requires == ["vm", "bare_metal"] + assert steps["teardown"].requires == ["vm", "bare_metal"] + assert steps["setup_cluster"].requires == ["kubernetes"] + assert steps["teardown_cluster"].requires == ["kubernetes"] + + +@pytest.mark.parametrize("capability", sorted(DECLARABLE_CAPABILITIES)) +def test_my_isv_scaffold_covers_every_declarable_capability(capability: str) -> None: + """Every capability an ISV can declare has a my-isv platform suite to copy. + + The UI emits `--suite ` against the default provider, so a + missing scaffold config turns a documented command into an error. + """ + resolved = resolve_suite("my-isv", capability, configs_root=CONFIGS_ROOT) + assert resolved.platform == capability + + +@pytest.mark.parametrize("provider", ["aws", "my-isv"]) +def test_storage_cluster_fixture_uses_its_own_output_contract(provider: str) -> None: + """The storage cluster fixture is not held to the platform `cluster` schema. + + `setup_cluster` auto-detects that schema by name, and it demands + cluster_name/node_count - inventory no storage check reads. + """ + config_path = CONFIGS_ROOT / "providers" / provider / "config" / "storage.yaml" + config = RunConfig.model_validate(merge_yaml_files([str(config_path)])) + steps = {step.name: step for step in config.get_steps("storage")} + + assert steps["setup_cluster"].output_schema == "generic" + + +def test_suite_name_survives_every_entry_path(tmp_path: Path) -> None: + """A run's suite must be recoverable from the configs, not just from --suite. + + `-f lab.yaml -f commands.yaml -f suites/k8s.yaml` is a documented entry + path, and there the first config is not the suite - taking its stem would + record the run against "lab". + """ + _write_catalog(tmp_path) + configs = tmp_path / "providers" / "acme" / "config" + (configs / "lab.yaml").write_text("context: {}\n") + + assert resolve_suite_name([configs / "eks.yaml"], tmp_path) == "kubernetes" + assert resolve_suite_name([configs / "storage.yaml"], tmp_path) == "storage" + assert resolve_suite_name([configs / "lab.yaml", configs / "eks.yaml"], tmp_path) == "kubernetes" + assert resolve_suite_name([tmp_path / "suites" / "k8s.yaml"], tmp_path) == "kubernetes" + + +def test_ad_hoc_config_falls_back_to_its_own_stem(tmp_path: Path) -> None: + """An unrecognized config still labels its run rather than recording nothing.""" + _write_catalog(tmp_path) + ad_hoc = tmp_path / "one-off.yaml" + ad_hoc.write_text("commands: {}\n") + + assert resolve_suite_name([ad_hoc], tmp_path) == "one_off" + assert resolve_suite_name([], tmp_path) is None diff --git a/isvctl/tests/test_test_cli_labels.py b/isvctl/tests/test_test_cli_labels.py index 701a23b78..e11e241ba 100644 --- a/isvctl/tests/test_test_cli_labels.py +++ b/isvctl/tests/test_test_cli_labels.py @@ -16,6 +16,7 @@ """Tests for isvctl test CLI label filtering.""" import json +import re from pathlib import Path from typing import Any, ClassVar @@ -27,6 +28,8 @@ runner = CliRunner() +_ANSI_ESCAPE = re.compile(r"\x1b\[[0-9;]*m") + def _write_config(tmp_path: Path) -> Path: """Write a minimal isvctl test config and return its path.""" @@ -62,8 +65,6 @@ def _write_provider_config(root: Path, provider: str, name: str, suite: str, pla {platform}: phases: [test] steps: [] -tests: - platform: {platform} """, encoding="utf-8", ) @@ -129,6 +130,49 @@ def test_test_run_forwards_label_filters(monkeypatch: pytest.MonkeyPatch, tmp_pa assert _FakeOrchestrator.captured["include_labels"] == ["gpu", "slow"] +def test_test_run_uploads_the_complete_catalog_document( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """The automatic result path forwards the same complete catalog it saves.""" + config = _write_config(tmp_path) + output_dir = tmp_path / "_output" + output_dir.mkdir() + document = { + "schemaVersion": 2, + "isvTestVersion": "1.2.3", + "platforms": ["kubernetes", "vm"], + "suites": ["storage"], + "entries": [{"name": "TestA"}], + } + captured: dict[str, Any] = {} + + monkeypatch.setattr(test_cli, "Orchestrator", _FakeOrchestrator) + monkeypatch.setattr(test_cli, "get_output_dir", lambda: output_dir) + monkeypatch.setattr(test_cli, "check_upload_credentials", lambda: (True, "client-id", "client-secret")) + monkeypatch.setattr( + test_cli, + "get_environment_config", + lambda: ("https://api.example.com", "https://ssa.example.com"), + ) + monkeypatch.setattr(test_cli, "create_test_run", lambda **_kwargs: "run-123") + monkeypatch.setattr(test_cli, "build_catalog", lambda: document["entries"]) + monkeypatch.setattr(test_cli, "get_catalog_version", lambda: document["isvTestVersion"]) + monkeypatch.setattr(test_cli, "catalog_document", lambda _entries, _version: document) + + def capture_update(**kwargs: Any) -> bool: + captured.update(kwargs) + return True + + monkeypatch.setattr(test_cli, "update_test_run", capture_update) + + result = runner.invoke(test_cli.app, ["run", "-f", str(config), "--lab-id", "7"]) + + assert result.exit_code == 0, result.output + assert captured["catalog_document"] == document + assert json.loads((output_dir / "test_catalog.json").read_text()) == document + + def test_short_l_flag_binds_to_label_not_lab_id(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: """`-l` is the short flag for `--label`, not the legacy `--lab-id`. @@ -224,7 +268,7 @@ def test_provider_label_discovery_dispatches_each_matching_config( result = runner.invoke(test_cli.app, ["run", "--provider", "aws", "--label", "network", "--no-upload"]) assert result.exit_code == 0, result.output - assert [call["platform"] for call in _FakeOrchestrator.calls] == ["network", "observability"] + assert [call["platform"] for call in _FakeOrchestrator.calls] == [None, None] assert [call["working_dir"] for call in _FakeOrchestrator.calls] == [ network_config.parent, observability_config.parent, @@ -258,3 +302,302 @@ def test_provider_label_discovery_dry_run_prints_plan_without_running( assert plan["labels"] == ["network"] assert [Path(item["config"]).name for item in plan["configs"]] == ["network.yaml", "observability.yaml"] assert [item["matched_checks"][0]["name"] for item in plan["configs"]] == ["NetworkCheck", "VpcFlowLogsCheck"] + + +def test_provider_without_suite_or_label_mentions_both_options(monkeypatch: pytest.MonkeyPatch) -> None: + """`--provider` alone tells the user to pick `--suite` or `--label`.""" + _FakeOrchestrator.calls = [] + monkeypatch.setattr(test_cli, "Orchestrator", _FakeOrchestrator) + + result = runner.invoke(test_cli.app, ["run", "--provider", "aws", "--dry-run", "--no-upload"]) + + assert result.exit_code == 1, result.output + assert "--suite NAME" in result.output + assert "--label/-l" in result.output + assert _FakeOrchestrator.calls == [] + + +def test_plain_suite_without_capability_defaults_to_core( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """A declared plain suite runs core checks unless a capability is selected.""" + configs_root = tmp_path / "configs" + suite_path = configs_root / "suites" / "storage.yaml" + suite_path.parent.mkdir(parents=True) + suite_path.write_text( + """\ +tests: + validations: + core: + checks: + CoreCheck: + test_id: "N/A" + labels: ["storage"] + requires: [] + vm: + checks: + VmCheck: + test_id: "N/A" + labels: ["storage"] + requires: [vm] +""", + encoding="utf-8", + ) + (configs_root / "suites" / "vm.yaml").write_text( + "tests:\n platform: vm\n validations: {}\n", + encoding="utf-8", + ) + _write_provider_config(configs_root, "aws", "storage.yaml", "storage.yaml", "storage") + monkeypatch.setattr(test_cli, "CONFIGS_ROOT", configs_root) + + core_result = runner.invoke( + test_cli.app, + ["run", "--provider", "aws", "--suite", "storage", "--dry-run", "--no-upload"], + ) + vm_result = runner.invoke( + test_cli.app, + [ + "run", + "--provider", + "aws", + "--suite", + "storage", + "--capability", + "vm", + "--dry-run", + "--no-upload", + ], + ) + + assert core_result.exit_code == 0, core_result.output + assert "Capability: core" in core_result.stdout + assert "[RUN] CoreCheck" in core_result.stdout + assert "[SKIP] VmCheck" in core_result.stdout + assert vm_result.exit_code == 0, vm_result.output + assert "Capability: vm" in vm_result.stdout + assert "[RUN] CoreCheck" in vm_result.stdout + assert "[RUN] VmCheck" in vm_result.stdout + + +def test_every_entry_path_resolves_the_same_capability_context( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """One config runs the same checks whether reached via --suite or -f. + + The context models what an ISV declared, which cannot depend on how the + config was named on the command line. + """ + configs_root = tmp_path / "configs" + suite_path = configs_root / "suites" / "storage.yaml" + suite_path.parent.mkdir(parents=True) + suite_path.write_text( + """\ +tests: + validations: + core: + checks: + CoreCheck: + test_id: "N/A" + labels: ["storage"] + requires: [] + vm: + checks: + VmCheck: + test_id: "N/A" + labels: ["storage"] + requires: [vm] +""", + encoding="utf-8", + ) + (configs_root / "suites" / "vm.yaml").write_text( + "tests:\n platform: vm\n validations: {}\n", + encoding="utf-8", + ) + provider_config = _write_provider_config(configs_root, "aws", "storage.yaml", "storage.yaml", "storage") + monkeypatch.setattr(test_cli, "CONFIGS_ROOT", configs_root) + + via_suite = runner.invoke( + test_cli.app, + ["run", "--provider", "aws", "--suite", "storage", "--dry-run", "--no-upload"], + ) + via_file = runner.invoke( + test_cli.app, + ["run", "-f", str(provider_config), "--dry-run", "--no-upload"], + ) + + assert via_suite.exit_code == 0, via_suite.output + assert via_file.exit_code == 0, via_file.output + for output in (via_suite.stdout, via_file.stdout): + assert "Capability: core" in output + assert "[RUN] CoreCheck" in output + assert "[SKIP] VmCheck" in output + + +def test_platform_suite_keeps_the_unfiltered_context( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """Platform suites declare no requires, so the core default must not apply.""" + configs_root = tmp_path / "configs" + suites = configs_root / "suites" + suites.mkdir(parents=True) + (suites / "vm.yaml").write_text( + """\ +tests: + platform: vm + validations: + vm: + checks: + PlatformCheck: + test_id: "N/A" + labels: ["vm"] +""", + encoding="utf-8", + ) + _write_provider_config(configs_root, "aws", "vm.yaml", "vm.yaml", "vm") + monkeypatch.setattr(test_cli, "CONFIGS_ROOT", configs_root) + + result = runner.invoke( + test_cli.app, + ["run", "--provider", "aws", "--suite", "vm", "--dry-run", "--no-upload"], + ) + + assert result.exit_code == 0, result.output + assert "Capability: not filtered" in result.stdout + assert "[RUN] PlatformCheck" in result.stdout + + +def test_capability_with_no_matching_check_warns( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """A capability no check in the suite requires is a likely mistake.""" + configs_root = tmp_path / "configs" + suite_path = configs_root / "suites" / "iam.yaml" + suite_path.parent.mkdir(parents=True) + suite_path.write_text( + """\ +tests: + validations: + iam: + checks: + CoreCheck: + test_id: "N/A" + labels: ["iam"] + requires: [] +""", + encoding="utf-8", + ) + (configs_root / "suites" / "kubernetes.yaml").write_text( + "tests:\n platform: kubernetes\n validations: {}\n", + encoding="utf-8", + ) + _write_provider_config(configs_root, "aws", "iam.yaml", "iam.yaml", "iam") + monkeypatch.setattr(test_cli, "CONFIGS_ROOT", configs_root) + + result = runner.invoke( + test_cli.app, + ["run", "--provider", "aws", "--suite", "iam", "--capability", "kubernetes", "--dry-run", "--no-upload"], + ) + + assert result.exit_code == 0, result.output + # print_warning goes to stderr; the dry-run body to stdout. + assert "No check in 'iam' requires kubernetes" in result.stderr + assert "[RUN] CoreCheck" in result.stdout + + +def test_suite_and_label_filters_compose( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """A suite selects one lifecycle while labels narrow its checks.""" + configs_root = tmp_path / "configs" + suite_path = configs_root / "suites" / "storage.yaml" + suite_path.parent.mkdir(parents=True) + suite_path.write_text( + """\ +tests: + validations: + storage: + checks: + FastCheck: + test_id: "N/A" + labels: ["storage"] + SlowCheck: + test_id: "N/A" + labels: ["storage", "slow"] + DestructiveSlowCheck: + test_id: "N/A" + labels: ["storage", "slow", "destructive"] +""", + encoding="utf-8", + ) + _write_provider_config(configs_root, "aws", "storage.yaml", "storage.yaml", "storage") + _FakeOrchestrator.captured = {} + _FakeOrchestrator.calls = [] + monkeypatch.setattr(test_cli, "CONFIGS_ROOT", configs_root) + monkeypatch.setattr(test_cli, "Orchestrator", _FakeOrchestrator) + + command = [ + "run", + "--provider", + "aws", + "--suite", + "storage", + "--label", + "slow", + "--exclude-label", + "destructive", + "--no-upload", + ] + run_result = runner.invoke(test_cli.app, command) + dry_run_result = runner.invoke(test_cli.app, [*command, "--dry-run"]) + + assert run_result.exit_code == 0, run_result.output + assert _FakeOrchestrator.captured["include_labels"] == ["slow"] + assert _FakeOrchestrator.captured["exclude_labels"] == ["destructive"] + assert dry_run_result.exit_code == 0, dry_run_result.output + assert "Labels: slow (all required)" in dry_run_result.stdout + assert "Excluded labels: destructive" in dry_run_result.stdout + assert "[SKIP] FastCheck: does not match all selected labels: slow" in dry_run_result.stdout + assert "[RUN] SlowCheck" in dry_run_result.stdout + assert "[SKIP] DestructiveSlowCheck: excluded by label: destructive" in dry_run_result.stdout + + +def test_unknown_option_before_separator_is_rejected(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """Stale flags like `--platform` fail before they can be forwarded to pytest.""" + config = _write_config(tmp_path) + _FakeOrchestrator.calls = [] + monkeypatch.setattr(test_cli, "Orchestrator", _FakeOrchestrator) + + result = runner.invoke( + test_cli.app, + ["run", "-f", str(config), "--platform", "k8s", "--no-upload", "--dry-run"], + ) + + # Typer forces rich styling under GITHUB_ACTIONS, which splices escape + # codes into the middle of the reported option name. + output = _ANSI_ESCAPE.sub("", result.output) + + assert result.exit_code != 0, output + assert "No such option" in output or "no such option" in output.lower() + assert "--platform" in output + assert _FakeOrchestrator.calls == [] + + +def test_pytest_args_after_separator_are_forwarded(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """Args after `--` still reach the orchestrator as pytest extras.""" + config = _write_config(tmp_path) + _FakeOrchestrator.captured = {} + _FakeOrchestrator.calls = [] + monkeypatch.setattr(test_cli, "Orchestrator", _FakeOrchestrator) + + result = runner.invoke( + test_cli.app, + ["run", "-f", str(config), "--no-upload", "--", "-k", "K8sNodeCountCheck"], + ) + + assert result.exit_code == 0, result.output + assert _FakeOrchestrator.captured["extra_pytest_args"] == ["-k", "K8sNodeCountCheck"] diff --git a/isvreporter/src/isvreporter/client.py b/isvreporter/src/isvreporter/client.py index cb58c5c3a..6765103f8 100644 --- a/isvreporter/src/isvreporter/client.py +++ b/isvreporter/src/isvreporter/client.py @@ -39,6 +39,8 @@ def create_test_run( start_time: str, isv_software_version: str | None = None, isv_test_version: str | None = None, + suite: str | None = None, + capability: str | None = None, ) -> dict[str, Any]: """ Create a new test run record. @@ -54,6 +56,10 @@ def create_test_run( start_time: Test run start time (ISO 8601 format) isv_software_version: ISV software stack version (opaque string from ISV) isv_test_version: ISV test tool version (e.g., "1.12.3") + suite: Suite that was run (e.g. "network", "vm") + capability: Capability context the suite ran under (e.g. "vm"); None + means the run was core-only, which is a different signal than the + same suite run under a capability Returns: API response dictionary containing test run ID @@ -75,6 +81,12 @@ def create_test_run( payload["isvSoftwareVersion"] = isv_software_version if isv_test_version: payload["isvTestVersion"] = isv_test_version + if suite: + payload["suite"] = suite + # Sent only when set: a core-only run carries no capability, and null is + # the signal for that rather than a sentinel value. + if capability: + payload["capability"] = capability headers = { "Content-Type": "application/json", @@ -88,6 +100,8 @@ def create_test_run( test_run_id = result["data"]["testRunId"] print("Test run created successfully") print(f" Test Run ID: {test_run_id}") + if suite: + print(f" Suite: {suite} ({capability or 'core only'})") print(f" URL: {endpoint}/v1/labs/{lab_id}/test-runs/{test_run_id}") # Save test run ID to file for later use in after_script @@ -282,8 +296,9 @@ def upload_test_catalog( isv_test_version: str, entries: list[dict[str, Any]], *, - schema_version: int = 1, - platforms: list[str] | None = None, + schema_version: int, + platforms: list[str], + suites: list[str], ) -> bool: """Upload test catalog for a suite version (idempotent per version). @@ -296,10 +311,10 @@ def upload_test_catalog( jwt_token: JWT access token isv_test_version: Test suite version string (e.g. "1.2.3") entries: List of catalog entry dicts with keys: - name, description, labels, module, platforms, test_ids + name, description, labels, source, suite, platform, requires, test_ids schema_version: Catalog document schema version. - platforms: Platform axis labels (e.g. ["KUBERNETES", "VM"]) - the - matrix columns; empty list when unknown. + platforms: Declarable capability vocabulary (platform suites). + suites: Plain suite names declared by the catalog. Returns: True if catalog was uploaded or already exists, False on error @@ -321,14 +336,17 @@ def upload_test_catalog( payload = { "schemaVersion": schema_version, "isvTestVersion": isv_test_version, - "platforms": platforms or [], + "platforms": platforms, + "suites": suites, "entries": [ { "name": e["name"], "description": e.get("description", ""), "labels": e.get("labels", []), - "module": e.get("module", ""), - "platforms": e.get("platforms", []), + "source": e.get("source", ""), + "suite": e.get("suite", ""), + "platform": e.get("platform"), + "requires": e.get("requires", []), "test_ids": e.get("test_ids", []), } for e in entries diff --git a/isvreporter/src/isvreporter/main.py b/isvreporter/src/isvreporter/main.py index d89f7dbc1..968c8264a 100644 --- a/isvreporter/src/isvreporter/main.py +++ b/isvreporter/src/isvreporter/main.py @@ -135,6 +135,20 @@ def create( help="ISV test tool version (e.g., '1.12.3')", ), ] = None, + suite: Annotated[ + str | None, + typer.Option( + "--suite", + help="Suite that was run (e.g. 'network', 'vm')", + ), + ] = None, + capability: Annotated[ + str | None, + typer.Option( + "--capability", + help="Capability context the suite ran under (e.g. 'vm'). Omit for a core-only run.", + ), + ] = None, ) -> None: """Create a new test run in ISV Lab Service. @@ -167,6 +181,8 @@ def create( start_time=start_time, isv_software_version=isv_software_version, isv_test_version=isv_test_version, + suite=suite, + capability=capability, ) @@ -300,6 +316,9 @@ def update( jwt_token=jwt_token, isv_test_version=catalog_version, entries=catalog_entries, + schema_version=catalog_data.get("schemaVersion", 1), + platforms=catalog_data.get("platforms", []), + suites=catalog_data.get("suites", []), ) except FileNotFoundError: typer.echo(f"Warning: Test catalog file not found: {test_catalog}", err=True) diff --git a/isvreporter/tests/test_catalog_upload.py b/isvreporter/tests/test_catalog_upload.py index fa09a411f..ab3f771fd 100644 --- a/isvreporter/tests/test_catalog_upload.py +++ b/isvreporter/tests/test_catalog_upload.py @@ -20,12 +20,24 @@ from unittest.mock import MagicMock, patch from urllib.error import HTTPError, URLError +import pytest + from isvreporter.client import upload_test_catalog class TestUploadTestCatalog: """Tests for upload_test_catalog function.""" + def test_requires_the_complete_catalog_envelope(self) -> None: + """Callers cannot silently upload a v2 catalog without its axes.""" + with pytest.raises(TypeError): + upload_test_catalog( + endpoint="https://api.example.com", + jwt_token="test-token", + isv_test_version="1.2.3", + entries=[{"name": "TestA"}], + ) + @patch("isvreporter.client.urlopen") def test_successful_upload(self, mock_urlopen: MagicMock) -> None: """Test successful catalog upload returns True.""" @@ -47,10 +59,21 @@ def test_successful_upload(self, mock_urlopen: MagicMock) -> None: "name": "TestA", "description": "Test A", "labels": ["k8s"], - "module": "mod.a", + "source": "mod.a", + "suite": "kubernetes", + "platform": "kubernetes", + "requires": [], "test_ids": ["K8S06-01"], }, - {"name": "TestB", "description": "Test B", "labels": [], "module": "mod.b"}, + { + "name": "TestB", + "description": "Test B", + "labels": [], + "source": "mod.b", + "suite": "storage", + "platform": None, + "requires": ["vm", "bare_metal"], + }, ] result = upload_test_catalog( @@ -58,6 +81,9 @@ def test_successful_upload(self, mock_urlopen: MagicMock) -> None: jwt_token="test-token", isv_test_version="1.2.3", entries=entries, + schema_version=2, + platforms=["kubernetes", "vm"], + suites=["storage"], ) assert result is True @@ -70,9 +96,9 @@ def test_successful_upload(self, mock_urlopen: MagicMock) -> None: payload = json.loads(request.data.decode()) assert payload["isvTestVersion"] == "1.2.3" - # Envelope defaults when axis metadata is not supplied. - assert payload["schemaVersion"] == 1 - assert payload["platforms"] == [] + assert payload["schemaVersion"] == 2 + assert payload["platforms"] == ["kubernetes", "vm"] + assert payload["suites"] == ["storage"] assert len(payload["entries"]) == 2 assert payload["entries"][0]["name"] == "TestA" assert payload["entries"][0]["labels"] == ["k8s"] @@ -95,6 +121,9 @@ def test_skips_upload_when_version_exists(self, mock_urlopen: MagicMock) -> None jwt_token="test-token", isv_test_version="1.2.3", entries=[{"name": "TestA"}], + schema_version=2, + platforms=["kubernetes"], + suites=["storage"], ) assert result is True @@ -116,6 +145,9 @@ def test_conflict_returns_true(self, mock_urlopen: MagicMock) -> None: jwt_token="test-token", isv_test_version="1.2.3", entries=[{"name": "TestA"}], + schema_version=2, + platforms=["kubernetes"], + suites=["storage"], ) assert result is True @@ -136,6 +168,9 @@ def test_server_error_returns_false(self, mock_urlopen: MagicMock) -> None: jwt_token="test-token", isv_test_version="1.2.3", entries=[{"name": "TestA"}], + schema_version=2, + platforms=["kubernetes"], + suites=["storage"], ) assert result is False @@ -150,6 +185,9 @@ def test_connection_error_returns_false(self, mock_urlopen: MagicMock) -> None: jwt_token="test-token", isv_test_version="1.2.3", entries=[{"name": "TestA"}], + schema_version=2, + platforms=["kubernetes"], + suites=["storage"], ) assert result is False @@ -170,6 +208,9 @@ def test_empty_optional_fields_use_defaults(self, mock_urlopen: MagicMock) -> No jwt_token="test-token", isv_test_version="1.0.0", entries=entries, + schema_version=2, + platforms=["kubernetes"], + suites=["storage"], ) call_args = mock_urlopen.call_args @@ -181,12 +222,15 @@ def test_empty_optional_fields_use_defaults(self, mock_urlopen: MagicMock) -> No assert entry["description"] == "" assert entry["labels"] == [] assert "markers" not in entry - assert entry["module"] == "" + assert entry["source"] == "" + assert entry["suite"] == "" + assert entry["platform"] is None + assert entry["requires"] == [] assert entry["test_ids"] == [] @patch("isvreporter.client.urlopen") - def test_forwards_platform_axis_metadata(self, mock_urlopen: MagicMock) -> None: - """schema_version/platforms are sent in the top-level envelope.""" + def test_forwards_catalog_axis_vocabulary(self, mock_urlopen: MagicMock) -> None: + """Schema version and catalog axis lists are sent in the envelope.""" get_response = MagicMock() get_response.read.return_value = json.dumps([]).encode() get_response.__enter__ = MagicMock(return_value=get_response) @@ -202,14 +246,16 @@ def test_forwards_platform_axis_metadata(self, mock_urlopen: MagicMock) -> None: jwt_token="test-token", isv_test_version="1.2.3", entries=[{"name": "TestA"}], - schema_version=1, - platforms=["KUBERNETES", "VM"], + schema_version=2, + platforms=["kubernetes", "vm"], + suites=["storage", "iam"], ) request = mock_urlopen.call_args_list[1][0][0] payload = json.loads(request.data.decode()) - assert payload["schemaVersion"] == 1 - assert payload["platforms"] == ["KUBERNETES", "VM"] + assert payload["schemaVersion"] == 2 + assert payload["platforms"] == ["kubernetes", "vm"] + assert payload["suites"] == ["storage", "iam"] @patch("isvreporter.client.urlopen") def test_markers_field_is_not_forwarded(self, mock_urlopen: MagicMock) -> None: @@ -225,6 +271,9 @@ def test_markers_field_is_not_forwarded(self, mock_urlopen: MagicMock) -> None: jwt_token="test-token", isv_test_version="1.0.0", entries=[{"name": "TestA", "labels": ["gpu"], "markers": ["gpu"]}], + schema_version=2, + platforms=["kubernetes"], + suites=["storage"], ) request = mock_urlopen.call_args[0][0] diff --git a/isvreporter/tests/test_client.py b/isvreporter/tests/test_client.py index b24529b5e..d2b4c55fb 100644 --- a/isvreporter/tests/test_client.py +++ b/isvreporter/tests/test_client.py @@ -15,10 +15,12 @@ """Tests for the ISV Lab Service API client.""" +import json from pathlib import Path -from unittest.mock import patch +from typing import Any +from unittest.mock import MagicMock, patch -from isvreporter.client import calculate_duration, load_test_run_id +from isvreporter.client import calculate_duration, create_test_run, load_test_run_id class TestCalculateDuration: @@ -88,3 +90,78 @@ def test_load_empty_test_run_id(self, tmp_path: Path) -> None: with patch("isvreporter.client.TEST_RUN_ID_FILE", test_run_file): result = load_test_run_id() assert result == "" + + +class TestCreateTestRunPayload: + """The create payload has to carry both axes of what the run exercised.""" + + @staticmethod + def _posted_payload(mock_urlopen: MagicMock) -> dict[str, Any]: + """Return the JSON body of the request the client posted.""" + request = mock_urlopen.call_args[0][0] + return json.loads(request.data.decode()) + + @staticmethod + def _response() -> MagicMock: + """Return a context-manager response yielding a created test run id.""" + response = MagicMock() + response.read.return_value = json.dumps({"data": {"testRunId": 42}}).encode() + response.__enter__ = lambda self: self + response.__exit__ = lambda *args: False + return response + + @patch("isvreporter.client.urlopen") + def test_sends_suite_and_capability(self, mock_urlopen: MagicMock, tmp_path: Path) -> None: + """Both axes of the run reach the service in the create payload.""" + mock_urlopen.return_value = self._response() + + with ( + patch("isvreporter.client.OUTPUT_DIR", tmp_path), + patch("isvreporter.client.TEST_RUN_ID_FILE", tmp_path / "testrun_id.txt"), + ): + create_test_run( + endpoint="https://api.example.com", + lab_id=1, + jwt_token="jwt", + test_target_type="VM", + tags=[], + executed_by="isvctl", + ci_reference="local-run", + start_time="2026-07-24T12:00:00Z", + suite="network", + capability="vm", + ) + + payload = self._posted_payload(mock_urlopen) + assert payload["suite"] == "network" + assert payload["capability"] == "vm" + + @patch("isvreporter.client.urlopen") + def test_core_only_run_omits_capability(self, mock_urlopen: MagicMock, tmp_path: Path) -> None: + """A core-only run sends no capability - the absence is the signal. + + Sending a sentinel instead would make it indistinguishable from a real + capability in every downstream filter and column. + """ + mock_urlopen.return_value = self._response() + + with ( + patch("isvreporter.client.OUTPUT_DIR", tmp_path), + patch("isvreporter.client.TEST_RUN_ID_FILE", tmp_path / "testrun_id.txt"), + ): + create_test_run( + endpoint="https://api.example.com", + lab_id=1, + jwt_token="jwt", + test_target_type="NETWORK", + tags=[], + executed_by="isvctl", + ci_reference="local-run", + start_time="2026-07-24T12:00:00Z", + suite="network", + capability=None, + ) + + payload = self._posted_payload(mock_urlopen) + assert payload["suite"] == "network" + assert "capability" not in payload diff --git a/isvtest/src/isvtest/catalog.py b/isvtest/src/isvtest/catalog.py index 25d5a7474..d553981dd 100644 --- a/isvtest/src/isvtest/catalog.py +++ b/isvtest/src/isvtest/catalog.py @@ -19,16 +19,12 @@ discover_all_tests() and serializing each BaseValidation subclass's metadata. The catalog is version-keyed by the installed isvtest package version. -Platform tagging uses two sources (union of both): - 1. Config files - which checks appear in each isvctl/configs/suites/*.yaml - 2. Wiring labels - e.g. a check wired with labels: [bare_metal] implies the - BARE_METAL platform - -This ensures checks get a platform badge in the UI even when they only appear -in provider configs (e.g. Bm* checks that run on-host, not via SSH). +Suite placement and capability requirements come only from canonical +``isvctl/configs/suites/*.yaml`` wiring. """ import logging +import os from collections.abc import Callable, Iterator from pathlib import Path from typing import Any @@ -36,8 +32,8 @@ import yaml from isvreporter.version import get_version -from isvtest.catalog_platforms import LABEL_TO_PLATFORM, PLATFORM_CONFIGS from isvtest.core.discovery import discover_all_tests +from isvtest.core.resolution import DECLARABLE_CAPABILITIES, canonical_suite_name, resolve_class_key from isvtest.release_manifest import INCLUDE_UNRELEASED_ENV, load_released_test_filter logger = logging.getLogger(__name__) @@ -45,7 +41,7 @@ # Version of the catalog document envelope (schemaVersion field), bumped only # when the top-level shape changes - independent of the isvtest package version # (isvTestVersion), which tracks the test content. -CATALOG_SCHEMA_VERSION = 1 +CATALOG_SCHEMA_VERSION = 2 def _find_configs_dir() -> Path | None: @@ -72,29 +68,40 @@ def iter_config_checks(config_path: Path) -> Iterator[tuple[str, dict[str, Any]] data = yaml.safe_load(config_path.read_text()) except Exception: return + for _, name, params in iter_checks_from_data(data): + yield name, params + +def iter_checks_from_data(data: Any) -> Iterator[tuple[str, str, dict[str, Any]]]: + """Yield ``(category, check_name, params)`` from an already-parsed document. + + Lets callers that have parsed the YAML for other reasons avoid a second + read and parse of the same file. This is the only walker of the wiring + shape - the catalog, the suite map, and the wiring validator all read it + here, so a new nesting form is taught to the codebase once. + """ validations = (data or {}).get("tests", {}).get("validations", {}) if not isinstance(validations, dict): return - def _from_mapping(mapping: Any) -> Iterator[tuple[str, dict[str, Any]]]: + def _from_mapping(category: str, mapping: Any) -> Iterator[tuple[str, str, dict[str, Any]]]: if isinstance(mapping, dict): for name, params in mapping.items(): - yield name, params if isinstance(params, dict) else {} + yield category, name, params if isinstance(params, dict) else {} - for cat_config in validations.values(): + for category, cat_config in validations.items(): if isinstance(cat_config, dict) and "checks" in cat_config: checks_val = cat_config["checks"] if isinstance(checks_val, dict): - yield from _from_mapping(checks_val) + yield from _from_mapping(category, checks_val) elif isinstance(checks_val, list): for check in checks_val: - yield from _from_mapping(check) + yield from _from_mapping(category, check) elif isinstance(cat_config, dict): - yield from _from_mapping(cat_config) + yield from _from_mapping(category, cat_config) elif isinstance(cat_config, list): for check in cat_config: - yield from _from_mapping(check) + yield from _from_mapping(category, check) def _extract_checks_from_config(config_path: Path) -> list[str]: @@ -196,42 +203,51 @@ def build_label_file_map() -> dict[str, set[str]]: return label_files -def _build_platform_map() -> dict[str, set[str]]: - """Build a mapping from test name to set of platform strings. - - Scans the canonical config files to determine which tests belong to - which platforms. - """ +def _iter_suite_docs() -> Iterator[tuple[Path, dict[str, Any]]]: + """Yield ``(path, document)`` for each canonical suite YAML, parsed once.""" configs_dir = _find_configs_dir() if not configs_dir: logger.warning("Could not locate isvctl/configs/ directory") - return {} + return + for config_path in sorted((configs_dir / "suites").glob("*.yaml")): + yield config_path, yaml.safe_load(config_path.read_text(encoding="utf-8")) or {} + - test_to_platforms: dict[str, set[str]] = {} +def _declared_platform(data: dict[str, Any]) -> str | None: + """Return the platform key a suite document declares, or None for a plain suite.""" + tests = data.get("tests") or {} + platform = tests.get("platform") if isinstance(tests, dict) else None + return platform if isinstance(platform, str) and platform else None - for platform, config_files in PLATFORM_CONFIGS.items(): - for config_file in config_files: - config_path = configs_dir / config_file - if not config_path.exists(): - logger.debug("Config not found: %s", config_path) - continue - checks = _extract_checks_from_config(config_path) - for check_name in checks: - if check_name not in test_to_platforms: - test_to_platforms[check_name] = set() - test_to_platforms[check_name].add(platform) +def _build_suite_map() -> dict[str, dict[str, Any]]: + """Map suite wiring names to suite placement and requirements. - return test_to_platforms + Duplicate wiring names currently last-wins. Global uniqueness enforcement + is deferred to a follow-up PR (``ISVCTL_ENFORCE_UNIQUE_WIRING=1``). + """ + enforce_unique = os.environ.get("ISVCTL_ENFORCE_UNIQUE_WIRING") == "1" + suite_map: dict[str, dict[str, Any]] = {} + for config_path, data in _iter_suite_docs(): + platform = _declared_platform(data) + suite = platform if platform else canonical_suite_name(config_path.stem) + for _, check_name, params in iter_checks_from_data(data): + if check_name in suite_map and enforce_unique: + raise ValueError(f"Suite wiring name {check_name!r} is not globally unique") + requires = params.get("requires", []) + suite_map[check_name] = { + "suite": suite, + "platform": platform, + "requires": list(requires) if isinstance(requires, list) else [], + } + return suite_map def build_catalog(*, released_only: bool = True) -> list[dict[str, Any]]: """Discover all validation tests and return structured catalog entries. - Each entry includes a 'platforms' field derived from the config files, - indicating which platforms the test belongs to. Variant entries from - configs (e.g. K8sNimHelmWorkload-1b) are included as separate entries - inheriting metadata from their base class. + Each entry is one suite wiring name. Plain suites carry ``requires`` while + platform suites carry their ``platform`` key. Args: released_only: When True, omit tests that are not in the committed @@ -244,10 +260,12 @@ def build_catalog(*, released_only: bool = True) -> list[dict[str, Any]]: - labels: List of public label strings (e.g. ["kubernetes", "gpu"]) - test_ids: List of test-plan ids declared on the wiring, "N/A" excluded (e.g. ["SEC07-01"]); empty when only intentional gaps - - module: Fully qualified module path - - platforms: List of platform strings (e.g. ["KUBERNETES"]) + - source: Fully qualified Python source module + - suite: Logical suite name + - platform: Declared platform key for platform suites, otherwise null + - requires: Capability prerequisites for plain suites """ - platform_map = _build_platform_map() + suite_map = _build_suite_map() label_map = build_label_map() test_id_map = build_test_id_map() @@ -262,44 +280,19 @@ def build_catalog(*, released_only: bool = True) -> list[dict[str, Any]]: class_meta[cls.__name__] = { "description": getattr(cls, "description", "") or "", "labels": labels, - "module": cls.__module__, + "source": cls.__module__, } - # Infer platforms from labels only for checks not already covered by - # canonical configs. Some labels (for example "security") are useful - # pytest filters but are not reliable platform ownership signals once a - # check appears in a suite file. - if cls.__name__ not in platform_map: - for label in labels: - platform = LABEL_TO_PLATFORM.get(label) - if platform: - platform_map.setdefault(cls.__name__, set()).add(platform) catalog: list[dict[str, Any]] = [] - seen: set[str] = set() - - # Add all discovered classes - for name, meta in class_meta.items(): - seen.add(name) - catalog.append( - { - "name": name, - "description": meta["description"], - "labels": meta["labels"], - "test_ids": sorted(test_id_map.get(name, set())), - "module": meta["module"], - "platforms": sorted(platform_map.get(name, [])), - } - ) - # Add variant entries from configs that aren't base classes - for name, platforms in platform_map.items(): - if name in seen: + for name, placement in suite_map.items(): + base = resolve_class_key(name, class_meta) + if base is None: + logger.warning("Omitting suite wiring %s because no validation class resolves it", name) continue - base = name.split("-")[0] if "-" in name else name if name in excluded_names or base in excluded_names: continue - seen.add(name) - meta = class_meta.get(base, {}) + meta = class_meta[base] variant_suffix = name[len(base) :] if base != name else "" desc = meta.get("description", "") if variant_suffix: @@ -312,8 +305,8 @@ def build_catalog(*, released_only: bool = True) -> list[dict[str, Any]]: "description": desc, "labels": labels, "test_ids": test_ids, - "module": meta.get("module", ""), - "platforms": sorted(platforms), + "source": meta.get("source", ""), + **placement, } ) @@ -322,8 +315,10 @@ def build_catalog(*, released_only: bool = True) -> list[dict[str, Any]]: if released_tests is None: logger.info("Including unreleased tests in catalog because %s is enabled", INCLUDE_UNRELEASED_ENV) else: - omitted_names = sorted(entry["name"] for entry in catalog if entry["name"] not in released_tests) - catalog = [entry for entry in catalog if entry["name"] in released_tests] + omitted_names = sorted( + entry["name"] for entry in catalog if resolve_class_key(entry["name"], released_tests) is None + ) + catalog = [entry for entry in catalog if resolve_class_key(entry["name"], released_tests) is not None] if omitted_names: logger.info("Omitted %d unreleased tests from catalog", len(omitted_names)) logger.debug("Unreleased tests omitted from catalog: %s", ", ".join(omitted_names)) @@ -332,29 +327,70 @@ def build_catalog(*, released_only: bool = True) -> list[dict[str, Any]]: return catalog -def build_platform_axis() -> list[str]: - """Return the sorted platform-axis labels (the catalog's capability columns). +def suite_vocabularies() -> tuple[list[str], list[str]]: + """Return ``(capabilities, plain_suites)`` from one pass over the suite YAML. + + A suite file is either a platform suite (it declares a declarable + capability) or a plain suite named after its file - one classification, so + the two vocabularies are read off a single parse of the directory rather + than a scan each. + """ + capabilities: set[str] = set() + suites: set[str] = set() + for config_path, data in _iter_suite_docs(): + platform = _declared_platform(data) + if platform in DECLARABLE_CAPABILITIES: + capabilities.add(str(platform)) + else: + suites.add(canonical_suite_name(config_path.stem)) + return sorted(capabilities), sorted(suites) + + +def build_capability_vocabulary() -> list[str]: + """Return declarable capabilities derived from platform suite YAML.""" + return suite_vocabularies()[0] + + +def build_suite_vocabulary() -> list[str]: + """Return plain suite names declared by canonical suite YAML.""" + return suite_vocabularies()[1] - Derived from :data:`PLATFORM_CONFIGS`, the canonical per-platform suite - mapping, so adding a platform there extends the axis automatically. The - values match each entry's ``platforms`` field (e.g. ``KUBERNETES``), so a - consumer can build the platform matrix straight from the catalog. + +def _assert_disjoint_vocabulary(platforms: list[str], suites: list[str]) -> None: + """Reject a plain suite named after a declarable capability. + + Capabilities and plain suites share one uppercased namespace downstream: the + frontend merges ``platforms`` and ``suites`` into a single selectable + "test target" list, and the backend re-splits a flat selection by + intersecting it with the declared ``platforms``. Both are unambiguous only + while the two namespaces are disjoint, so enforce it at the upload + chokepoint (checked against the full reserved set, not just declared + platforms, so an undeclared capability word like ``slurm`` is caught too). """ - return sorted(PLATFORM_CONFIGS.keys()) + collisions = sorted(set(suites) & (set(platforms) | DECLARABLE_CAPABILITIES)) + if collisions: + raise ValueError( + "plain suite names collide with declarable capabilities: " + f"{', '.join(collisions)}; rename the suite file(s) so the capability " + "and suite namespaces stay disjoint" + ) def catalog_document(entries: list[dict[str, Any]], version: str) -> dict[str, Any]: """Wrap catalog ``entries`` in the versioned upload/artifact envelope. - Adds the schema version, the isvtest package version, and the platform axis - list (see :func:`build_platform_axis`). The per-entry ``labels`` are - intentionally not summarized at the top level - a consumer can derive the - label universe from the entries when needed. + Adds the schema version, the isvtest package version, and the catalog axis + vocabulary expected by the backend upload contract. The per-entry + ``labels`` are intentionally not summarized at the top level - a consumer + can derive the label universe from the entries when needed. """ + platforms, suites = suite_vocabularies() + _assert_disjoint_vocabulary(platforms, suites) return { "schemaVersion": CATALOG_SCHEMA_VERSION, "isvTestVersion": version, - "platforms": build_platform_axis(), + "platforms": platforms, + "suites": suites, "entries": entries, } diff --git a/isvtest/src/isvtest/catalog_platforms.py b/isvtest/src/isvtest/catalog_platforms.py deleted file mode 100644 index 5478dba3c..000000000 --- a/isvtest/src/isvtest/catalog_platforms.py +++ /dev/null @@ -1,43 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Canonical platform registry for the test catalog. - -Leaf module with no dependencies so lightweight consumers (for example the -validate-suites pre-commit hook) can read the registry without triggering -test discovery and its heavy transitive imports via :mod:`isvtest.catalog`. -""" - -# Configs that define the canonical test list per platform. -# Relative to the isvctl/configs/ directory. -PLATFORM_CONFIGS: dict[str, list[str]] = { - "BARE_METAL": ["suites/bare_metal.yaml"], - "CONTROL_PLANE": ["suites/control-plane.yaml"], - "IAM": ["suites/iam.yaml"], - "IMAGE_REGISTRY": ["suites/image-registry.yaml"], - "KUBERNETES": ["suites/k8s.yaml"], - "NETWORK": ["suites/network.yaml"], - "OBSERVABILITY": ["suites/observability.yaml"], - "SECURITY": ["suites/security.yaml"], - "SLURM": ["suites/slurm.yaml"], - "STORAGE": ["suites/storage.yaml"], - "VM": ["suites/vm.yaml"], -} - -# Maps wiring labels to platform strings so a check's platform can be inferred -# from its labels when it isn't otherwise tied to a platform. Derived from -# PLATFORM_CONFIGS: every platform's label is its lowercase name. Trait labels -# like "gpu", "ssh", "workload", and "slow" are intentionally not platforms. -LABEL_TO_PLATFORM: dict[str, str] = {platform.lower(): platform for platform in PLATFORM_CONFIGS} diff --git a/isvtest/src/isvtest/core/resolution.py b/isvtest/src/isvtest/core/resolution.py index 2e52b61f0..afbcd01b1 100644 --- a/isvtest/src/isvtest/core/resolution.py +++ b/isvtest/src/isvtest/core/resolution.py @@ -33,6 +33,35 @@ ADAPTER_HANDLED_CATEGORIES = {"reframe"} DEFAULT_VALIDATION_PHASE = "test" +DECLARABLE_CAPABILITIES = frozenset({"vm", "bare_metal", "kubernetes", "slurm"}) + + +def requires_error(values: Any) -> str | None: + """Return why ``values`` is not a valid ``requires`` list, or None when it is. + + One statement of the rule for every place that enforces it - the pydantic + step and suite validators, the runtime entry-shape check, and the suite + wiring script - so adding a capability or relaxing the rule is a single + edit and the four call sites cannot report different verdicts. + """ + if not isinstance(values, list) or any( + not isinstance(value, str) or value not in DECLARABLE_CAPABILITIES for value in values + ): + return f"requires must be a list containing only: {', '.join(sorted(DECLARABLE_CAPABILITIES))}" + if len(values) != len(set(values)): + return "requires must not contain duplicates" + return None + + +def canonical_suite_name(value: str) -> str: + """Normalize a CLI spelling, filename stem, or platform key to a suite name. + + Suite name is the join key between a test run and its catalog entries, so + the producer, the CLI resolver, and the wiring validator all have to spell + it the same way - including the ``k8s`` filename alias. + """ + normalized = value.strip().lower().replace("-", "_") + return "kubernetes" if normalized == "k8s" else normalized class State(StrEnum): @@ -53,6 +82,7 @@ class SkipReason(StrEnum): STEP_NO_OUTPUT = "step_no_output" # step ran but produced no JSON output STEP_NOT_CONFIGURED = "step_not_configured" # step the entry binds to isn't in the platform's step list UNRELEASED = "unreleased" # not in released_tests.json (gated until release) + CAPABILITY_REQUIREMENT = "capability_requirement" # declared capabilities do not satisfy ``requires`` class ErrorReason(StrEnum): @@ -73,6 +103,7 @@ class ValidationEntry: step: str | None = None phase: str | None = None labels: tuple[str, ...] = () + requires: tuple[str, ...] = () @dataclass @@ -110,6 +141,25 @@ def _wiring_labels(params_template: Any) -> tuple[str, ...]: return tuple(labels) +def _wiring_requires(params_template: Any) -> tuple[str, ...]: + """Return the capability prerequisites declared on a check's YAML wiring.""" + value = params_template.get("requires") if isinstance(params_template, dict) else None + if not isinstance(value, list): + return () + return tuple(item for item in value if isinstance(item, str)) + + +def requirements_satisfied(requires: Iterable[str], capability: str) -> bool: + """Return whether the single capability context satisfies a check's requirements. + + The four capabilities are mutually exclusive execution environments, so a run + carries exactly one. A core check (empty ``requires``) always runs; otherwise + the check runs when the context capability is among its any-match prerequisites. + """ + required = set(requires) + return not required or capability in required + + def parse_validations(raw_config: Mapping[str, Any]) -> list[ValidationEntry]: """Parse raw validation config into ordered validation entries. @@ -144,6 +194,7 @@ def parse_validations(raw_config: Mapping[str, Any]) -> list[ValidationEntry]: params_template = copy.deepcopy(params_template) labels = _wiring_labels(params_template) + requires = _wiring_requires(params_template) entries.append( ValidationEntry( name=name, @@ -152,6 +203,7 @@ def parse_validations(raw_config: Mapping[str, Any]) -> list[ValidationEntry]: step=entry_step if isinstance(entry_step, str) else None, phase=entry_phase if isinstance(entry_phase, str) else None, labels=labels, + requires=requires, ) ) @@ -169,6 +221,7 @@ def resolve_entries( exclude_tests: AbstractSet[str], released_tests: AbstractSet[str] | None, render_context: Mapping[str, Any], + capability: str | None = None, ) -> list[ResolvedEntry]: """Resolve validation entries into ready or terminal outcomes. @@ -182,6 +235,7 @@ def resolve_entries( exclude_tests: Validation names excluded by config. released_tests: Released test manifest, or None when unreleased checks are included. render_context: Jinja context for validation parameter rendering. + capability: Declared capability context (a single platform), or None to disable requirement filtering. Returns: A resolved entry for every input entry, in input order. @@ -212,6 +266,18 @@ def resolve_entries( resolved.append(_skip(entry, SkipReason.EXCLUDED, f"validation '{entry.name}' is excluded by name")) continue + if capability is not None and not requirements_satisfied(entry.requires, capability): + requirement_list = ", ".join(entry.requires) or "(none)" + context_list = capability + resolved.append( + _skip( + entry, + SkipReason.CAPABILITY_REQUIREMENT, + f"requires {requirement_list} (context: {context_list})", + ) + ) + continue + missing_include_labels = sorted(set(include_labels).difference(entry.labels)) if missing_include_labels: label_list = ", ".join(sorted(include_labels)) @@ -293,6 +359,7 @@ def resolve_entries( rendered_params.pop("step", None) rendered_params["step_output"] = copy.deepcopy(step_outputs[entry.step]) rendered_params.pop("phase", None) + rendered_params.pop("requires", None) rendered_params["_category"] = entry.category resolved.append(ResolvedEntry(entry=entry, rendered_params=rendered_params)) @@ -414,6 +481,11 @@ def _validate_entry_shape(entry: ValidationEntry) -> str | None: invalid_message = entry.params_template.get("_invalid_config") if invalid_message: return str(invalid_message) + raw_requires = entry.params_template.get("requires") + if raw_requires is not None: + message = requires_error(raw_requires) + if message: + return f"validation '{entry.name}' {message}" return None diff --git a/isvtest/tests/test_catalog.py b/isvtest/tests/test_catalog.py index ee9d33c65..e4da53abb 100644 --- a/isvtest/tests/test_catalog.py +++ b/isvtest/tests/test_catalog.py @@ -17,10 +17,14 @@ from unittest.mock import patch +import pytest + from isvtest.catalog import ( CATALOG_SCHEMA_VERSION, + _assert_disjoint_vocabulary, + build_capability_vocabulary, build_catalog, - build_platform_axis, + build_suite_vocabulary, catalog_document, get_catalog_version, ) @@ -38,79 +42,72 @@ def run(self) -> None: class TestCatalogDocument: - """Tests for the platform axis and the versioned catalog envelope.""" - - def test_derives_platform_axis_from_configs(self) -> None: - """The platform axis lists every platform in PLATFORM_CONFIGS, sorted.""" - assert build_platform_axis() == [ - "BARE_METAL", - "CONTROL_PLANE", - "IAM", - "IMAGE_REGISTRY", - "KUBERNETES", - "NETWORK", - "OBSERVABILITY", - "SECURITY", - "SLURM", - "STORAGE", - "VM", - ] + """Tests for capability vocabulary and the versioned catalog envelope.""" + + def test_derives_capabilities_from_platform_suites(self) -> None: + """Only real platform suite keys are declarable capabilities.""" + assert build_capability_vocabulary() == ["bare_metal", "kubernetes", "slurm", "vm"] + + def test_derives_suite_vocabulary_from_plain_suites(self) -> None: + """Plain suite YAML files are listed separately from platform suites.""" + suites = build_suite_vocabulary() + assert "iam" in suites + assert "storage" in suites + assert "kubernetes" not in suites + assert "vm" not in suites def test_catalog_document_wraps_entries_with_metadata(self) -> None: - """The envelope carries schema version, package version, and the platform axis.""" + """The envelope carries schema version, package version, and axis lists.""" entries = [{"name": "X", "labels": ["iam"]}] doc = catalog_document(entries, "1.2.3") assert doc["schemaVersion"] == CATALOG_SCHEMA_VERSION assert doc["isvTestVersion"] == "1.2.3" assert doc["entries"] == entries - assert doc["platforms"] == build_platform_axis() + assert doc["platforms"] == build_capability_vocabulary() + assert doc["suites"] == build_suite_vocabulary() + assert "capabilities" not in doc # The label universe is intentionally not summarized at the top level. assert "labels" not in doc + def test_disjoint_vocabulary_accepts_distinct_namespaces(self) -> None: + """Plain suite names that are not capability words pass the guard.""" + _assert_disjoint_vocabulary(["vm", "kubernetes"], ["storage", "iam", "network"]) -class TestBuildCatalog: - """Tests for build_catalog function.""" + def test_disjoint_vocabulary_rejects_suite_named_after_capability(self) -> None: + """A plain suite named after any declarable capability is a namespace collision.""" + with pytest.raises(ValueError, match="kubernetes"): + _assert_disjoint_vocabulary(["vm", "kubernetes"], ["storage", "kubernetes"]) - def test_returns_list_of_dicts(self) -> None: - """Test that build_catalog returns a list of dicts.""" - catalog = build_catalog() - assert isinstance(catalog, list) - assert len(catalog) > 0 - for entry in catalog: - assert isinstance(entry, dict) + def test_disjoint_vocabulary_rejects_undeclared_capability_word(self) -> None: + """Collision is checked against the full reserved set, not just declared platforms.""" + with pytest.raises(ValueError, match="slurm"): + _assert_disjoint_vocabulary(["vm"], ["slurm"]) - def test_entries_have_required_keys(self) -> None: - """Test that each entry has the required keys.""" - catalog = build_catalog() - for entry in catalog: - assert "name" in entry - assert "description" in entry - assert "labels" in entry - assert "test_ids" in entry - assert "module" in entry - assert "markers" not in entry - - def test_entries_have_correct_types(self) -> None: - """Test that entry values have the correct types.""" - catalog = build_catalog() - for entry in catalog: - assert isinstance(entry["name"], str) - assert isinstance(entry["description"], str) - assert isinstance(entry["labels"], list) - assert isinstance(entry["module"], str) - def test_no_duplicate_names(self) -> None: - """Test that there are no duplicate test names in the catalog.""" - catalog = build_catalog() - names = [e["name"] for e in catalog] - assert len(names) == len(set(names)) +class TestBuildCatalog: + """Tests for build_catalog function.""" - def test_known_tests_present(self) -> None: - """Test that some known validation tests appear in the catalog.""" - catalog = build_catalog() - names = {e["name"] for e in catalog} - assert "StepSuccessCheck" in names - assert "FieldExistsCheck" in names + def test_entries_have_suite_contract(self) -> None: + """Catalog rows expose suite placement and requirement metadata.""" + catalog = build_catalog(released_only=False) + names = [entry["name"] for entry in catalog] + assert catalog + assert len(names) == len(set(names)) + for entry in catalog: + assert set(entry) == { + "name", + "description", + "labels", + "test_ids", + "source", + "suite", + "platform", + "requires", + } + assert isinstance(entry["source"], str) + assert isinstance(entry["requires"], list) + if entry["platform"]: + assert entry["requires"] == [] def test_extract_checks_supports_direct_dict_category_form(self, tmp_path) -> None: """Direct dict category wiring is included in catalog config scans.""" @@ -166,24 +163,18 @@ def test_entries_expose_wired_test_ids(self) -> None: assert all(isinstance(tid, str) for tid in entry["test_ids"]) assert "N/A" not in entry["test_ids"] - # Single mapping, and a duality unioned across the bm/vm suites. + # Single mappings retain their requirement and suite placement. assert by_name["MfaEnforcedCheck"]["test_ids"] == ["SEC07-01"] - assert by_name["GpuCheck"]["test_ids"] == ["BMAAS08-01", "VMAAS06-01"] - - def test_variant_test_ids_propagate_to_base(self) -> None: - """A variant's wired test_id surfaces on its base-class catalog entry.""" - catalog = build_catalog(released_only=False) - by_name = {e["name"]: e for e in catalog} - - assert by_name["StepSuccessCheck-delete_tenant"]["test_ids"] == ["CP10-01"] - assert "CP10-01" in by_name["StepSuccessCheck"]["test_ids"] + assert by_name["MfaEnforcedCheck"]["suite"] == "security" + assert by_name["MfaEnforcedCheck"]["requires"] == [] def test_released_only_filters_catalog(self) -> None: """Default catalog generation excludes tests not in the release manifest.""" with patch("isvtest.catalog.load_released_test_filter", return_value={"StepSuccessCheck"}): catalog = build_catalog() - assert {e["name"] for e in catalog} == {"StepSuccessCheck"} + assert catalog + assert all(entry["name"].startswith("StepSuccessCheck") for entry in catalog) def test_unreleased_env_includes_full_catalog(self) -> None: """When the release filter is disabled, default catalog generation includes all tests.""" @@ -205,7 +196,16 @@ def test_catalog_emits_explicit_labels(self) -> None: """Per-wiring YAML labels are surfaced as catalog tag metadata.""" with ( patch("isvtest.catalog.discover_all_tests", return_value=[ExplicitLabelCatalogCheck]), - patch("isvtest.catalog._build_platform_map", return_value={}), + patch( + "isvtest.catalog._build_suite_map", + return_value={ + "ExplicitLabelCatalogCheck": { + "suite": "demo", + "platform": None, + "requires": ["vm", "bare_metal"], + } + }, + ), patch( "isvtest.catalog.build_label_map", return_value={"ExplicitLabelCatalogCheck": {"accelerator", "long_running"}}, @@ -221,97 +221,19 @@ def test_catalog_emits_explicit_labels(self) -> None: "description": "Explicit labels", "labels": ["accelerator", "long_running"], "test_ids": [], - "module": __name__, - "platforms": [], + "source": __name__, + "suite": "demo", + "platform": None, + "requires": ["vm", "bare_metal"], } ] - def test_modules_are_valid_python_paths(self) -> None: - """Test that module paths look like valid Python module paths.""" + def test_sources_are_valid_python_paths(self) -> None: + """Source paths remain useful implementation metadata, not a suite axis.""" catalog = build_catalog() for entry in catalog: - assert "." in entry["module"] - assert entry["module"].startswith("isvtest.") - - def test_suite_membership_overrides_label_platforms(self) -> None: - """Regression: trait labels must not add extra platform ownership. - - A check can carry labels like ``("security", "network")`` for pytest - filtering AND appear in a single suite YAML (e.g. ``security.yaml``). - ``_build_platform_map`` must use the suite as the source of truth and - skip label-derived platform inference in that case - otherwise the - UI shows phantom platform badges. - - DO NOT add per-check asserts to this test. It is a property test - that already covers every check in the catalog. If a new validation - breaks the invariant, the failure message names it. - """ - from isvtest.catalog import ( - LABEL_TO_PLATFORM, - PLATFORM_CONFIGS, - _extract_checks_from_config, - _find_configs_dir, - ) - - configs_dir = _find_configs_dir() - assert configs_dir is not None, "isvctl/configs/ not found" - - suite_platforms: dict[str, set[str]] = {} - for platform, files in PLATFORM_CONFIGS.items(): - for relpath in files: - for name in _extract_checks_from_config(configs_dir / relpath): - suite_platforms.setdefault(name, set()).add(platform) - - for entry in build_catalog(released_only=False): - name = entry["name"] - if name not in suite_platforms: - continue - label_platforms = {LABEL_TO_PLATFORM[label] for label in entry["labels"] if label in LABEL_TO_PLATFORM} - expected = suite_platforms[name] - actual = set(entry["platforms"]) - phantom = (label_platforms - expected) & actual - assert not phantom, ( - f"{name}: label-derived platforms {sorted(phantom)} leaked " - f"into catalog; expected exactly {sorted(expected)}, " - f"got {sorted(actual)}" - ) - assert actual == expected, ( - f"{name}: platforms should equal suite assignment {sorted(expected)}, got {sorted(actual)}" - ) - - def test_observability_label_infers_platform_for_unlisted_checks(self) -> None: - """Checks labelled with `observability` are tagged OBSERVABILITY when not in any suite.""" - - class ObservabilityLabelledCheck(BaseValidation): - description = "Observability check labelled but not in any suite" - - def run(self) -> None: - self.set_passed() - - ObservabilityLabelledCheck.__module__ = "isvtest.validations.fake" - - with ( - patch("isvtest.catalog.discover_all_tests", return_value=[ObservabilityLabelledCheck]), - patch("isvtest.catalog._build_platform_map", return_value={}), - patch( - "isvtest.catalog.build_label_map", - return_value={"ObservabilityLabelledCheck": {"observability"}}, - ), - patch("isvtest.catalog.build_test_id_map", return_value={}), - patch("isvtest.catalog.load_released_test_filter", return_value=None), - ): - catalog = build_catalog() - - assert catalog == [ - { - "name": "ObservabilityLabelledCheck", - "description": "Observability check labelled but not in any suite", - "labels": ["observability"], - "test_ids": [], - "module": "isvtest.validations.fake", - "platforms": ["OBSERVABILITY"], - } - ] + assert "." in entry["source"] + assert entry["source"].startswith("isvtest.") class TestGetCatalogVersion: diff --git a/isvtest/tests/test_resolution.py b/isvtest/tests/test_resolution.py index c518a88bd..fdf0a54ab 100644 --- a/isvtest/tests/test_resolution.py +++ b/isvtest/tests/test_resolution.py @@ -64,6 +64,7 @@ def _entry( step: str | None = None, phase: str | None = None, labels: tuple[str, ...] = (), + requires: tuple[str, ...] = (), ) -> ValidationEntry: """Build a minimal validation entry.""" return ValidationEntry( @@ -73,6 +74,7 @@ def _entry( step=step, phase=phase, labels=labels, + requires=requires, ) @@ -87,6 +89,7 @@ def _resolve( exclude_tests: set[str] | None = None, released_tests: set[str] | None = None, render_context: dict[str, Any] | None = None, + capability: str | None = None, ) -> ResolvedEntry: """Resolve one entry and return the single result.""" results = resolve_entries( @@ -99,11 +102,35 @@ def _resolve( exclude_tests=set() if exclude_tests is None else exclude_tests, released_tests=released_tests, render_context={} if render_context is None else render_context, + capability=capability, ) assert len(results) == 1 return results[0] +def test_any_declared_requirement_satisfies_capability_filter() -> None: + """Orthogonal platform requirements use OR semantics.""" + entry = _entry(requires=("vm", "bare_metal")) + + assert _resolve(entry, capability="vm").is_ready + assert _resolve(entry, capability="bare_metal").is_ready + assert _resolve(_entry(requires=()), capability="kubernetes").is_ready + + +def test_capability_filter_has_explicit_skip_reason() -> None: + """An unmet prerequisite reports both the requirement and active context.""" + resolved = _resolve(_entry(requires=("vm", "bare_metal")), capability="kubernetes") + + assert resolved.state == State.SKIPPED + assert resolved.skip_reason == SkipReason.CAPABILITY_REQUIREMENT + assert resolved.message == "requires vm, bare_metal (context: kubernetes)" + + +def test_omitted_capability_disables_requirement_filtering() -> None: + """Local development without a capability context runs every check.""" + assert _resolve(_entry(requires=("kubernetes",)), capability=None).is_ready + + @pytest.mark.parametrize( ("entry", "kwargs", "expected_reason"), [ diff --git a/scripts/tests/test_validate_suite_wiring.py b/scripts/tests/test_validate_suite_wiring.py index 310f15ed9..b860de059 100644 --- a/scripts/tests/test_validate_suite_wiring.py +++ b/scripts/tests/test_validate_suite_wiring.py @@ -27,15 +27,18 @@ def test_wiring_errors_flags_missing_metadata(tmp_path: Path) -> None: checks: GoodCheck: test_id: "SEC01-01" - labels: ["security"] + labels: ["demo", "security"] + requires: [] BadCheck: - labels: ["security"] + labels: ["demo", "security"] + requires: [] AlsoBad: test_id: "N/A" + requires: [] """ ) errors = validate_suite_wiring.wiring_errors(tmp_path) - assert any("demo.yaml:8" in err and "BadCheck" in err and "missing test_id" in err for err in errors) + assert any("demo.yaml:" in err and "BadCheck" in err and "missing test_id" in err for err in errors) assert any("demo.yaml:" in err and "AlsoBad" in err and "missing labels" in err for err in errors) assert not any("GoodCheck" in err for err in errors) @@ -46,6 +49,7 @@ def test_wiring_errors_rejects_scalar_labels(tmp_path: Path) -> None: suite.write_text( """\ tests: + platform: kubernetes validations: example: checks: @@ -64,6 +68,7 @@ def test_wiring_errors_require_canonical_suite_label(tmp_path: Path) -> None: suite.write_text( """\ tests: + platform: kubernetes validations: example: checks: @@ -111,21 +116,93 @@ def test_repo_suites_declare_test_id_and_labels() -> None: assert not errors, "suite wiring validation failed:\n " + "\n ".join(errors) -def test_platform_registration_errors_flags_unregistered_suite(tmp_path: Path) -> None: - """A suite file not present in PLATFORM_CONFIGS is reported. +def test_plain_suite_requires_are_explicit_and_valid(tmp_path: Path) -> None: + """Plain suites require an allowed list, including an explicit empty list.""" + (tmp_path / "demo.yaml").write_text( + """\ +tests: + validations: + sample: + checks: + MissingCheck: + test_id: "N/A" + labels: ["demo"] + InvalidCheck: + test_id: "N/A" + labels: ["demo"] + requires: [foundational] +""" + ) + + errors = validate_suite_wiring.wiring_errors(tmp_path) + assert any("MissingCheck" in error and "missing requires" in error for error in errors) + assert any("InvalidCheck" in error and "requires must be a list containing only" in error for error in errors) + + +def test_wiring_errors_rejects_plain_suite_named_after_capability(tmp_path: Path) -> None: + """A plain suite file named like a declarable capability is a namespace collision.""" + (tmp_path / "kubernetes.yaml").write_text( + """\ +tests: + validations: + sample: + checks: + SomeCheck: + test_id: "N/A" + labels: ["demo"] + requires: [] +""" + ) + + errors = validate_suite_wiring.wiring_errors(tmp_path) + assert any("kubernetes" in error and "collides with a declarable capability" in error for error in errors) + + +def test_wiring_errors_flags_requires_with_no_platform_suite(tmp_path: Path) -> None: + """A `requires` naming a capability that has no platform suite is unreachable.""" + (tmp_path / "vm.yaml").write_text( + """\ +tests: + platform: vm + validations: + example: + checks: + VmCheck: + test_id: "N/A" + labels: ["vm"] +""" + ) + (tmp_path / "demo.yaml").write_text( + """\ +tests: + validations: + example: + checks: + DeadCheck: + test_id: "N/A" + labels: ["demo"] + requires: [slurm] +""" + ) + + errors = validate_suite_wiring.wiring_errors(tmp_path) + assert any("DeadCheck" in error and "slurm" in error and "no platform suite" in error for error in errors) - Enforced against the real repo only by the validate-suites pre-commit - hook, not by a repo-level pytest guard, so untracked scratch suites - don't break `make test`. - """ - (tmp_path / "newcap.yaml").write_text("tests:\n validations: {}\n") - errors = validate_suite_wiring.platform_registration_errors(tmp_path) - assert errors == [ - "suites/newcap.yaml: not registered in isvtest.catalog_platforms.PLATFORM_CONFIGS (catalog platform axis)" - ] +def test_wiring_errors_allows_platform_suite_named_after_capability(tmp_path: Path) -> None: + """The kubernetes *platform* suite (declares tests.platform) is not a collision.""" + (tmp_path / "k8s.yaml").write_text( + """\ +tests: + platform: kubernetes + validations: + sample: + checks: + SomeCheck: + test_id: "N/A" + labels: ["kubernetes"] +""" + ) -def test_registered_platforms_round_trip_through_isvreporter() -> None: - """Guardrail: every catalog platform is recognized by isvreporter.""" - errors = validate_suite_wiring.registry_consistency_errors() - assert not errors, "registry consistency failed:\n " + "\n ".join(errors) + errors = validate_suite_wiring.wiring_errors(tmp_path) + assert not any("collides with a declarable capability" in error for error in errors) diff --git a/scripts/validate_suite_wiring.py b/scripts/validate_suite_wiring.py index a2427e734..025939e3f 100644 --- a/scripts/validate_suite_wiring.py +++ b/scripts/validate_suite_wiring.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Require ``test_id`` and ``labels`` on every check wired in suite YAML. +"""Validate suite identity and per-check metadata in canonical YAML. Suite configs under ``isvctl/configs/suites/`` are the source of truth for validation metadata on this branch. Each wired check must declare: @@ -25,13 +25,6 @@ Each canonical suite check must include its suite label, for example checks in ``bare_metal.yaml`` must include ``bare_metal``. -Every suite file must also be registered in -``isvtest.catalog_platforms.PLATFORM_CONFIGS``: an unregistered suite is -silently dropped from the pushed catalog's platform axis and its tests carry no -platform, so the capability never shows up in the catalog UI. Registered -platforms must in turn be recognized by ``isvreporter.platform`` — an unknown -platform is silently reported as the default on test-run upload. - Usage: python3 scripts/validate_suite_wiring.py python3 scripts/validate_suite_wiring.py --check # exit 1 on violations @@ -40,6 +33,7 @@ from __future__ import annotations import argparse +import os import re import sys from collections import defaultdict @@ -48,17 +42,14 @@ from typing import Any import yaml -from isvreporter.platform import normalize_platform -from isvtest.catalog_platforms import PLATFORM_CONFIGS +from isvtest.catalog import iter_checks_from_data +from isvtest.core.resolution import DECLARABLE_CAPABILITIES, canonical_suite_name, requires_error REPO_ROOT = Path(__file__).resolve().parent.parent SUITES_DIR = REPO_ROOT / "isvctl" / "configs" / "suites" _NEXT_CATEGORY_LINE = re.compile(r"^ \S") -# The label every check in a canonical suite must carry, derived from the -# platform registry: suite file stem -> lowercase platform name. -SUITE_REQUIRED_LABELS: dict[str, str] = { - Path(config).stem: platform.lower() for platform, configs in PLATFORM_CONFIGS.items() for config in configs -} +# Opt-in until unique wiring names land in a dedicated PR. +ENFORCE_UNIQUE_WIRING = os.environ.get("ISVCTL_ENFORCE_UNIQUE_WIRING") == "1" def _check_line_patterns(check_name: str) -> tuple[re.Pattern[str], ...]: @@ -106,7 +97,7 @@ def _normalize_test_id(value: Any) -> str | None: def required_suite_label(config_path: Path) -> str | None: """Return the label every check in a known canonical suite must carry.""" - return SUITE_REQUIRED_LABELS.get(config_path.stem) + return canonical_suite_name(config_path.stem) def iter_suite_checks(config_path: Path) -> Iterator[tuple[str, str, dict[str, Any]]]: @@ -115,28 +106,7 @@ def iter_suite_checks(config_path: Path) -> Iterator[tuple[str, str, dict[str, A data = yaml.safe_load(config_path.read_text()) except (OSError, yaml.YAMLError) as exc: raise ValueError(f"failed to read/parse {config_path}: {exc}") from exc - - validations = (data or {}).get("tests", {}).get("validations", {}) - if not isinstance(validations, dict): - return - - def _from_mapping(category: str, mapping: Any) -> Iterator[tuple[str, str, dict[str, Any]]]: - """Yield wired checks from a dict- or list-form ``checks`` mapping.""" - if isinstance(mapping, dict): - for name, params in mapping.items(): - yield category, name, params if isinstance(params, dict) else {} - - for category, cat_config in validations.items(): - if isinstance(cat_config, dict) and "checks" in cat_config: - checks_val = cat_config["checks"] - if isinstance(checks_val, dict): - yield from _from_mapping(category, checks_val) - elif isinstance(checks_val, list): - for check in checks_val: - yield from _from_mapping(category, check) - elif isinstance(cat_config, list): - for check in cat_config: - yield from _from_mapping(category, check) + yield from iter_checks_from_data(data) def _format_location(config_path: Path, category: str, check_name: str, line_number: int | None) -> str: @@ -154,14 +124,50 @@ def wiring_errors(suites_dir: Path = SUITES_DIR) -> list[str]: """Return human-readable errors for incomplete suite check wiring.""" errors: list[str] = [] occurrence: dict[tuple[Path, str, str], int] = defaultdict(int) + wiring_locations: dict[str, str] = {} + # Read and parse each suite once; both the dead-requirement pre-pass and the + # per-check loop below work off these parsed documents. + parsed: list[tuple[Path, list[str], dict[str, Any]]] = [] for path in sorted(suites_dir.glob("*.yaml")): try: - lines = path.read_text().splitlines() - checks = list(iter_suite_checks(path)) - except ValueError as exc: - errors.append(str(exc)) + text = path.read_text() + parsed.append((path, text.splitlines(), yaml.safe_load(text) or {})) + except (OSError, yaml.YAMLError) as exc: + errors.append(f"failed to read/parse {path}: {exc}") + + # A `requires` value is only satisfiable if an ISV can declare that + # capability, which requires a platform suite to exist for it. Collect the + # platform capabilities that actually have a suite so unreachable (dead) + # requirements can be flagged below. + declared_platforms: set[str] = set() + for _, _, data in parsed: + tests = data.get("tests") if isinstance(data, dict) else None + platform = tests.get("platform") if isinstance(tests, dict) else None + if isinstance(platform, str) and platform in DECLARABLE_CAPABILITIES: + declared_platforms.add(platform) + + for path, lines, data in parsed: + try: + checks = list(iter_checks_from_data(data)) + except (ValueError, AttributeError) as exc: + errors.append(f"failed to read/parse {path}: {exc}") continue + tests = data.get("tests") or {} + platform = tests.get("platform") if isinstance(tests, dict) else None + module = tests.get("module") if isinstance(tests, dict) else None + if module is not None: + errors.append(f"{path}: tests.module is no longer supported") + if platform is not None and platform not in DECLARABLE_CAPABILITIES: + errors.append(f"{path}: tests.platform must be one of: {', '.join(sorted(DECLARABLE_CAPABILITIES))}") + suite_is_platform = isinstance(platform, str) and platform in DECLARABLE_CAPABILITIES + if not suite_is_platform: + suite_name = canonical_suite_name(path.stem) + if suite_name in DECLARABLE_CAPABILITIES: + errors.append( + f"{path}: plain suite name {suite_name!r} collides with a declarable " + "capability; rename the file so capability and suite namespaces stay disjoint" + ) for category, name, params in checks: key = (path, category, name) line_numbers = find_check_line_numbers(lines, category, name) @@ -172,41 +178,41 @@ def wiring_errors(suites_dir: Path = SUITES_DIR) -> list[str]: test_id = _normalize_test_id(params.get("test_id")) labels = _normalize_labels(params.get("labels")) required_label = required_suite_label(path) + previous_location = wiring_locations.get(name) + # Uniqueness enforcement is intentionally deferred to a follow-up + # PR. Keep the check so it can be re-enabled without rediscovery. + if previous_location: + if ENFORCE_UNIQUE_WIRING: + errors.append(f"{location}: wiring name is not globally unique (also at {previous_location})") + else: + wiring_locations[name] = location if test_id is None: errors.append(f'{location}: missing test_id (use a plan id or "N/A")') if not labels: errors.append(f"{location}: missing labels (non-empty list required)") elif required_label and required_label not in labels: errors.append(f"{location}: missing suite label {required_label!r}") + if "platforms" in params: + errors.append(f"{location}: legacy platforms is not supported; use requires in plain suites") + if platform: + if "requires" in params: + errors.append(f"{location}: requires is not allowed in platform suites") + else: + requires = params.get("requires") + if not isinstance(requires, list): + errors.append(f"{location}: missing requires (use [] for core checks)") + elif message := requires_error(requires): + errors.append(f"{location}: {message}") + else: + dead = sorted(set(requires) - declared_platforms) + if dead: + errors.append( + f"{location}: requires names {', '.join(dead)} which has no platform " + "suite; no ISV can declare it, so the check is unreachable" + ) return errors -def platform_registration_errors(suites_dir: Path = SUITES_DIR) -> list[str]: - """Return errors for suite files not registered in the catalog platform axis.""" - registered = {config for configs in PLATFORM_CONFIGS.values() for config in configs} - return [ - f"{suite}: not registered in isvtest.catalog_platforms.PLATFORM_CONFIGS (catalog platform axis)" - for suite in (f"suites/{path.name}" for path in sorted(suites_dir.glob("*.yaml"))) - if suite not in registered - ] - - -def registry_consistency_errors() -> list[str]: - """Return errors for catalog platforms that isvreporter cannot round-trip. - - Suite configs declare ``tests.platform`` as the lowercase platform name; - ``normalize_platform`` silently coerces unknown values to the default - platform when reporting test runs, so a platform registered in the catalog - but missing from isvreporter's constants misattributes every test run. - """ - return [ - f"platform {platform}: not recognized by isvreporter.platform.normalize_platform " - "(add it to ALL_PLATFORMS and PLATFORM_ALIASES)" - for platform in sorted(PLATFORM_CONFIGS) - if normalize_platform(platform.lower()) != platform - ] - - def main(argv: list[str] | None = None) -> int: """CLI entry point. Returns a process exit code.""" parser = argparse.ArgumentParser(description=__doc__) @@ -220,7 +226,7 @@ def main(argv: list[str] | None = None) -> int: ) args = parser.parse_args(argv) - errors = wiring_errors() + platform_registration_errors() + registry_consistency_errors() + errors = wiring_errors() if errors: header = f"suite wiring validation failed ({len(errors)} issue(s)):" message = header + "\n " + "\n ".join(errors) @@ -230,10 +236,7 @@ def main(argv: list[str] | None = None) -> int: print(message) return 0 - ok = ( - f"OK: all wired checks in {SUITES_DIR.relative_to(REPO_ROOT)} declare test_id, labels, " - "and suite labels, and every suite is registered as a catalog platform." - ) + ok = f"OK: all wired checks in {SUITES_DIR.relative_to(REPO_ROOT)} declare valid suite metadata." print(ok) return 0