feat: select validations by suite and capability

Makefile

@@ -1,5 +1,5 @@
-MY_ISV_DOMAINS := bare_metal control-plane iam image-registry network observability security storage vm
-DEMO_TARGETS := $(addprefix demo-,$(MY_ISV_DOMAINS))
+MY_ISV_SUITES := bare_metal control-plane iam image-registry network observability security storage vm
+DEMO_TARGETS := $(addprefix demo-,$(MY_ISV_SUITES))
 
 .PHONY: help pre-commit build test coverage clean lint format install bump-patch bump-fix bump-minor bump-feat bump-major bump bump-check \
 	security-trivy security-trivy-detail security-trufflehog ci-security demo-test demo-all $(DEMO_TARGETS) plan plan-coverage validate-suites \
@@ -84,29 +84,46 @@ test: ## Run tests for all packages
 	@echo ""
 	@echo "✅ All tests passed!"
 
+# run_demo,<suite>[,<capability>] - a plain suite with no capability runs only
+# its core checks, so suites holding gated checks name one to exercise those
+# stubs. Unlisted suites are core-only or platform suites.
+DEMO_CAP_network       := vm
+DEMO_CAP_observability := vm
+DEMO_CAP_security      := vm
+DEMO_CAP_storage       := vm
+
 define run_demo
 	@echo ""
 	@echo "=========================================="
-	@echo "Demo test: $(1)"
+	@echo "Demo test: $(1)$(if $(2), --capability $(2),)"
 	@echo "=========================================="
-	@echo "Running cmd: ISVCTL_DEMO_MODE=1 uv run isvctl test run -f isvctl/configs/providers/my-isv/config/$(1).yaml"
+	@echo "Running cmd: ISVCTL_DEMO_MODE=1 uv run isvctl test run -f isvctl/configs/providers/my-isv/config/$(1).yaml$(if $(2), --capability $(2),)"
 	@ISVCTL_DEMO_MODE=1 uv run isvctl test run \
-		-f isvctl/configs/providers/my-isv/config/$(1).yaml
+		-f isvctl/configs/providers/my-isv/config/$(1).yaml \
+		$(if $(2),--capability $(2),)
 endef
 
 demo-test: demo-all ## Alias for demo-all (backward compat)
 
-demo-all: ## Run all my-isv living examples (or demo-<domain> for one, e.g. demo-security)
+demo-all: ## Run all my-isv living examples (or demo-<suite> for one, e.g. demo-security)
 	@echo "Running my-isv living examples in demo mode..."
-	@for domain in $(MY_ISV_DOMAINS); do \
-		$(MAKE) --no-print-directory demo-$$domain || exit 1; \
+	@for suite in $(MY_ISV_SUITES); do \
+		$(MAKE) --no-print-directory demo-$$suite || exit 1; \
 	done
 	@echo ""
 	@echo "✅ All my-isv living examples passed in demo mode!"
-	@echo "Domains: $(MY_ISV_DOMAINS)"
-
-$(DEMO_TARGETS): demo-%:
-	$(call run_demo,$*)
+	@echo "Suites: $(MY_ISV_SUITES)"
+
+# image-registry is excluded here because it has its own rule below; leaving it
+# in would make every make invocation warn about an overridden recipe.
+$(filter-out demo-image-registry,$(DEMO_TARGETS)): demo-%:
+	$(call run_demo,$*,$(DEMO_CAP_$*))
+
+# image-registry splits its gated checks between vm and bare_metal, so one run
+# cannot reach both.
+demo-image-registry:
+	$(call run_demo,image-registry,vm)
+	$(call run_demo,image-registry,bare_metal)
 
 coverage: ## Run tests with coverage and generate combined report
 	@echo "Running tests with coverage..."

docs/guides/configuration.md

@@ -124,7 +124,6 @@ commands:
         timeout: 300
 
 tests:
-  platform: network
   cluster_name: "aws-network-test"
 
   settings:
@@ -195,6 +194,28 @@ Each step defines a command to execute:
 | `skip` | No | Skip this step |
 | `continue_on_failure` | No | Continue even if this step fails |
 | `output_schema` | No | Schema name for output validation |
+| `requires` | No | Capability contexts this step runs in (see [Capabilities](#capabilities-and-requires)) |
+
+#### Gating a step with `requires`
+
+A step is skipped automatically when **every** validation bound to it is
+filtered out by the run's capability. That inference covers most steps, but it
+cannot reach a step no validation binds to — typically a teardown step:
+
+```yaml
+# Runs only under --capability kubernetes. Without the gate, a core run would
+# try to tear down a cluster it never created.
+- name: teardown_cluster
+  phase: teardown
+  command: "./scripts/teardown_cluster.sh"
+  requires: [kubernetes]
+```
+
+Rule of thumb: **if a step builds or destroys a fixture that only some contexts
+need, give it an explicit `requires:`** — and give both halves of the fixture the
+same one, so setup and teardown always move together. A step that survives the
+gate must not reference a gated-off step's output; use `default(...)` if it
+legitimately might be absent.
 
 ### Validation Configuration
 
@@ -267,6 +288,64 @@ The part before the dash must match an existing validation class name (e.g., `K8
 - The suffix is free-form: `K8sNimHelmWorkload-small`, `SlurmPartition-cpu`, `SlurmGpuAllocation-1gpu` are all valid.
 - Each variant is a distinct test entry in coverage tracking.
 
+## Capabilities and `requires`
+
+An ISV declares which **capabilities** it supports. There are exactly four, and
+they are mutually exclusive execution environments — you run on one at a time,
+never a combination:
+
+`vm` · `bare_metal` · `kubernetes` · `slurm`
+
+Each has a **platform suite** (`suites/vm.yaml`, `suites/k8s.yaml`, ...) carrying
+the checks you owe by declaring it. Its `tests.platform:` key names the
+capability, and its checks declare no `requires:` — they all run.
+
+Everything else is a **plain suite** (`storage`, `network`, `iam`, ...), named by
+its filename. A plain suite mixes checks that need no particular infrastructure
+with checks that presuppose some. Each check says which:
+
+```yaml
+requires: []                # core - runs in every context
+requires: [kubernetes]      # runs only under --capability kubernetes
+requires: [vm, bare_metal]  # any-match: either context satisfies it
+```
+
+`requires` is **any-match**, not a set to satisfy simultaneously: a check runs
+when its list is empty, or when the run's capability appears in it. There is
+deliberately no way to express "needs vm AND kubernetes" — no check needs it,
+and the mutual exclusivity above means such a check could never run.
+
+### What runs, and when
+
+One rule, and it does not depend on how you named the config — `--suite`, `-f`,
+and `--label` discovery behave identically:
+
+> **A plain suite with no `--capability` runs its core checks.** Name a
+> capability to add the checks gated on it.
+
+```bash
+isvctl test run --provider acme --suite storage                         # core only
+isvctl test run --provider acme --suite storage --capability kubernetes # core + k8s checks
+isvctl test run --provider acme --suite kubernetes                      # the platform suite
+```
+
+There is no "run everything" context: a plain suite always carries exactly one.
+Passing a capability no check in the suite requires is allowed but warns, since
+a flag that silently does nothing is usually a typo.
+
+Two consequences worth internalising:
+
+- **Nothing is mandatory.** A check is in scope only if you declared the suite
+  that contains it. 100% is always relative to what you declared, so declaring
+  a subset legitimately yields zero checks from the suites you left out.
+- **A capability and a plain suite compose.** The 15 CSI checks in `storage`
+  need `storage` *and* `kubernetes`. Declaring `kubernetes` alone runs the
+  Kubernetes platform suite but no storage CSI checks.
+
+Capability names and plain-suite names share one namespace, so a plain suite may
+not be named after a capability. `catalog_document` and
+`scripts/validate_suite_wiring.py` both reject the collision.
+
 ## Import and Override
 
 Provider configs can import a canonical test suite and override command definitions while inheriting validations (unless explicitly overridden):

docs/guides/external-validation-guide.md

@@ -126,7 +126,6 @@ commands:
         timeout: 300
 
 tests:
-  platform: myplatform
   cluster_name: "my-validation"
 
   settings:
@@ -186,9 +185,64 @@ For validation timing and phase control, see the [Configuration Guide](configura
 
 ---
 
+## Capabilities and check requirements
+
+You declare which **capabilities** you support. There are four, and they are
+mutually exclusive execution environments — you run on one at a time:
+
+`vm` · `bare_metal` · `kubernetes` · `slurm`
+
+Each has a **platform suite** you owe by declaring it (`vm`, `bare_metal`,
+`kubernetes`, `slurm`). Everything else is a **plain suite** (`storage`,
+`network`, `iam`, ...) that mixes checks needing no particular infrastructure
+with checks that presuppose some. Each check declares which:
+
+```yaml
+requires: []                # core - runs in every context
+requires: [kubernetes]      # runs only under --capability kubernetes
+requires: [vm, bare_metal]  # any-match: either context satisfies it
+```
+
+**Nothing is mandatory.** A check is in scope only if you declared the suite
+containing it, so 100% is always relative to what you declared. Declaring a
+subset legitimately yields zero checks from the suites you left out — that is
+the design, not a gap.
+
+One rule decides what runs, and it does not depend on how you named the config
+(`--suite`, `-f` and `--label` behave identically):
+
+> **A plain suite with no `--capability` runs its core checks.** Name a
+> capability to add the checks gated on it.
+
+If a step builds or destroys a fixture only some contexts need, gate it the same
+way so a core run neither provisions nor leaks it:
+
+```yaml
+- name: teardown_cluster
+  phase: teardown
+  command: "./scripts/teardown_cluster.sh"
+  requires: [kubernetes]
+```
+
+See the [Configuration Guide](configuration.md#capabilities-and-requires) for the
+full model.
+
+---
+
 ## Running Validations
 
 ```bash
+# One suite for your provider - the form the UI emits
+isvctl test run --provider acme --suite storage                         # core checks
+isvctl test run --provider acme --suite storage --capability kubernetes # + k8s checks
+isvctl test run --provider acme --suite kubernetes                      # a platform suite
+
+# See what would run, without executing anything
+isvctl test run --provider acme --suite storage --capability vm --dry-run
+
+# Point at a config file directly (same capability rule applies)
+isvctl test run -f config.yaml
+
 # Run all phases
 isvctl test run -f config.yaml
 
@@ -209,10 +263,20 @@ isvctl test run -f config.yaml -- -k "ConnectivityCheck"
 isvctl test run -f config.yaml --label gpu
 isvctl test run -f config.yaml -- -m "not slow"
 
+# Labels compose with suite selection
+isvctl test run --provider acme --suite storage --capability vm --label min_req
+
+# Re-run one failed check in its lifecycle context (pytest passthrough)
+isvctl test run --provider acme --suite storage --capability kubernetes -- -k K8sCsiPvcExpandCheck
+
 # Debug: full output on failure
 isvctl test run -f config.yaml -v -- -s --tb=long
 ```
 
+Re-running a single failed check is pytest passthrough after `--`; there are no
+dedicated rerun flags. Setup steps re-run, which is the deliberate trade for not
+maintaining a dependency graph.
+
 > **Teardown behavior:** By default, teardown runs even when setup or test validations fail, ensuring cloud resources are cleaned up. Individual teardown step failures don't block remaining teardown steps (best-effort execution).
 
 ---
@@ -266,9 +330,13 @@ Preview the whole pipeline with no cloud:
 
 ```bash
 make demo-test   # sets ISVCTL_DEMO_MODE=1 and runs all my-isv configs (~10s)
-# Domains are listed in the Makefile MY_ISV_DOMAINS variable.
+# Suites are listed in the Makefile MY_ISV_SUITES variable; DEMO_CAP_<suite>
+# names the capability a suite runs under so its gated checks are exercised.
 ```
 
+The k8s and Slurm examples are excluded: they drive a real cluster, so a
+dummy-success stub has nothing to return for them.
+
 ---
 
 ## Related Documentation

docs/guides/local-development.md

@@ -149,7 +149,6 @@ commands:
         timeout: 60
 
 tests:
-  platform: network
   cluster_name: "local-test"
   settings:
     region: "us-west-2"

docs/test-plan.yaml

@@ -3243,6 +3243,7 @@ domains:
                 labels:
                   - kubernetes
                   - min_req
+                  - storage
                 priority: P0
                 milestone: M5
                 notes: ""
@@ -3255,6 +3256,7 @@ domains:
                 labels:
                   - kubernetes
                   - min_req
+                  - storage
                 priority: P0
                 milestone: M5
                 notes: ""
@@ -3267,6 +3269,7 @@ domains:
                 labels:
                   - kubernetes
                   - min_req
+                  - storage
                 priority: P0
                 milestone: M5
                 notes: ""
@@ -3279,6 +3282,7 @@ domains:
                 labels:
                   - kubernetes
                   - min_req
+                  - storage
                 priority: P0
                 milestone: M5
                 notes: ""

isvctl/configs/providers/aws/config/image-registry.yaml

@@ -102,18 +102,18 @@ commands:
         timeout: 120
 
       # Step 5: Cleanup all resources
-      - name: teardown
+      # Teardown is split by owner so each half can be gated independently.
+      # This half cleans up what launch_instance created, so it carries the
+      # same `requires: [vm]` gate and is skipped whole in other contexts.
+      # Declared first: terminate the instance before deregistering the AMI it
+      # was launched from.
+      - name: teardown_instance
         phase: teardown
         command: "python3 ../scripts/image-registry/teardown.py"
+        requires: [vm]
         args:
           - "--instance-id"
           - "{{steps.launch_instance.instance_id}}"
-          - "--ami-id"
-          - "{{steps.upload_image.image_id}}"
-          - "--snapshot-ids"
-          - "{{steps.upload_image.disk_ids | join(',')}}"
-          - "--bucket-name"
-          - "{{steps.upload_image.storage_bucket}}"
           - "--key-name"
           - "{{steps.launch_instance.key_name}}"
           - "--security-group-id"
@@ -125,6 +125,23 @@ commands:
           - "{{teardown_flag}}"
         timeout: 1800
 
+      # This half cleans up what the core upload_image step created, so it is
+      # ungated and runs in every capability context.
+      - name: teardown_image
+        phase: teardown
+        command: "python3 ../scripts/image-registry/teardown.py"
+        args:
+          - "--ami-id"
+          - "{{steps.upload_image.image_id}}"
+          - "--snapshot-ids"
+          - "{{steps.upload_image.disk_ids | join(',')}}"
+          - "--bucket-name"
+          - "{{steps.upload_image.storage_bucket}}"
+          - "--region"
+          - "{{region}}"
+          - "{{teardown_flag}}"
+        timeout: 1800
+
 tests:
   cluster_name: "aws-image-registry-validation"
   description: "AWS VM Import ISO/VMDK validation tests"