From c8b9e57b5249be9e6363d555ec44f3d52dfdca07 Mon Sep 17 00:00:00 2001 From: Alexandre Begnoche Date: Wed, 22 Jul 2026 13:31:42 -0400 Subject: [PATCH 01/34] feat(isvtest): add requires capability expand and filter Introduce declarable capability vocabulary, expand_capabilities() for compute inheritance from vm/bare_metal, per-check requires filtering with explicit capability_requirement skip reasons, and validation for requires values on plain-suite checks. Signed-off-by: Alexandre Begnoche --- isvtest/src/isvtest/core/resolution.py | 49 ++++++++++++++++++++++++++ isvtest/tests/test_resolution.py | 26 ++++++++++++++ 2 files changed, 75 insertions(+) diff --git a/isvtest/src/isvtest/core/resolution.py b/isvtest/src/isvtest/core/resolution.py index 2e52b61f0..d0d9267d3 100644 --- a/isvtest/src/isvtest/core/resolution.py +++ b/isvtest/src/isvtest/core/resolution.py @@ -33,6 +33,8 @@ ADAPTER_HANDLED_CATEGORIES = {"reframe"} DEFAULT_VALIDATION_PHASE = "test" +DECLARABLE_CAPABILITIES = frozenset({"vm", "bare_metal", "kubernetes", "slurm"}) +REQUIREMENT_VOCABULARY = DECLARABLE_CAPABILITIES | {"compute"} class State(StrEnum): @@ -53,6 +55,7 @@ class SkipReason(StrEnum): STEP_NO_OUTPUT = "step_no_output" # step ran but produced no JSON output STEP_NOT_CONFIGURED = "step_not_configured" # step the entry binds to isn't in the platform's step list UNRELEASED = "unreleased" # not in released_tests.json (gated until release) + CAPABILITY_REQUIREMENT = "capability_requirement" # declared capabilities do not satisfy ``requires`` class ErrorReason(StrEnum): @@ -73,6 +76,7 @@ class ValidationEntry: step: str | None = None phase: str | None = None labels: tuple[str, ...] = () + requires: tuple[str, ...] = () @dataclass @@ -110,6 +114,22 @@ def _wiring_labels(params_template: Any) -> tuple[str, ...]: return tuple(labels) +def _wiring_requires(params_template: Any) -> tuple[str, ...]: + """Return the capability prerequisites declared on a check's YAML wiring.""" + value = params_template.get("requires") if isinstance(params_template, dict) else None + if not isinstance(value, list): + return () + return tuple(item for item in value if isinstance(item, str)) + + +def expand_capabilities(declared: Iterable[str]) -> frozenset[str]: + """Expand declarable capabilities with requirement-only aliases.""" + expanded = set(declared) + if expanded & {"vm", "bare_metal"}: + expanded.add("compute") + return frozenset(expanded) + + def parse_validations(raw_config: Mapping[str, Any]) -> list[ValidationEntry]: """Parse raw validation config into ordered validation entries. @@ -144,6 +164,7 @@ def parse_validations(raw_config: Mapping[str, Any]) -> list[ValidationEntry]: params_template = copy.deepcopy(params_template) labels = _wiring_labels(params_template) + requires = _wiring_requires(params_template) entries.append( ValidationEntry( name=name, @@ -152,6 +173,7 @@ def parse_validations(raw_config: Mapping[str, Any]) -> list[ValidationEntry]: step=entry_step if isinstance(entry_step, str) else None, phase=entry_phase if isinstance(entry_phase, str) else None, labels=labels, + requires=requires, ) ) @@ -169,6 +191,7 @@ def resolve_entries( exclude_tests: AbstractSet[str], released_tests: AbstractSet[str] | None, render_context: Mapping[str, Any], + capabilities: AbstractSet[str] | None = None, ) -> list[ResolvedEntry]: """Resolve validation entries into ready or terminal outcomes. @@ -182,12 +205,14 @@ def resolve_entries( exclude_tests: Validation names excluded by config. released_tests: Released test manifest, or None when unreleased checks are included. render_context: Jinja context for validation parameter rendering. + capabilities: Declared capability context, or None to disable requirement filtering. Returns: A resolved entry for every input entry, in input order. """ resolved: list[ResolvedEntry] = [] env = _create_jinja_env() + expanded_capabilities = expand_capabilities(capabilities or ()) if capabilities is not None else None for entry in entries: config_error = _validate_entry_shape(entry) @@ -212,6 +237,18 @@ def resolve_entries( resolved.append(_skip(entry, SkipReason.EXCLUDED, f"validation '{entry.name}' is excluded by name")) continue + if expanded_capabilities is not None and not set(entry.requires).issubset(expanded_capabilities): + requirement_list = ", ".join(entry.requires) or "(none)" + context_list = ", ".join(sorted(capabilities or ())) or "(none)" + resolved.append( + _skip( + entry, + SkipReason.CAPABILITY_REQUIREMENT, + f"requires {requirement_list} (context: {context_list})", + ) + ) + continue + missing_include_labels = sorted(set(include_labels).difference(entry.labels)) if missing_include_labels: label_list = ", ".join(sorted(include_labels)) @@ -293,6 +330,7 @@ def resolve_entries( rendered_params.pop("step", None) rendered_params["step_output"] = copy.deepcopy(step_outputs[entry.step]) rendered_params.pop("phase", None) + rendered_params.pop("requires", None) rendered_params["_category"] = entry.category resolved.append(ResolvedEntry(entry=entry, rendered_params=rendered_params)) @@ -414,6 +452,17 @@ def _validate_entry_shape(entry: ValidationEntry) -> str | None: invalid_message = entry.params_template.get("_invalid_config") if invalid_message: return str(invalid_message) + raw_requires = entry.params_template.get("requires") + if raw_requires is not None: + if not isinstance(raw_requires, list) or any( + not isinstance(value, str) or value not in REQUIREMENT_VOCABULARY for value in raw_requires + ): + return ( + f"validation '{entry.name}' requires must be a list containing only: " + f"{', '.join(sorted(REQUIREMENT_VOCABULARY))}" + ) + if len(raw_requires) != len(set(raw_requires)): + return f"validation '{entry.name}' requires must not contain duplicates" return None diff --git a/isvtest/tests/test_resolution.py b/isvtest/tests/test_resolution.py index c518a88bd..8cb0c7f0e 100644 --- a/isvtest/tests/test_resolution.py +++ b/isvtest/tests/test_resolution.py @@ -64,6 +64,7 @@ def _entry( step: str | None = None, phase: str | None = None, labels: tuple[str, ...] = (), + requires: tuple[str, ...] = (), ) -> ValidationEntry: """Build a minimal validation entry.""" return ValidationEntry( @@ -73,6 +74,7 @@ def _entry( step=step, phase=phase, labels=labels, + requires=requires, ) @@ -87,6 +89,7 @@ def _resolve( exclude_tests: set[str] | None = None, released_tests: set[str] | None = None, render_context: dict[str, Any] | None = None, + capabilities: set[str] | None = None, ) -> ResolvedEntry: """Resolve one entry and return the single result.""" results = resolve_entries( @@ -99,11 +102,34 @@ def _resolve( exclude_tests=set() if exclude_tests is None else exclude_tests, released_tests=released_tests, render_context={} if render_context is None else render_context, + capabilities=capabilities, ) assert len(results) == 1 return results[0] +def test_compute_requirement_expands_from_vm_or_bare_metal() -> None: + """Either concrete compute capability satisfies a compute prerequisite.""" + entry = _entry(requires=("compute",)) + + assert _resolve(entry, capabilities={"vm"}).is_ready + assert _resolve(entry, capabilities={"bare_metal"}).is_ready + + +def test_capability_filter_has_explicit_skip_reason() -> None: + """An unmet prerequisite reports both the requirement and active context.""" + resolved = _resolve(_entry(requires=("compute",)), capabilities={"kubernetes"}) + + assert resolved.state == State.SKIPPED + assert resolved.skip_reason == SkipReason.CAPABILITY_REQUIREMENT + assert resolved.message == "requires compute (context: kubernetes)" + + +def test_omitted_capabilities_disables_requirement_filtering() -> None: + """Local development without a capability context runs every check.""" + assert _resolve(_entry(requires=("kubernetes",)), capabilities=None).is_ready + + @pytest.mark.parametrize( ("entry", "kwargs", "expected_reason"), [ From 5b4cafd49e77e438286161f975201515e48dae2b Mon Sep 17 00:00:00 2001 From: Alexandre Begnoche Date: Wed, 22 Jul 2026 13:31:47 -0400 Subject: [PATCH 02/34] feat(isvctl): validate requires and reject legacy suite axes Reject tests.module, tests.platform=compute, requires on platform suites, and legacy per-check platforms fields. Rewrite validate_suite_wiring for plain vs platform suite rules and globally unique wiring names. Signed-off-by: Alexandre Begnoche --- isvctl/schemas/config.schema.json | 16 +++- isvctl/src/isvctl/config/schema.py | 22 ++++- scripts/tests/test_validate_suite_wiring.py | 48 ++++++----- scripts/validate_suite_wiring.py | 92 ++++++++++----------- 4 files changed, 105 insertions(+), 73 deletions(-) diff --git a/isvctl/schemas/config.schema.json b/isvctl/schemas/config.schema.json index 9cba8929f..44efdbe3b 100644 --- a/isvctl/schemas/config.schema.json +++ b/isvctl/schemas/config.schema.json @@ -62,7 +62,7 @@ }, "PlatformCommands": { "additionalProperties": true, - "description": "Lifecycle commands for a specific platform.\n\nGroups commands for a platform (kubernetes, slurm, bare_metal, network, vm, iam, image_registry, security).\nSupports skip at both platform level (skips all phases) and phase level.\n\nThe `phases` field defines the execution order. Steps are grouped by their `phase`\nfield and executed in the order defined by `phases`. Validations run after each phase.\n\nExample:\n ```yaml\n network:\n phases: [\"setup\", \"teardown\"] # Defines execution order\n steps:\n - name: create_vpc\n phase: setup\n command: \"./create_vpc.py\"\n - name: cleanup\n phase: teardown\n command: \"./teardown.py\"\n ```\n\nIf a step's phase is not in the phases list, an error is raised.\n\nSupports skip at both platform level (skips all phases) and step level.", + "description": "Lifecycle commands for a specific platform.\n\nGroups commands for a platform (kubernetes, slurm, bare_metal, network, vm, iam, image_registry, security, observability).\nSupports skip at both platform level (skips all phases) and phase level.\n\nThe `phases` field defines the execution order. Steps are grouped by their `phase`\nfield and executed in the order defined by `phases`. Validations run after each phase.\n\nExample:\n ```yaml\n network:\n phases: [\"setup\", \"test\", \"teardown\"] # Defines execution order\n steps:\n - name: create_vpc\n phase: setup\n command: \"./create_vpc.py\"\n - name: cleanup\n phase: teardown\n command: \"./teardown.py\"\n ```\n\nIf a step's phase is not in the phases list, an error is raised.\n\nSupports skip at both platform level (skips all phases) and step level.", "properties": { "skip": { "default": false, @@ -145,6 +145,14 @@ "title": "Skip", "type": "boolean" }, + "requires_available_validations": { + "description": "Validation names that must be available after release filtering for this step to run. Unreleased validations are available only when ISVTEST_INCLUDE_UNRELEASED=1.", + "items": { + "type": "string" + }, + "title": "Requires Available Validations", + "type": "array" + }, "continue_on_failure": { "default": false, "description": "Continue to next step even if this step fails", @@ -188,7 +196,7 @@ }, "ValidationConfig": { "additionalProperties": true, - "description": "Test configuration section.\n\nValidations are grouped by meaningful category names (e.g., 'network', 'ssh', 'gpu').\nEach validation can have a `phase` field to control execution timing:\n\n- No `phase`: Runs after setup steps complete (default)\n- `phase: teardown`: Runs after teardown steps complete\n- `phase: test`: Runs after test steps (if any exist)\n\nTwo formats are supported:\n\n1. List format (each validation specifies its own step/phase):\n validations:\n network:\n - VpcCrudCheck:\n step: vpc_crud\n - NetworkProvisionedCheck:\n step: create_network\n\n2. Group defaults format (step/phase apply to all checks):\n validations:\n credentials:\n step: test_credentials\n phase: test\n checks:\n - StepSuccessCheck: {}\n - FieldExistsCheck:\n field: \"account_id\"", + "description": "Test configuration section.\n\nValidations are grouped by meaningful category names (e.g., 'network', 'ssh', 'gpu').\nEach validation can have a `phase` field to control execution timing:\n\n- No `phase`: Runs after setup steps complete (default)\n- `phase: teardown`: Runs after teardown steps complete\n- `phase: test`: Runs after test steps (if any exist)\n\nThree formats are supported:\n\n1. List format (each validation specifies its own step/phase):\n validations:\n network:\n - VpcCrudCheck:\n step: vpc_crud\n - NetworkProvisionedCheck:\n step: create_network\n\n2. Group defaults with list checks (step/phase apply to all checks):\n validations:\n credentials:\n step: test_credentials\n phase: test\n checks:\n - StepSuccessCheck: {}\n - FieldExistsCheck:\n field: \"account_id\"\n\n3. Group defaults with dict checks (deep-merge friendly):\n validations:\n credentials:\n step: test_credentials\n checks:\n StepSuccessCheck: {}\n FieldExistsCheck:\n field: \"account_id\"", "properties": { "cluster_name": { "anyOf": [ @@ -226,7 +234,7 @@ } ], "default": null, - "description": "Platform: kubernetes, slurm, bare_metal, network, vm, iam, image_registry, security", + "description": "Platform type: KUBERNETES, SLURM, BARE_METAL, CONTROL_PLANE, IAM, NETWORK, SECURITY, VM, IMAGE_REGISTRY, OBSERVABILITY, STORAGE", "title": "Platform" }, "settings": { @@ -291,7 +299,7 @@ "additionalProperties": { "$ref": "#/$defs/PlatformCommands" }, - "description": "Lifecycle commands by platform (kubernetes, slurm, bare_metal, network, vm, iam, image_registry, security)", + "description": "Lifecycle commands by platform (kubernetes, slurm, bare_metal, network, vm, iam, image_registry, security, observability)", "title": "Commands", "type": "object" }, diff --git a/isvctl/src/isvctl/config/schema.py b/isvctl/src/isvctl/config/schema.py index 7ef1d56ed..75eda669f 100644 --- a/isvctl/src/isvctl/config/schema.py +++ b/isvctl/src/isvctl/config/schema.py @@ -24,7 +24,9 @@ from typing import Any -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from isvtest.core.resolution import DECLARABLE_CAPABILITIES, parse_validations class LabConfig(BaseModel): @@ -375,6 +377,24 @@ class ValidationConfig(BaseModel): description="Exclusion rules (keys: labels, platforms, tests, files)", ) + @model_validator(mode="after") + def validate_suite_shape(self) -> "ValidationConfig": + """Reject legacy axes and requirements on platform suite checks.""" + if self.model_extra and "module" in self.model_extra: + raise ValueError("tests.module is no longer supported; plain suites have no axis key") + if self.platform == "compute": + raise ValueError("compute is requirement-only and cannot be declared as tests.platform") + if self.platform and self.platform not in DECLARABLE_CAPABILITIES: + raise ValueError( + f"tests.platform must be one of: {', '.join(sorted(DECLARABLE_CAPABILITIES))}" + ) + entries = parse_validations(self.validations) + if self.platform and any("requires" in entry.params_template for entry in entries): + raise ValueError("requires is not allowed in platform suites") + if any("platforms" in entry.params_template for entry in entries): + raise ValueError("per-check platforms is no longer supported; use requires in plain suites") + return self + class RunConfig(BaseModel): """Unified configuration schema for a complete test run. diff --git a/scripts/tests/test_validate_suite_wiring.py b/scripts/tests/test_validate_suite_wiring.py index 310f15ed9..55d41e6ac 100644 --- a/scripts/tests/test_validate_suite_wiring.py +++ b/scripts/tests/test_validate_suite_wiring.py @@ -27,15 +27,18 @@ def test_wiring_errors_flags_missing_metadata(tmp_path: Path) -> None: checks: GoodCheck: test_id: "SEC01-01" - labels: ["security"] + labels: ["demo", "security"] + requires: [] BadCheck: - labels: ["security"] + labels: ["demo", "security"] + requires: [] AlsoBad: test_id: "N/A" + requires: [] """ ) errors = validate_suite_wiring.wiring_errors(tmp_path) - assert any("demo.yaml:8" in err and "BadCheck" in err and "missing test_id" in err for err in errors) + assert any("demo.yaml:" in err and "BadCheck" in err and "missing test_id" in err for err in errors) assert any("demo.yaml:" in err and "AlsoBad" in err and "missing labels" in err for err in errors) assert not any("GoodCheck" in err for err in errors) @@ -46,6 +49,7 @@ def test_wiring_errors_rejects_scalar_labels(tmp_path: Path) -> None: suite.write_text( """\ tests: + platform: kubernetes validations: example: checks: @@ -64,6 +68,7 @@ def test_wiring_errors_require_canonical_suite_label(tmp_path: Path) -> None: suite.write_text( """\ tests: + platform: kubernetes validations: example: checks: @@ -111,21 +116,24 @@ def test_repo_suites_declare_test_id_and_labels() -> None: assert not errors, "suite wiring validation failed:\n " + "\n ".join(errors) -def test_platform_registration_errors_flags_unregistered_suite(tmp_path: Path) -> None: - """A suite file not present in PLATFORM_CONFIGS is reported. - - Enforced against the real repo only by the validate-suites pre-commit - hook, not by a repo-level pytest guard, so untracked scratch suites - don't break `make test`. - """ - (tmp_path / "newcap.yaml").write_text("tests:\n validations: {}\n") - errors = validate_suite_wiring.platform_registration_errors(tmp_path) - assert errors == [ - "suites/newcap.yaml: not registered in isvtest.catalog_platforms.PLATFORM_CONFIGS (catalog platform axis)" - ] - +def test_plain_suite_requires_are_explicit_and_valid(tmp_path: Path) -> None: + """Plain suites require an allowed list, including an explicit empty list.""" + (tmp_path / "demo.yaml").write_text( + """\ +tests: + validations: + sample: + checks: + MissingCheck: + test_id: "N/A" + labels: ["demo"] + InvalidCheck: + test_id: "N/A" + labels: ["demo"] + requires: [foundational] +""" + ) -def test_registered_platforms_round_trip_through_isvreporter() -> None: - """Guardrail: every catalog platform is recognized by isvreporter.""" - errors = validate_suite_wiring.registry_consistency_errors() - assert not errors, "registry consistency failed:\n " + "\n ".join(errors) + errors = validate_suite_wiring.wiring_errors(tmp_path) + assert any("MissingCheck" in error and "missing requires" in error for error in errors) + assert any("InvalidCheck" in error and "requires must contain only" in error for error in errors) diff --git a/scripts/validate_suite_wiring.py b/scripts/validate_suite_wiring.py index a2427e734..c7091c160 100644 --- a/scripts/validate_suite_wiring.py +++ b/scripts/validate_suite_wiring.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Require ``test_id`` and ``labels`` on every check wired in suite YAML. +"""Validate suite identity and per-check metadata in canonical YAML. Suite configs under ``isvctl/configs/suites/`` are the source of truth for validation metadata on this branch. Each wired check must declare: @@ -25,13 +25,6 @@ Each canonical suite check must include its suite label, for example checks in ``bare_metal.yaml`` must include ``bare_metal``. -Every suite file must also be registered in -``isvtest.catalog_platforms.PLATFORM_CONFIGS``: an unregistered suite is -silently dropped from the pushed catalog's platform axis and its tests carry no -platform, so the capability never shows up in the catalog UI. Registered -platforms must in turn be recognized by ``isvreporter.platform`` — an unknown -platform is silently reported as the default on test-run upload. - Usage: python3 scripts/validate_suite_wiring.py python3 scripts/validate_suite_wiring.py --check # exit 1 on violations @@ -48,17 +41,11 @@ from typing import Any import yaml -from isvreporter.platform import normalize_platform -from isvtest.catalog_platforms import PLATFORM_CONFIGS - REPO_ROOT = Path(__file__).resolve().parent.parent SUITES_DIR = REPO_ROOT / "isvctl" / "configs" / "suites" _NEXT_CATEGORY_LINE = re.compile(r"^ \S") -# The label every check in a canonical suite must carry, derived from the -# platform registry: suite file stem -> lowercase platform name. -SUITE_REQUIRED_LABELS: dict[str, str] = { - Path(config).stem: platform.lower() for platform, configs in PLATFORM_CONFIGS.items() for config in configs -} +DECLARABLE_CAPABILITIES = {"vm", "bare_metal", "kubernetes", "slurm"} +REQUIREMENT_VOCABULARY = DECLARABLE_CAPABILITIES | {"compute"} def _check_line_patterns(check_name: str) -> tuple[re.Pattern[str], ...]: @@ -106,7 +93,9 @@ def _normalize_test_id(value: Any) -> str | None: def required_suite_label(config_path: Path) -> str | None: """Return the label every check in a known canonical suite must carry.""" - return SUITE_REQUIRED_LABELS.get(config_path.stem) + if config_path.stem == "k8s": + return "kubernetes" + return config_path.stem.replace("-", "_") def iter_suite_checks(config_path: Path) -> Iterator[tuple[str, str, dict[str, Any]]]: @@ -154,14 +143,25 @@ def wiring_errors(suites_dir: Path = SUITES_DIR) -> list[str]: """Return human-readable errors for incomplete suite check wiring.""" errors: list[str] = [] occurrence: dict[tuple[Path, str, str], int] = defaultdict(int) + wiring_locations: dict[str, str] = {} for path in sorted(suites_dir.glob("*.yaml")): try: lines = path.read_text().splitlines() + data = yaml.safe_load(path.read_text()) or {} checks = list(iter_suite_checks(path)) - except ValueError as exc: - errors.append(str(exc)) + except (ValueError, yaml.YAMLError) as exc: + errors.append(f"failed to read/parse {path}: {exc}") continue + tests = data.get("tests") or {} + platform = tests.get("platform") if isinstance(tests, dict) else None + module = tests.get("module") if isinstance(tests, dict) else None + if module is not None: + errors.append(f"{path}: tests.module is no longer supported") + if platform is not None and platform not in DECLARABLE_CAPABILITIES: + errors.append( + f"{path}: tests.platform must be one of: {', '.join(sorted(DECLARABLE_CAPABILITIES))}" + ) for category, name, params in checks: key = (path, category, name) line_numbers = find_check_line_numbers(lines, category, name) @@ -172,41 +172,38 @@ def wiring_errors(suites_dir: Path = SUITES_DIR) -> list[str]: test_id = _normalize_test_id(params.get("test_id")) labels = _normalize_labels(params.get("labels")) required_label = required_suite_label(path) + previous_location = wiring_locations.get(name) + if previous_location: + errors.append(f"{location}: wiring name is not globally unique (also at {previous_location})") + else: + wiring_locations[name] = location if test_id is None: errors.append(f'{location}: missing test_id (use a plan id or "N/A")') if not labels: errors.append(f"{location}: missing labels (non-empty list required)") elif required_label and required_label not in labels: errors.append(f"{location}: missing suite label {required_label!r}") + if "platforms" in params: + errors.append(f"{location}: legacy platforms is not supported; use requires in plain suites") + if platform: + if "requires" in params: + errors.append(f"{location}: requires is not allowed in platform suites") + else: + requires = params.get("requires") + if not isinstance(requires, list): + errors.append(f"{location}: missing requires (use [] for core checks)") + elif any( + not isinstance(requirement, str) or requirement not in REQUIREMENT_VOCABULARY + for requirement in requires + ): + errors.append( + f"{location}: requires must contain only: {', '.join(sorted(REQUIREMENT_VOCABULARY))}" + ) + elif len(requires) != len(set(requires)): + errors.append(f"{location}: requires must not contain duplicates") return errors -def platform_registration_errors(suites_dir: Path = SUITES_DIR) -> list[str]: - """Return errors for suite files not registered in the catalog platform axis.""" - registered = {config for configs in PLATFORM_CONFIGS.values() for config in configs} - return [ - f"{suite}: not registered in isvtest.catalog_platforms.PLATFORM_CONFIGS (catalog platform axis)" - for suite in (f"suites/{path.name}" for path in sorted(suites_dir.glob("*.yaml"))) - if suite not in registered - ] - - -def registry_consistency_errors() -> list[str]: - """Return errors for catalog platforms that isvreporter cannot round-trip. - - Suite configs declare ``tests.platform`` as the lowercase platform name; - ``normalize_platform`` silently coerces unknown values to the default - platform when reporting test runs, so a platform registered in the catalog - but missing from isvreporter's constants misattributes every test run. - """ - return [ - f"platform {platform}: not recognized by isvreporter.platform.normalize_platform " - "(add it to ALL_PLATFORMS and PLATFORM_ALIASES)" - for platform in sorted(PLATFORM_CONFIGS) - if normalize_platform(platform.lower()) != platform - ] - - def main(argv: list[str] | None = None) -> int: """CLI entry point. Returns a process exit code.""" parser = argparse.ArgumentParser(description=__doc__) @@ -220,7 +217,7 @@ def main(argv: list[str] | None = None) -> int: ) args = parser.parse_args(argv) - errors = wiring_errors() + platform_registration_errors() + registry_consistency_errors() + errors = wiring_errors() if errors: header = f"suite wiring validation failed ({len(errors)} issue(s)):" message = header + "\n " + "\n ".join(errors) @@ -231,8 +228,7 @@ def main(argv: list[str] | None = None) -> int: return 0 ok = ( - f"OK: all wired checks in {SUITES_DIR.relative_to(REPO_ROOT)} declare test_id, labels, " - "and suite labels, and every suite is registered as a catalog platform." + f"OK: all wired checks in {SUITES_DIR.relative_to(REPO_ROOT)} declare valid suite metadata." ) print(ok) return 0 From 4099fc5de44fbef4148ca00aee9551a6b26cdc9c Mon Sep 17 00:00:00 2001 From: Alexandre Begnoche Date: Wed, 22 Jul 2026 13:31:47 -0400 Subject: [PATCH 03/34] feat(isvctl): add --suite/--capabilities CLI and step gating Resolve platform or plain suite configs via unified --suite selection, parse optional --capabilities filter context (rejecting compute), apply capability filtering in orchestration with human-readable dry-run plans, and skip setup steps when all bound checks are filtered out. Signed-off-by: Alexandre Begnoche --- isvctl/src/isvctl/cli/test.py | 90 ++++++++++++-- isvctl/src/isvctl/config/suite_resolution.py | 121 +++++++++++++++++++ isvctl/src/isvctl/orchestrator/loop.py | 52 ++++++-- isvctl/tests/test_cli_streams.py | 18 +-- isvctl/tests/test_suite_resolution.py | 61 ++++++++++ isvctl/tests/test_test_cli_labels.py | 4 +- 6 files changed, 318 insertions(+), 28 deletions(-) create mode 100644 isvctl/src/isvctl/config/suite_resolution.py create mode 100644 isvctl/tests/test_suite_resolution.py diff --git a/isvctl/src/isvctl/cli/test.py b/isvctl/src/isvctl/cli/test.py index 5b721c79c..002f0df2b 100644 --- a/isvctl/src/isvctl/cli/test.py +++ b/isvctl/src/isvctl/cli/test.py @@ -27,7 +27,8 @@ import typer import yaml -from isvtest.catalog import build_catalog, get_catalog_version +from isvtest.catalog import build_catalog, catalog_document, get_catalog_version +from isvtest.core.resolution import expand_capabilities, parse_validations from isvtest.release_manifest import load_released_test_filter from isvctl.cli import setup_logging @@ -47,6 +48,7 @@ ) from isvctl.config.merger import merge_yaml_files from isvctl.config.schema import RunConfig +from isvctl.config.suite_resolution import SuiteResolutionError, parse_capabilities, resolve_suite from isvctl.orchestrator.loop import Orchestrator, Phase from isvctl.redaction import redact_dict from isvctl.reporting import check_upload_credentials, create_test_run, get_environment_config, update_test_run @@ -115,6 +117,31 @@ def _junitxml_for_discovered_config(junitxml: Path, match: ProviderConfigMatch, return junitxml.with_name(f"{junitxml.stem}-{match.config_path.stem}{junitxml.suffix}") +def _human_readable_dry_run(config: RunConfig, capabilities: set[str] | None) -> str: + """Render the validation requirement plan without executing lifecycle steps.""" + platform = config.tests.platform if config.tests and config.tests.platform else None + suite_type = f"platform ({platform})" if platform else "plain" + context = "not filtered" if capabilities is None else ", ".join(sorted(capabilities)) or "(none)" + expanded = expand_capabilities(capabilities or ()) if capabilities is not None else None + validations = config.tests.validations if config.tests else {} + entries = parse_validations(validations) + + lines = [ + "Dry-run plan", + f" Suite type: {suite_type}", + f" Capabilities: {context}", + f" Checks: {len(entries)}", + ] + for entry in entries: + requirement = ", ".join(entry.requires) or "core" + if expanded is not None and not set(entry.requires).issubset(expanded): + declared = ", ".join(sorted(capabilities or ())) or "(none)" + lines.append(f" [SKIP] {entry.name}: requires {requirement} (context: {declared})") + else: + lines.append(f" [RUN] {entry.name}: requires {requirement}") + return "\n".join(lines) + + @app.command("run", context_settings={"allow_extra_args": True, "ignore_unknown_options": True}) def run( ctx: typer.Context, @@ -137,6 +164,20 @@ def run( help="Provider name for label discovery when no --config/-f files are supplied.", ), ] = None, + suite: Annotated[ + str | None, + typer.Option( + "--suite", + help="Run one platform or plain suite from the selected provider.", + ), + ] = None, + capabilities: Annotated[ + str | None, + typer.Option( + "--capabilities", + help="Comma-separated capability context used to filter check requirements.", + ), + ] = None, set_values: Annotated[ list[str] | None, typer.Option( @@ -160,6 +201,13 @@ def run( help="Label to filter validations (can be repeated; all selected labels must match)", ), ] = None, + exclude_labels: Annotated[ + list[str] | None, + typer.Option( + "--exclude-label", + help="Exclude validations carrying this label (can be repeated).", + ), + ] = None, dry_run: Annotated[ bool, typer.Option( @@ -252,6 +300,31 @@ def run( setup_logging(verbose) apply_user_config(no_user_config) + try: + capability_context = parse_capabilities(capabilities, CONFIGS_ROOT) + except SuiteResolutionError as exc: + print_error(str(exc)) + raise typer.Exit(code=1) + + if suite: + if not provider: + print_error("--suite requires --provider.") + raise typer.Exit(code=1) + if config_files: + print_error("--suite cannot be combined with --config/-f.") + raise typer.Exit(code=1) + if labels: + print_error("--suite cannot be combined with --label/-l; use labels after -- with pytest selection.") + raise typer.Exit(code=1) + try: + selected_suite = resolve_suite(provider, suite, configs_root=CONFIGS_ROOT) + except SuiteResolutionError as exc: + print_error(str(exc)) + raise typer.Exit(code=1) + print_progress(f"Selected {selected_suite.name!r} suite for provider {provider!r}.") + config_files = [selected_suite.config_path] + provider = None + if provider: if config_files: print_error("--provider discovery cannot be combined with --config/-f.") @@ -289,9 +362,12 @@ def run( ctx, config_files=[match.config_path], provider=None, + suite=None, + capabilities=capabilities, set_values=set_values, phase=phase, labels=labels, + exclude_labels=exclude_labels, dry_run=False, working_dir=working_dir, verbose=verbose, @@ -351,11 +427,7 @@ def run( raise typer.Exit(code=1) if dry_run: - # Keep stdout as pure JSON so it can be consumed with `json.loads`; - # decorative headers and extra args go to stderr. - print_progress("\n--- Dry Run: Configuration ---") - redacted_config = redact_dict(config.model_dump(mode="json")) - typer.echo(json.dumps(redacted_config, indent=2)) + typer.echo(_human_readable_dry_run(config, capability_context)) if extra_pytest_args: print_progress(f"\n--- Extra pytest args ---\n{extra_pytest_args}") return @@ -400,7 +472,7 @@ def run( # Create test run before running tests if upload_results and lab_id: print_progress("Creating test run in ISV Lab Service...") - platform = config.tests.platform if config.tests and config.tests.platform else "kubernetes" + platform = config.tests.platform if config.tests and config.tests.platform else next(iter(config.commands), "unknown") test_run_id = create_test_run( lab_id=lab_id, platform=platform, @@ -432,6 +504,8 @@ def run( phases=phases, extra_pytest_args=extra_pytest_args, include_labels=labels, + exclude_labels=exclude_labels, + capabilities=capability_context, verbose=verbose, junitxml=str(junitxml), ) @@ -442,7 +516,7 @@ def run( print_progress(f"Built test catalog: {len(catalog_entries)} tests (version: {catalog_version})") catalog_path = output_dir / "test_catalog.json" catalog_path.write_text( - json.dumps({"isvTestVersion": catalog_version, "entries": catalog_entries}, indent=2) + json.dumps(catalog_document(catalog_entries, catalog_version), indent=2) ) print_progress(f" Saved test catalog to: {catalog_path}") except Exception as e: diff --git a/isvctl/src/isvctl/config/suite_resolution.py b/isvctl/src/isvctl/config/suite_resolution.py new file mode 100644 index 000000000..3bc69c57c --- /dev/null +++ b/isvctl/src/isvctl/config/suite_resolution.py @@ -0,0 +1,121 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Resolve one platform or plain suite to a provider configuration.""" + +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import yaml + +from isvctl.config.merger import merge_yaml_files + + +class SuiteResolutionError(Exception): + """Raised when a suite selection cannot be resolved unambiguously.""" + + +@dataclass(frozen=True) +class ResolvedSuite: + """A provider configuration selected for one logical suite.""" + + config_path: Path + name: str + platform: str | None + + +def _normalize_name(value: str) -> str: + """Normalize CLI and filename spellings to catalog suite names.""" + normalized = value.strip().lower().replace("-", "_") + return "kubernetes" if normalized == "k8s" else normalized + + +def platform_vocabulary(configs_root: Path) -> frozenset[str]: + """Return declarable capabilities from canonical platform suite YAML.""" + platforms: set[str] = set() + for path in (configs_root / "suites").glob("*.yaml"): + try: + data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + except (OSError, yaml.YAMLError): + continue + platform = (data.get("tests") or {}).get("platform") if isinstance(data, dict) else None + if isinstance(platform, str) and platform: + platforms.add(_normalize_name(platform)) + return frozenset(platforms) + + +def parse_capabilities(value: str | None, configs_root: Path) -> set[str] | None: + """Parse a comma-separated capability context, rejecting aliases like compute.""" + if value is None: + return None + capabilities = {_normalize_name(item) for item in value.split(",") if item.strip()} + allowed = platform_vocabulary(configs_root) + unknown = sorted(capabilities - allowed) + if unknown: + raise SuiteResolutionError( + f"Unknown or non-declarable capabilities: {', '.join(unknown)}. " + f"Available capabilities: {', '.join(sorted(allowed)) or '(none)'}." + ) + return capabilities + + +def _raw_imports(config_path: Path) -> list[str]: + """Return import paths declared directly by a provider config.""" + try: + data: Any = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {} + except (OSError, yaml.YAMLError) as exc: + raise SuiteResolutionError(f"Failed to read {config_path}: {exc}") from exc + value = data.get("import") if isinstance(data, dict) else None + if isinstance(value, str): + return [value] + if isinstance(value, list): + return [item for item in value if isinstance(item, str)] + return [] + + +def _suite_name(config_path: Path, declarable: frozenset[str]) -> tuple[str, str | None]: + """Return the logical suite name and optional platform key for a config.""" + merged = merge_yaml_files([config_path]) + tests = merged.get("tests") or {} + platform = tests.get("platform") if isinstance(tests, dict) else None + if isinstance(platform, str) and _normalize_name(platform) in declarable: + normalized = _normalize_name(platform) + return normalized, normalized + + suite_imports = [Path(value).stem for value in _raw_imports(config_path) if "suites" in Path(value).parts] + if len(suite_imports) > 1: + raise SuiteResolutionError( + f"Config {config_path} imports multiple suites ({', '.join(suite_imports)}); use --config/-f." + ) + name = suite_imports[0] if suite_imports else config_path.stem + return _normalize_name(name), None + + +def resolve_suite(provider: str, suite: str, *, configs_root: Path) -> ResolvedSuite: + """Resolve exactly one provider config for a platform or plain suite.""" + config_dir = configs_root / "providers" / provider / "config" + if not config_dir.is_dir(): + raise SuiteResolutionError(f"Provider {provider!r} has no config directory at {config_dir}.") + + requested = _normalize_name(suite) + declarable = platform_vocabulary(configs_root) + classified = [ + (path, *_suite_name(path, declarable)) + for path in sorted(config_dir.glob("*.yaml")) + ] + matches = [item for item in classified if item[1] == requested] + if not matches: + available = sorted({name for _, name, _ in classified}) + raise SuiteResolutionError( + f"Provider {provider!r} has no {requested!r} suite. " + f"Available suites: {', '.join(available) or '(none)'}." + ) + if len(matches) > 1: + paths = ", ".join(str(path) for path, _, _ in matches) + raise SuiteResolutionError( + f"Provider {provider!r} has multiple {requested!r} suite configs ({paths}). " + "Disambiguate with --config/-f." + ) + path, name, platform = matches[0] + return ResolvedSuite(config_path=path, name=name, platform=platform) diff --git a/isvctl/src/isvctl/orchestrator/loop.py b/isvctl/src/isvctl/orchestrator/loop.py index eb36db07a..a8c15cd7a 100644 --- a/isvctl/src/isvctl/orchestrator/loop.py +++ b/isvctl/src/isvctl/orchestrator/loop.py @@ -35,6 +35,7 @@ SkipReason, State, ValidationEntry, + expand_capabilities, get_entry_phase, parse_validations, resolve_class_key, @@ -334,6 +335,26 @@ def _apply_step_validation_gates(steps: list[Any], released_tests: set[str] | No return gated_steps +def _apply_capability_step_gates( + steps: list[Any], + validation_entries: list[ValidationEntry], + capabilities: set[str] | None, +) -> list[Any]: + """Skip a step when every validation bound to it is requirement-filtered.""" + if capabilities is None: + return steps + expanded = expand_capabilities(capabilities) + gated_steps: list[Any] = [] + for step in steps: + bound_entries = [entry for entry in validation_entries if entry.step == step.name] + if bound_entries and all(not set(entry.requires).issubset(expanded) for entry in bound_entries): + logger.info("Skipping step '%s' because all bound validations are capability-filtered", step.name) + gated_steps.append(step.model_copy(update={"skip": True})) + else: + gated_steps.append(step) + return gated_steps + + class Orchestrator: """Orchestrates the full test lifecycle using step-based execution. @@ -361,6 +382,8 @@ def __init__( self._results: list[PhaseResult] = [] self._extra_pytest_args: list[str] | None = None self._include_labels: list[str] = [] + self._exclude_labels: list[str] = [] + self._capabilities: set[str] | None = None self._verbose: bool = False self._junitxml: str | None = None @@ -370,6 +393,8 @@ def run( teardown_on_failure: bool = True, extra_pytest_args: list[str] | None = None, include_labels: list[str] | None = None, + exclude_labels: list[str] | None = None, + capabilities: set[str] | None = None, verbose: bool = False, junitxml: str | None = None, ) -> OrchestratorResult: @@ -383,6 +408,8 @@ def run( - `-m kubernetes`: Run only validations whose labels include "kubernetes" (labels are mirrored as pytest marks) include_labels: Labels that selected validations must all contain. + exclude_labels: Labels that selected validations must not contain. + capabilities: Capability context used to filter validation requirements. verbose: Enable verbose output for validations junitxml: Path to write JUnit XML report for validations @@ -395,6 +422,8 @@ def run( self._results = [] self._extra_pytest_args = extra_pytest_args self._include_labels = include_labels or [] + self._exclude_labels = exclude_labels or [] + self._capabilities = capabilities self._verbose = verbose self._junitxml = junitxml @@ -472,6 +501,11 @@ def _run_steps_mode( logger.info(f"Including unreleased validations because {INCLUDE_UNRELEASED_ENV} is enabled") steps = _apply_step_validation_gates(steps, released_tests) + all_validations = {} + if self.config.tests and self.config.tests.validations: + all_validations = self.config.tests.validations + validation_entries = parse_validations(all_validations) + steps = _apply_capability_step_gates(steps, validation_entries, self._capabilities) logger.info(f"Configured phases: {config_phases}") @@ -503,10 +537,6 @@ def _run_steps_mode( step_phase = (step.phase or "setup").lower() self.context.set_step_phase(step.name, step_phase) - all_validations = {} - if self.config.tests and self.config.tests.validations: - all_validations = self.config.tests.validations - validation_entries = parse_validations(all_validations) resolved_validations_by_index: dict[int, ResolvedEntry] = {} exclude_labels: list[str] = [] @@ -517,7 +547,9 @@ def _run_steps_mode( skip_config_label_exclusions = bool(self._include_labels) or _has_explicit_pytest_selection( self._extra_pytest_args ) - resolution_exclude_labels = [] if skip_config_label_exclusions else exclude_labels + resolution_exclude_labels = set(self._exclude_labels) + if not skip_config_label_exclusions: + resolution_exclude_labels.update(exclude_labels) phase_results: list[PhaseResult] = [] overall_success = True @@ -598,7 +630,7 @@ def _run_steps_mode( phase_entries, requested_phase_names if Phase.ALL not in requested_phases else set(config_phases), set(self._include_labels), - set(resolution_exclude_labels), + resolution_exclude_labels, set(exclude_tests), released_tests, ) @@ -667,7 +699,7 @@ def _run_steps_mode( remaining_entries, requested_phase_names if Phase.ALL not in requested_phases else set(config_phases), set(self._include_labels), - set(resolution_exclude_labels), + resolution_exclude_labels, set(exclude_tests), released_tests, config_phases, @@ -792,6 +824,7 @@ def _resolve_validation_entries( exclude_tests=exclude_tests, released_tests=released_tests, render_context=self.context.get_accumulated_context(), + capabilities=self._capabilities, ) def _resolve_remaining_validation_entries( @@ -860,9 +893,10 @@ def _append_resolution_only_phase_results( def _detect_platform(self) -> str | None: """Detect platform from configuration. - Checks multiple locations for platform: + Checks execution identity without requiring a plain-suite axis key: 1. tests.platform (isvctl schema) 2. Root-level platform (legacy isvtest schema) + 3. The sole commands mapping key (plain suites) Returns: Platform string (e.g., 'kubernetes', 'slurm', 'bare_metal') or None @@ -875,6 +909,8 @@ def _detect_platform(self) -> str | None: # Fall back to root-level platform (legacy isvtest configs) elif hasattr(self.config, "model_extra") and self.config.model_extra: platform = self.config.model_extra.get("platform") + if not platform and len(self.config.commands) == 1: + platform = next(iter(self.config.commands)) if platform: # Normalize 'k8s' to 'kubernetes' diff --git a/isvctl/tests/test_cli_streams.py b/isvctl/tests/test_cli_streams.py index 87fb93afe..a165f093d 100644 --- a/isvctl/tests/test_cli_streams.py +++ b/isvctl/tests/test_cli_streams.py @@ -38,8 +38,10 @@ "name": "AlphaCheck", "description": "Alpha description", "labels": ["kubernetes"], - "module": "isvtest.validations.alpha", - "platforms": ["KUBERNETES"], + "source": "isvtest.validations.alpha", + "suite": "kubernetes", + "platform": "kubernetes", + "requires": [], }, ] @@ -66,20 +68,18 @@ def _write_config(tmp_path: Path) -> Path: return config -def test_dry_run_stdout_is_pure_json(tmp_path: Path) -> None: - """`test run --dry-run` emits only JSON on stdout; progress goes to stderr.""" +def test_dry_run_stdout_is_human_readable(tmp_path: Path) -> None: + """`test run --dry-run` summarizes the execution plan for operators.""" config = _write_config(tmp_path) result = runner.invoke(test_cli.app, ["run", "-f", str(config), "--no-upload", "--dry-run"]) assert result.exit_code == 0, result.output - # stdout must be parseable JSON with nothing else mixed in. - payload = json.loads(result.stdout) - assert payload["tests"]["platform"] == "kubernetes" - # Progress lives on stderr, never on the machine-readable stdout stream. + assert "Dry-run plan" in result.stdout + assert "Suite type: platform (kubernetes)" in result.stdout + assert "Checks: 0" in result.stdout assert "Validating configuration" in result.stderr assert "Validating configuration" not in result.stdout - assert "--- Dry Run: Configuration ---" not in result.stdout def test_catalog_list_json_stdout_is_pure_json() -> None: diff --git a/isvctl/tests/test_suite_resolution.py b/isvctl/tests/test_suite_resolution.py new file mode 100644 index 000000000..7c990341a --- /dev/null +++ b/isvctl/tests/test_suite_resolution.py @@ -0,0 +1,61 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for provider suite selection and capability parsing.""" + +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from isvctl.config.schema import RunConfig +from isvctl.config.suite_resolution import ( + SuiteResolutionError, + parse_capabilities, + resolve_suite, +) + + +def _write_catalog(root: Path) -> None: + """Write one platform suite, one plain suite, and provider imports.""" + suites = root / "suites" + configs = root / "providers" / "acme" / "config" + suites.mkdir(parents=True) + configs.mkdir(parents=True) + (suites / "k8s.yaml").write_text("tests:\n platform: kubernetes\n validations: {}\n") + (suites / "storage.yaml").write_text("tests:\n validations: {}\n") + (configs / "eks.yaml").write_text("import: ../../../suites/k8s.yaml\ncommands: {}\n") + (configs / "storage.yaml").write_text("import: ../../../suites/storage.yaml\ncommands: {}\n") + + +def test_one_suite_flag_resolves_platform_and_plain_suites(tmp_path: Path) -> None: + """The same selector resolves both suite kinds by effective YAML identity.""" + _write_catalog(tmp_path) + + platform = resolve_suite("acme", "kubernetes", configs_root=tmp_path) + plain = resolve_suite("acme", "storage", configs_root=tmp_path) + + assert platform.config_path.name == "eks.yaml" + assert platform.platform == "kubernetes" + assert plain.config_path.name == "storage.yaml" + assert plain.platform is None + + +def test_capabilities_use_catalog_vocabulary_and_reject_compute(tmp_path: Path) -> None: + """Compute remains requirement-only while omitted context disables filtering.""" + _write_catalog(tmp_path) + + assert parse_capabilities(None, tmp_path) is None + assert parse_capabilities("k8s", tmp_path) == {"kubernetes"} + with pytest.raises(SuiteResolutionError, match="non-declarable capabilities: compute"): + parse_capabilities("compute", tmp_path) + + +def test_platform_suites_reject_requires_and_compute_context() -> None: + """Platform placement is its obligation; it cannot declare check requirements.""" + validation = {"sample": {"checks": {"PlainCheck": {"requires": []}}}} + + with pytest.raises(ValidationError, match="requires is not allowed in platform suites"): + RunConfig.model_validate({"tests": {"platform": "kubernetes", "validations": validation}}) + with pytest.raises(ValidationError, match="compute is requirement-only"): + RunConfig.model_validate({"tests": {"platform": "compute", "validations": {}}}) diff --git a/isvctl/tests/test_test_cli_labels.py b/isvctl/tests/test_test_cli_labels.py index 701a23b78..866292ee1 100644 --- a/isvctl/tests/test_test_cli_labels.py +++ b/isvctl/tests/test_test_cli_labels.py @@ -62,8 +62,6 @@ def _write_provider_config(root: Path, provider: str, name: str, suite: str, pla {platform}: phases: [test] steps: [] -tests: - platform: {platform} """, encoding="utf-8", ) @@ -224,7 +222,7 @@ def test_provider_label_discovery_dispatches_each_matching_config( result = runner.invoke(test_cli.app, ["run", "--provider", "aws", "--label", "network", "--no-upload"]) assert result.exit_code == 0, result.output - assert [call["platform"] for call in _FakeOrchestrator.calls] == ["network", "observability"] + assert [call["platform"] for call in _FakeOrchestrator.calls] == [None, None] assert [call["working_dir"] for call in _FakeOrchestrator.calls] == [ network_config.parent, observability_config.parent, From 16915d3f88801961f352d687262fb683b5703524 Mon Sep 17 00:00:00 2001 From: Alexandre Begnoche Date: Wed, 22 Jul 2026 13:31:47 -0400 Subject: [PATCH 04/34] feat(catalog): migrate envelope to schema v2 requires model Build catalog entries from canonical suite YAML with source, suite, platform, and requires metadata. Bump schemaVersion to 2, replace platforms with capabilities in the envelope, and remove the legacy catalog_platforms registry module. Signed-off-by: Alexandre Begnoche --- isvctl/src/isvctl/cli/catalog.py | 15 +- isvctl/tests/test_catalog_cli.py | 17 +- isvreporter/src/isvreporter/client.py | 17 +- isvreporter/tests/test_catalog_upload.py | 37 ++-- isvtest/src/isvtest/catalog.py | 135 ++++++--------- isvtest/src/isvtest/catalog_platforms.py | 43 ----- isvtest/tests/test_catalog.py | 212 ++++++----------------- 7 files changed, 153 insertions(+), 323 deletions(-) delete mode 100644 isvtest/src/isvtest/catalog_platforms.py diff --git a/isvctl/src/isvctl/cli/catalog.py b/isvctl/src/isvctl/cli/catalog.py index 9df35bf88..7a55940f5 100644 --- a/isvctl/src/isvctl/cli/catalog.py +++ b/isvctl/src/isvctl/cli/catalog.py @@ -86,20 +86,17 @@ def list_cmd( ) table.add_column("Test", style="green", no_wrap=True) table.add_column("Test IDs", style="magenta", max_width=32) - table.add_column("Labels (Platforms)", style="dim", max_width=40) + table.add_column("Suite / Requirements", style="dim", max_width=40) table.add_column("Description") for entry in sorted(catalog_entries, key=lambda e: e["name"]): - labels = ", ".join(entry.get("labels") or []) - platforms = ", ".join(entry.get("platforms") or []) - if labels and platforms: - labels_platforms = f"{labels} ({platforms})" - else: - labels_platforms = labels or platforms + suite = entry.get("suite") or "-" + requirement = entry.get("platform") or ", ".join(entry.get("requires") or []) or "core" + suite_requirement = f"{suite} / {requirement}" table.add_row( entry["name"], ", ".join(entry.get("test_ids") or []) or "-", - labels_platforms or "-", + suite_requirement, entry.get("description") or "-", ) @@ -227,7 +224,7 @@ def push( isv_test_version=catalog_version, entries=catalog_entries, schema_version=document["schemaVersion"], - platforms=document["platforms"], + capabilities=document["capabilities"], ): print_progress(typer.style("[OK]", fg=typer.colors.GREEN) + " Catalog push complete") else: diff --git a/isvctl/tests/test_catalog_cli.py b/isvctl/tests/test_catalog_cli.py index 1767819a2..93f6a32ae 100644 --- a/isvctl/tests/test_catalog_cli.py +++ b/isvctl/tests/test_catalog_cli.py @@ -29,15 +29,19 @@ "name": "AlphaCheck", "description": "Alpha description", "labels": ["kubernetes"], - "module": "isvtest.validations.alpha", - "platforms": ["KUBERNETES"], + "source": "isvtest.validations.alpha", + "suite": "kubernetes", + "platform": "kubernetes", + "requires": [], }, { "name": "BetaCheck", "description": "", "labels": [], - "module": "isvtest.validations.beta", - "platforms": [], + "source": "isvtest.validations.beta", + "suite": "storage", + "platform": None, + "requires": ["compute"], }, ] @@ -73,11 +77,10 @@ def test_catalog_list_json() -> None: assert result.exit_code == 0, result.output payload = json.loads(result.output) - assert payload["schemaVersion"] == 1 + assert payload["schemaVersion"] == 2 assert payload["isvTestVersion"] == "1.2.3" assert payload["entries"] == _FAKE_ENTRIES - # The platform axis is derived from the real configs and drives the UI matrix. - assert "KUBERNETES" in payload["platforms"] + assert "kubernetes" in payload["capabilities"] def test_catalog_labels_table() -> None: diff --git a/isvreporter/src/isvreporter/client.py b/isvreporter/src/isvreporter/client.py index cb58c5c3a..bf7e70c4f 100644 --- a/isvreporter/src/isvreporter/client.py +++ b/isvreporter/src/isvreporter/client.py @@ -282,8 +282,8 @@ def upload_test_catalog( isv_test_version: str, entries: list[dict[str, Any]], *, - schema_version: int = 1, - platforms: list[str] | None = None, + schema_version: int = 2, + capabilities: list[str] | None = None, ) -> bool: """Upload test catalog for a suite version (idempotent per version). @@ -296,10 +296,9 @@ def upload_test_catalog( jwt_token: JWT access token isv_test_version: Test suite version string (e.g. "1.2.3") entries: List of catalog entry dicts with keys: - name, description, labels, module, platforms, test_ids + name, description, labels, source, suite, platform, requires, test_ids schema_version: Catalog document schema version. - platforms: Platform axis labels (e.g. ["KUBERNETES", "VM"]) - the - matrix columns; empty list when unknown. + capabilities: Declarable capability vocabulary. Returns: True if catalog was uploaded or already exists, False on error @@ -321,14 +320,16 @@ def upload_test_catalog( payload = { "schemaVersion": schema_version, "isvTestVersion": isv_test_version, - "platforms": platforms or [], + "capabilities": capabilities or [], "entries": [ { "name": e["name"], "description": e.get("description", ""), "labels": e.get("labels", []), - "module": e.get("module", ""), - "platforms": e.get("platforms", []), + "source": e.get("source", ""), + "suite": e.get("suite", ""), + "platform": e.get("platform"), + "requires": e.get("requires", []), "test_ids": e.get("test_ids", []), } for e in entries diff --git a/isvreporter/tests/test_catalog_upload.py b/isvreporter/tests/test_catalog_upload.py index fa09a411f..29e8c7684 100644 --- a/isvreporter/tests/test_catalog_upload.py +++ b/isvreporter/tests/test_catalog_upload.py @@ -47,10 +47,21 @@ def test_successful_upload(self, mock_urlopen: MagicMock) -> None: "name": "TestA", "description": "Test A", "labels": ["k8s"], - "module": "mod.a", + "source": "mod.a", + "suite": "kubernetes", + "platform": "kubernetes", + "requires": [], "test_ids": ["K8S06-01"], }, - {"name": "TestB", "description": "Test B", "labels": [], "module": "mod.b"}, + { + "name": "TestB", + "description": "Test B", + "labels": [], + "source": "mod.b", + "suite": "storage", + "platform": None, + "requires": ["compute"], + }, ] result = upload_test_catalog( @@ -70,9 +81,8 @@ def test_successful_upload(self, mock_urlopen: MagicMock) -> None: payload = json.loads(request.data.decode()) assert payload["isvTestVersion"] == "1.2.3" - # Envelope defaults when axis metadata is not supplied. - assert payload["schemaVersion"] == 1 - assert payload["platforms"] == [] + assert payload["schemaVersion"] == 2 + assert payload["capabilities"] == [] assert len(payload["entries"]) == 2 assert payload["entries"][0]["name"] == "TestA" assert payload["entries"][0]["labels"] == ["k8s"] @@ -181,12 +191,15 @@ def test_empty_optional_fields_use_defaults(self, mock_urlopen: MagicMock) -> No assert entry["description"] == "" assert entry["labels"] == [] assert "markers" not in entry - assert entry["module"] == "" + assert entry["source"] == "" + assert entry["suite"] == "" + assert entry["platform"] is None + assert entry["requires"] == [] assert entry["test_ids"] == [] @patch("isvreporter.client.urlopen") - def test_forwards_platform_axis_metadata(self, mock_urlopen: MagicMock) -> None: - """schema_version/platforms are sent in the top-level envelope.""" + def test_forwards_capability_vocabulary(self, mock_urlopen: MagicMock) -> None: + """Schema version and declarable capabilities are sent in the envelope.""" get_response = MagicMock() get_response.read.return_value = json.dumps([]).encode() get_response.__enter__ = MagicMock(return_value=get_response) @@ -202,14 +215,14 @@ def test_forwards_platform_axis_metadata(self, mock_urlopen: MagicMock) -> None: jwt_token="test-token", isv_test_version="1.2.3", entries=[{"name": "TestA"}], - schema_version=1, - platforms=["KUBERNETES", "VM"], + schema_version=2, + capabilities=["kubernetes", "vm"], ) request = mock_urlopen.call_args_list[1][0][0] payload = json.loads(request.data.decode()) - assert payload["schemaVersion"] == 1 - assert payload["platforms"] == ["KUBERNETES", "VM"] + assert payload["schemaVersion"] == 2 + assert payload["capabilities"] == ["kubernetes", "vm"] @patch("isvreporter.client.urlopen") def test_markers_field_is_not_forwarded(self, mock_urlopen: MagicMock) -> None: diff --git a/isvtest/src/isvtest/catalog.py b/isvtest/src/isvtest/catalog.py index 25d5a7474..fb54e4f30 100644 --- a/isvtest/src/isvtest/catalog.py +++ b/isvtest/src/isvtest/catalog.py @@ -19,13 +19,8 @@ discover_all_tests() and serializing each BaseValidation subclass's metadata. The catalog is version-keyed by the installed isvtest package version. -Platform tagging uses two sources (union of both): - 1. Config files - which checks appear in each isvctl/configs/suites/*.yaml - 2. Wiring labels - e.g. a check wired with labels: [bare_metal] implies the - BARE_METAL platform - -This ensures checks get a platform badge in the UI even when they only appear -in provider configs (e.g. Bm* checks that run on-host, not via SSH). +Suite placement and capability requirements come only from canonical +``isvctl/configs/suites/*.yaml`` wiring. """ import logging @@ -36,8 +31,8 @@ import yaml from isvreporter.version import get_version -from isvtest.catalog_platforms import LABEL_TO_PLATFORM, PLATFORM_CONFIGS from isvtest.core.discovery import discover_all_tests +from isvtest.core.resolution import resolve_class_key from isvtest.release_manifest import INCLUDE_UNRELEASED_ENV, load_released_test_filter logger = logging.getLogger(__name__) @@ -45,7 +40,7 @@ # Version of the catalog document envelope (schemaVersion field), bumped only # when the top-level shape changes - independent of the isvtest package version # (isvTestVersion), which tracks the test content. -CATALOG_SCHEMA_VERSION = 1 +CATALOG_SCHEMA_VERSION = 2 def _find_configs_dir() -> Path | None: @@ -196,42 +191,36 @@ def build_label_file_map() -> dict[str, set[str]]: return label_files -def _build_platform_map() -> dict[str, set[str]]: - """Build a mapping from test name to set of platform strings. - - Scans the canonical config files to determine which tests belong to - which platforms. - """ +def _build_suite_map() -> dict[str, dict[str, Any]]: + """Map globally unique wiring names to suite placement and requirements.""" configs_dir = _find_configs_dir() if not configs_dir: logger.warning("Could not locate isvctl/configs/ directory") return {} - test_to_platforms: dict[str, set[str]] = {} - - for platform, config_files in PLATFORM_CONFIGS.items(): - for config_file in config_files: - config_path = configs_dir / config_file - if not config_path.exists(): - logger.debug("Config not found: %s", config_path) - continue - - checks = _extract_checks_from_config(config_path) - for check_name in checks: - if check_name not in test_to_platforms: - test_to_platforms[check_name] = set() - test_to_platforms[check_name].add(platform) - - return test_to_platforms + suite_map: dict[str, dict[str, Any]] = {} + for config_path in sorted((configs_dir / "suites").glob("*.yaml")): + data = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {} + tests = data.get("tests") or {} + platform = tests.get("platform") if isinstance(tests, dict) else None + suite = str(platform) if isinstance(platform, str) and platform else config_path.stem.replace("-", "_") + for check_name, params in iter_config_checks(config_path): + if check_name in suite_map: + raise ValueError(f"Suite wiring name {check_name!r} is not globally unique") + requires = params.get("requires", []) + suite_map[check_name] = { + "suite": suite, + "platform": platform, + "requires": list(requires) if isinstance(requires, list) else [], + } + return suite_map def build_catalog(*, released_only: bool = True) -> list[dict[str, Any]]: """Discover all validation tests and return structured catalog entries. - Each entry includes a 'platforms' field derived from the config files, - indicating which platforms the test belongs to. Variant entries from - configs (e.g. K8sNimHelmWorkload-1b) are included as separate entries - inheriting metadata from their base class. + Each entry is one globally unique canonical suite wiring. Plain suites + carry ``requires`` while platform suites carry their ``platform`` key. Args: released_only: When True, omit tests that are not in the committed @@ -244,10 +233,12 @@ def build_catalog(*, released_only: bool = True) -> list[dict[str, Any]]: - labels: List of public label strings (e.g. ["kubernetes", "gpu"]) - test_ids: List of test-plan ids declared on the wiring, "N/A" excluded (e.g. ["SEC07-01"]); empty when only intentional gaps - - module: Fully qualified module path - - platforms: List of platform strings (e.g. ["KUBERNETES"]) + - source: Fully qualified Python source module + - suite: Logical suite name + - platform: Declared platform key for platform suites, otherwise null + - requires: Capability prerequisites for plain suites """ - platform_map = _build_platform_map() + suite_map = _build_suite_map() label_map = build_label_map() test_id_map = build_test_id_map() @@ -262,44 +253,19 @@ def build_catalog(*, released_only: bool = True) -> list[dict[str, Any]]: class_meta[cls.__name__] = { "description": getattr(cls, "description", "") or "", "labels": labels, - "module": cls.__module__, + "source": cls.__module__, } - # Infer platforms from labels only for checks not already covered by - # canonical configs. Some labels (for example "security") are useful - # pytest filters but are not reliable platform ownership signals once a - # check appears in a suite file. - if cls.__name__ not in platform_map: - for label in labels: - platform = LABEL_TO_PLATFORM.get(label) - if platform: - platform_map.setdefault(cls.__name__, set()).add(platform) catalog: list[dict[str, Any]] = [] - seen: set[str] = set() - # Add all discovered classes - for name, meta in class_meta.items(): - seen.add(name) - catalog.append( - { - "name": name, - "description": meta["description"], - "labels": meta["labels"], - "test_ids": sorted(test_id_map.get(name, set())), - "module": meta["module"], - "platforms": sorted(platform_map.get(name, [])), - } - ) - - # Add variant entries from configs that aren't base classes - for name, platforms in platform_map.items(): - if name in seen: + for name, placement in suite_map.items(): + base = resolve_class_key(name, class_meta) + if base is None: + logger.warning("Omitting suite wiring %s because no validation class resolves it", name) continue - base = name.split("-")[0] if "-" in name else name if name in excluded_names or base in excluded_names: continue - seen.add(name) - meta = class_meta.get(base, {}) + meta = class_meta[base] variant_suffix = name[len(base) :] if base != name else "" desc = meta.get("description", "") if variant_suffix: @@ -312,8 +278,8 @@ def build_catalog(*, released_only: bool = True) -> list[dict[str, Any]]: "description": desc, "labels": labels, "test_ids": test_ids, - "module": meta.get("module", ""), - "platforms": sorted(platforms), + "source": meta.get("source", ""), + **placement, } ) @@ -322,8 +288,12 @@ def build_catalog(*, released_only: bool = True) -> list[dict[str, Any]]: if released_tests is None: logger.info("Including unreleased tests in catalog because %s is enabled", INCLUDE_UNRELEASED_ENV) else: - omitted_names = sorted(entry["name"] for entry in catalog if entry["name"] not in released_tests) - catalog = [entry for entry in catalog if entry["name"] in released_tests] + omitted_names = sorted( + entry["name"] for entry in catalog if resolve_class_key(entry["name"], released_tests) is None + ) + catalog = [ + entry for entry in catalog if resolve_class_key(entry["name"], released_tests) is not None + ] if omitted_names: logger.info("Omitted %d unreleased tests from catalog", len(omitted_names)) logger.debug("Unreleased tests omitted from catalog: %s", ", ".join(omitted_names)) @@ -332,29 +302,24 @@ def build_catalog(*, released_only: bool = True) -> list[dict[str, Any]]: return catalog -def build_platform_axis() -> list[str]: - """Return the sorted platform-axis labels (the catalog's capability columns). - - Derived from :data:`PLATFORM_CONFIGS`, the canonical per-platform suite - mapping, so adding a platform there extends the axis automatically. The - values match each entry's ``platforms`` field (e.g. ``KUBERNETES``), so a - consumer can build the platform matrix straight from the catalog. - """ - return sorted(PLATFORM_CONFIGS.keys()) +def build_capability_vocabulary() -> list[str]: + """Return declarable capabilities derived from platform suite YAML.""" + suite_map = _build_suite_map() + return sorted({entry["platform"] for entry in suite_map.values() if entry["platform"]}) def catalog_document(entries: list[dict[str, Any]], version: str) -> dict[str, Any]: """Wrap catalog ``entries`` in the versioned upload/artifact envelope. - Adds the schema version, the isvtest package version, and the platform axis - list (see :func:`build_platform_axis`). The per-entry ``labels`` are + Adds the schema version, the isvtest package version, and the declarable + capability vocabulary. The per-entry ``labels`` are intentionally not summarized at the top level - a consumer can derive the label universe from the entries when needed. """ return { "schemaVersion": CATALOG_SCHEMA_VERSION, "isvTestVersion": version, - "platforms": build_platform_axis(), + "capabilities": build_capability_vocabulary(), "entries": entries, } diff --git a/isvtest/src/isvtest/catalog_platforms.py b/isvtest/src/isvtest/catalog_platforms.py deleted file mode 100644 index 5478dba3c..000000000 --- a/isvtest/src/isvtest/catalog_platforms.py +++ /dev/null @@ -1,43 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Canonical platform registry for the test catalog. - -Leaf module with no dependencies so lightweight consumers (for example the -validate-suites pre-commit hook) can read the registry without triggering -test discovery and its heavy transitive imports via :mod:`isvtest.catalog`. -""" - -# Configs that define the canonical test list per platform. -# Relative to the isvctl/configs/ directory. -PLATFORM_CONFIGS: dict[str, list[str]] = { - "BARE_METAL": ["suites/bare_metal.yaml"], - "CONTROL_PLANE": ["suites/control-plane.yaml"], - "IAM": ["suites/iam.yaml"], - "IMAGE_REGISTRY": ["suites/image-registry.yaml"], - "KUBERNETES": ["suites/k8s.yaml"], - "NETWORK": ["suites/network.yaml"], - "OBSERVABILITY": ["suites/observability.yaml"], - "SECURITY": ["suites/security.yaml"], - "SLURM": ["suites/slurm.yaml"], - "STORAGE": ["suites/storage.yaml"], - "VM": ["suites/vm.yaml"], -} - -# Maps wiring labels to platform strings so a check's platform can be inferred -# from its labels when it isn't otherwise tied to a platform. Derived from -# PLATFORM_CONFIGS: every platform's label is its lowercase name. Trait labels -# like "gpu", "ssh", "workload", and "slow" are intentionally not platforms. -LABEL_TO_PLATFORM: dict[str, str] = {platform.lower(): platform for platform in PLATFORM_CONFIGS} diff --git a/isvtest/tests/test_catalog.py b/isvtest/tests/test_catalog.py index ee9d33c65..209dae01d 100644 --- a/isvtest/tests/test_catalog.py +++ b/isvtest/tests/test_catalog.py @@ -19,8 +19,8 @@ from isvtest.catalog import ( CATALOG_SCHEMA_VERSION, + build_capability_vocabulary, build_catalog, - build_platform_axis, catalog_document, get_catalog_version, ) @@ -38,32 +38,20 @@ def run(self) -> None: class TestCatalogDocument: - """Tests for the platform axis and the versioned catalog envelope.""" - - def test_derives_platform_axis_from_configs(self) -> None: - """The platform axis lists every platform in PLATFORM_CONFIGS, sorted.""" - assert build_platform_axis() == [ - "BARE_METAL", - "CONTROL_PLANE", - "IAM", - "IMAGE_REGISTRY", - "KUBERNETES", - "NETWORK", - "OBSERVABILITY", - "SECURITY", - "SLURM", - "STORAGE", - "VM", - ] + """Tests for capability vocabulary and the versioned catalog envelope.""" + + def test_derives_capabilities_from_platform_suites(self) -> None: + """Only real platform suite keys are declarable capabilities.""" + assert build_capability_vocabulary() == ["bare_metal", "kubernetes", "slurm", "vm"] def test_catalog_document_wraps_entries_with_metadata(self) -> None: - """The envelope carries schema version, package version, and the platform axis.""" + """The envelope carries schema version, package version, and capabilities.""" entries = [{"name": "X", "labels": ["iam"]}] doc = catalog_document(entries, "1.2.3") assert doc["schemaVersion"] == CATALOG_SCHEMA_VERSION assert doc["isvTestVersion"] == "1.2.3" assert doc["entries"] == entries - assert doc["platforms"] == build_platform_axis() + assert doc["capabilities"] == build_capability_vocabulary() # The label universe is intentionally not summarized at the top level. assert "labels" not in doc @@ -71,46 +59,27 @@ def test_catalog_document_wraps_entries_with_metadata(self) -> None: class TestBuildCatalog: """Tests for build_catalog function.""" - def test_returns_list_of_dicts(self) -> None: - """Test that build_catalog returns a list of dicts.""" - catalog = build_catalog() - assert isinstance(catalog, list) - assert len(catalog) > 0 - for entry in catalog: - assert isinstance(entry, dict) - - def test_entries_have_required_keys(self) -> None: - """Test that each entry has the required keys.""" - catalog = build_catalog() - for entry in catalog: - assert "name" in entry - assert "description" in entry - assert "labels" in entry - assert "test_ids" in entry - assert "module" in entry - assert "markers" not in entry - - def test_entries_have_correct_types(self) -> None: - """Test that entry values have the correct types.""" - catalog = build_catalog() - for entry in catalog: - assert isinstance(entry["name"], str) - assert isinstance(entry["description"], str) - assert isinstance(entry["labels"], list) - assert isinstance(entry["module"], str) - - def test_no_duplicate_names(self) -> None: - """Test that there are no duplicate test names in the catalog.""" - catalog = build_catalog() - names = [e["name"] for e in catalog] + def test_entries_have_suite_contract(self) -> None: + """Every catalog row is one globally unique canonical suite wiring.""" + catalog = build_catalog(released_only=False) + names = [entry["name"] for entry in catalog] + assert catalog assert len(names) == len(set(names)) - - def test_known_tests_present(self) -> None: - """Test that some known validation tests appear in the catalog.""" - catalog = build_catalog() - names = {e["name"] for e in catalog} - assert "StepSuccessCheck" in names - assert "FieldExistsCheck" in names + for entry in catalog: + assert set(entry) == { + "name", + "description", + "labels", + "test_ids", + "source", + "suite", + "platform", + "requires", + } + assert isinstance(entry["source"], str) + assert isinstance(entry["requires"], list) + if entry["platform"]: + assert entry["requires"] == [] def test_extract_checks_supports_direct_dict_category_form(self, tmp_path) -> None: """Direct dict category wiring is included in catalog config scans.""" @@ -166,24 +135,18 @@ def test_entries_expose_wired_test_ids(self) -> None: assert all(isinstance(tid, str) for tid in entry["test_ids"]) assert "N/A" not in entry["test_ids"] - # Single mapping, and a duality unioned across the bm/vm suites. + # Single mappings retain their requirement and suite placement. assert by_name["MfaEnforcedCheck"]["test_ids"] == ["SEC07-01"] - assert by_name["GpuCheck"]["test_ids"] == ["BMAAS08-01", "VMAAS06-01"] - - def test_variant_test_ids_propagate_to_base(self) -> None: - """A variant's wired test_id surfaces on its base-class catalog entry.""" - catalog = build_catalog(released_only=False) - by_name = {e["name"]: e for e in catalog} - - assert by_name["StepSuccessCheck-delete_tenant"]["test_ids"] == ["CP10-01"] - assert "CP10-01" in by_name["StepSuccessCheck"]["test_ids"] + assert by_name["MfaEnforcedCheck"]["suite"] == "security" + assert by_name["MfaEnforcedCheck"]["requires"] == [] def test_released_only_filters_catalog(self) -> None: """Default catalog generation excludes tests not in the release manifest.""" with patch("isvtest.catalog.load_released_test_filter", return_value={"StepSuccessCheck"}): catalog = build_catalog() - assert {e["name"] for e in catalog} == {"StepSuccessCheck"} + assert catalog + assert all(entry["name"].startswith("StepSuccessCheck") for entry in catalog) def test_unreleased_env_includes_full_catalog(self) -> None: """When the release filter is disabled, default catalog generation includes all tests.""" @@ -191,8 +154,8 @@ def test_unreleased_env_includes_full_catalog(self) -> None: catalog = build_catalog() names = {e["name"] for e in catalog} - assert "StepSuccessCheck" in names - assert "FieldExistsCheck" in names + assert "StepSuccessCheck-storage_fixture_volume" in names + assert "FieldExistsCheck-storage_fixture_volume" in names def test_labels_are_lists_of_strings(self) -> None: """Test that labels are lists of strings.""" @@ -205,7 +168,16 @@ def test_catalog_emits_explicit_labels(self) -> None: """Per-wiring YAML labels are surfaced as catalog tag metadata.""" with ( patch("isvtest.catalog.discover_all_tests", return_value=[ExplicitLabelCatalogCheck]), - patch("isvtest.catalog._build_platform_map", return_value={}), + patch( + "isvtest.catalog._build_suite_map", + return_value={ + "ExplicitLabelCatalogCheck": { + "suite": "demo", + "platform": None, + "requires": ["compute"], + } + }, + ), patch( "isvtest.catalog.build_label_map", return_value={"ExplicitLabelCatalogCheck": {"accelerator", "long_running"}}, @@ -221,97 +193,19 @@ def test_catalog_emits_explicit_labels(self) -> None: "description": "Explicit labels", "labels": ["accelerator", "long_running"], "test_ids": [], - "module": __name__, - "platforms": [], + "source": __name__, + "suite": "demo", + "platform": None, + "requires": ["compute"], } ] - def test_modules_are_valid_python_paths(self) -> None: - """Test that module paths look like valid Python module paths.""" + def test_sources_are_valid_python_paths(self) -> None: + """Source paths remain useful implementation metadata, not a suite axis.""" catalog = build_catalog() for entry in catalog: - assert "." in entry["module"] - assert entry["module"].startswith("isvtest.") - - def test_suite_membership_overrides_label_platforms(self) -> None: - """Regression: trait labels must not add extra platform ownership. - - A check can carry labels like ``("security", "network")`` for pytest - filtering AND appear in a single suite YAML (e.g. ``security.yaml``). - ``_build_platform_map`` must use the suite as the source of truth and - skip label-derived platform inference in that case - otherwise the - UI shows phantom platform badges. - - DO NOT add per-check asserts to this test. It is a property test - that already covers every check in the catalog. If a new validation - breaks the invariant, the failure message names it. - """ - from isvtest.catalog import ( - LABEL_TO_PLATFORM, - PLATFORM_CONFIGS, - _extract_checks_from_config, - _find_configs_dir, - ) - - configs_dir = _find_configs_dir() - assert configs_dir is not None, "isvctl/configs/ not found" - - suite_platforms: dict[str, set[str]] = {} - for platform, files in PLATFORM_CONFIGS.items(): - for relpath in files: - for name in _extract_checks_from_config(configs_dir / relpath): - suite_platforms.setdefault(name, set()).add(platform) - - for entry in build_catalog(released_only=False): - name = entry["name"] - if name not in suite_platforms: - continue - label_platforms = {LABEL_TO_PLATFORM[label] for label in entry["labels"] if label in LABEL_TO_PLATFORM} - expected = suite_platforms[name] - actual = set(entry["platforms"]) - phantom = (label_platforms - expected) & actual - assert not phantom, ( - f"{name}: label-derived platforms {sorted(phantom)} leaked " - f"into catalog; expected exactly {sorted(expected)}, " - f"got {sorted(actual)}" - ) - assert actual == expected, ( - f"{name}: platforms should equal suite assignment {sorted(expected)}, got {sorted(actual)}" - ) - - def test_observability_label_infers_platform_for_unlisted_checks(self) -> None: - """Checks labelled with `observability` are tagged OBSERVABILITY when not in any suite.""" - - class ObservabilityLabelledCheck(BaseValidation): - description = "Observability check labelled but not in any suite" - - def run(self) -> None: - self.set_passed() - - ObservabilityLabelledCheck.__module__ = "isvtest.validations.fake" - - with ( - patch("isvtest.catalog.discover_all_tests", return_value=[ObservabilityLabelledCheck]), - patch("isvtest.catalog._build_platform_map", return_value={}), - patch( - "isvtest.catalog.build_label_map", - return_value={"ObservabilityLabelledCheck": {"observability"}}, - ), - patch("isvtest.catalog.build_test_id_map", return_value={}), - patch("isvtest.catalog.load_released_test_filter", return_value=None), - ): - catalog = build_catalog() - - assert catalog == [ - { - "name": "ObservabilityLabelledCheck", - "description": "Observability check labelled but not in any suite", - "labels": ["observability"], - "test_ids": [], - "module": "isvtest.validations.fake", - "platforms": ["OBSERVABILITY"], - } - ] + assert "." in entry["source"] + assert entry["source"].startswith("isvtest.") class TestGetCatalogVersion: From 694a7ebffec0b0cf2a185e97ea12443649f8b358 Mon Sep 17 00:00:00 2001 From: Alexandre Begnoche Date: Wed, 22 Jul 2026 13:31:54 -0400 Subject: [PATCH 05/34] refactor(suites): migrate plain suites to requires model Remove tests.platform from plain capability suites, add per-check requires from requires-worksheet, move kubernetes storage checks into storage.yaml with ensure_cluster wiring, add my-isv demo stub and AWS reference setup, and drop provider platform keys from plain-suite command groups. Signed-off-by: Alexandre Begnoche --- .../providers/aws/config/bare_metal.yaml | 16 +- .../providers/aws/config/observability.yaml | 1 - .../configs/providers/aws/config/storage.yaml | 7 + .../my-isv/config/observability.yaml | 1 - .../providers/my-isv/config/storage.yaml | 5 + .../my-isv/scripts/storage/ensure_cluster.py | 56 ++++ .../providers/nico/config/control-plane.yaml | 3 +- isvctl/configs/providers/nico/config/iam.yaml | 5 +- .../providers/nico/config/network.yaml | 5 +- .../providers/nico/config/observability.yaml | 1 - isvctl/configs/suites/bare_metal.yaml | 90 +++--- isvctl/configs/suites/control-plane.yaml | 25 +- isvctl/configs/suites/iam.yaml | 14 +- isvctl/configs/suites/image-registry.yaml | 58 ++-- isvctl/configs/suites/k8s.yaml | 209 +------------ isvctl/configs/suites/network.yaml | 38 ++- isvctl/configs/suites/observability.yaml | 21 +- isvctl/configs/suites/security.yaml | 26 +- isvctl/configs/suites/storage.yaml | 277 +++++++++++++++++- isvctl/configs/suites/vm.yaml | 64 ++-- 20 files changed, 570 insertions(+), 352 deletions(-) create mode 100644 isvctl/configs/providers/my-isv/scripts/storage/ensure_cluster.py diff --git a/isvctl/configs/providers/aws/config/bare_metal.yaml b/isvctl/configs/providers/aws/config/bare_metal.yaml index 6fe86a349..c38a4aaea 100644 --- a/isvctl/configs/providers/aws/config/bare_metal.yaml +++ b/isvctl/configs/providers/aws/config/bare_metal.yaml @@ -322,7 +322,7 @@ tests: serial_console: step: serial_console checks: - - SerialConsoleCheck: {} + - SerialConsoleCheck-bm_serial_console: {} # GPU stress - AWS metal has 8 GPUs gpu_stress: @@ -352,14 +352,14 @@ tests: start_gpu: step: start_instance checks: - GpuCheck: + GpuCheck-bm_start_gpu: expected_gpus: 8 # Power-cycle GPU - AWS metal has 8 GPUs power_cycle_gpu: step: power_cycle_instance checks: - GpuCheck: + GpuCheck-bm_power_cycle_gpu: expected_gpus: 8 # NVLink - AWS metal has 8 GPUs @@ -373,16 +373,16 @@ tests: image_installed: step: verify_image checks: - StepSuccessCheck: {} - FieldExistsCheck: + StepSuccessCheck-bm_image_installed: {} + FieldExistsCheck-bm_image_installed: fields: ["instance_id", "image_id", "image_name", "instance_state"] - InstanceStateCheck: + InstanceStateCheck-bm_image_installed: expected_state: "running" # AWS-specific: verify install config can provision BM config_installable: step: verify_config checks: - StepSuccessCheck: {} - FieldExistsCheck: + StepSuccessCheck-bm_config_installable: {} + FieldExistsCheck-bm_config_installable: fields: ["config_id", "config_name", "dry_run_passed"] diff --git a/isvctl/configs/providers/aws/config/observability.yaml b/isvctl/configs/providers/aws/config/observability.yaml index 7aeda1534..8573f3bb7 100644 --- a/isvctl/configs/providers/aws/config/observability.yaml +++ b/isvctl/configs/providers/aws/config/observability.yaml @@ -382,7 +382,6 @@ commands: timeout: 900 tests: - platform: observability cluster_name: "aws-observability-validation" description: "AWS observability validation for VPC Flow Logs and host syslogs" diff --git a/isvctl/configs/providers/aws/config/storage.yaml b/isvctl/configs/providers/aws/config/storage.yaml index 80e549324..d88620ca3 100644 --- a/isvctl/configs/providers/aws/config/storage.yaml +++ b/isvctl/configs/providers/aws/config/storage.yaml @@ -51,6 +51,13 @@ commands: storage: phases: ["setup", "test", "teardown"] steps: + # Idempotent reference implementation: the EKS setup reuses Terraform + # state and emits kubeconfig + CSI StorageClass names. + - name: ensure_cluster + phase: setup + command: "../scripts/eks/setup.sh" + timeout: 5400 + # ─── Block volume (EBS) ──────────────────────────────────────── - name: launch_instance phase: setup diff --git a/isvctl/configs/providers/my-isv/config/observability.yaml b/isvctl/configs/providers/my-isv/config/observability.yaml index 50a8bf095..f5325c036 100644 --- a/isvctl/configs/providers/my-isv/config/observability.yaml +++ b/isvctl/configs/providers/my-isv/config/observability.yaml @@ -255,7 +255,6 @@ commands: timeout: 60 tests: - platform: observability cluster_name: "my-isv-observability-validation" description: "my-isv observability validation (generic stubs, dummy overrides)" diff --git a/isvctl/configs/providers/my-isv/config/storage.yaml b/isvctl/configs/providers/my-isv/config/storage.yaml index e311a70c9..d5579e454 100644 --- a/isvctl/configs/providers/my-isv/config/storage.yaml +++ b/isvctl/configs/providers/my-isv/config/storage.yaml @@ -49,6 +49,11 @@ commands: storage: phases: ["setup", "test", "teardown"] steps: + - name: ensure_cluster + phase: setup + command: "python ../scripts/storage/ensure_cluster.py" + timeout: 60 + - name: launch_instance phase: setup command: "python ../scripts/vm/launch_instance.py" diff --git a/isvctl/configs/providers/my-isv/scripts/storage/ensure_cluster.py b/isvctl/configs/providers/my-isv/scripts/storage/ensure_cluster.py new file mode 100644 index 000000000..12441400f --- /dev/null +++ b/isvctl/configs/providers/my-isv/scripts/storage/ensure_cluster.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Ensure a Kubernetes cluster with the provider's CSI drivers is available.""" + +import json +import os +import sys +from typing import Any + +DEMO_MODE = os.environ.get("ISVCTL_DEMO_MODE") == "1" + + +def main() -> int: + """Emit the provider-neutral cluster and StorageClass contract.""" + result: dict[str, Any] = { + "success": False, + "platform": "kubernetes", + "kubeconfig_path": "", + "csi": { + "block_storage_class": "", + "shared_fs_storage_class": "", + "nfs_storage_class": "", + "static_volume_handle": "", + "static_driver_name": "", + "static_volume_az": "", + }, + } + + # TODO: Idempotently reuse or create a cluster with your CSI drivers, + # then return its kubeconfig and StorageClass names using this contract. + if DEMO_MODE: + result.update( + { + "success": True, + "kubeconfig_path": "/tmp/isvctl-demo-kubeconfig", + "csi": { + "block_storage_class": "demo-block", + "shared_fs_storage_class": "demo-shared", + "nfs_storage_class": "demo-nfs", + "static_volume_handle": "", + "static_driver_name": "", + "static_volume_az": "", + }, + } + ) + else: + result["error"] = "Not implemented - ensure a Kubernetes cluster with CSI installed" + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/providers/nico/config/control-plane.yaml b/isvctl/configs/providers/nico/config/control-plane.yaml index 6ae155635..b70369c2f 100644 --- a/isvctl/configs/providers/nico/config/control-plane.yaml +++ b/isvctl/configs/providers/nico/config/control-plane.yaml @@ -45,7 +45,6 @@ commands: timeout: 120 tests: - platform: control_plane cluster_name: "nico-control-plane-validation" description: "Verify NICo API health" @@ -58,7 +57,7 @@ tests: api_health: step: check_api checks: - FieldExistsCheck: + FieldExistsCheck-cp_api_health: test_id: "N/A" fields: ["account_id", "tests"] FieldValueCheck: diff --git a/isvctl/configs/providers/nico/config/iam.yaml b/isvctl/configs/providers/nico/config/iam.yaml index 2738919c3..18afdc621 100644 --- a/isvctl/configs/providers/nico/config/iam.yaml +++ b/isvctl/configs/providers/nico/config/iam.yaml @@ -44,7 +44,6 @@ commands: timeout: 120 tests: - platform: iam cluster_name: "nico-iam-validation" description: "Verify NICo credential readiness" @@ -57,10 +56,10 @@ tests: credential_readiness: step: check_credentials checks: - FieldExistsCheck: + FieldExistsCheck-iam_credential_readiness: test_id: "N/A" fields: ["account_id", "authenticated", "tests"] - StepSuccessCheck: + StepSuccessCheck-iam_credential_readiness: test_id: "N/A" exclude: diff --git a/isvctl/configs/providers/nico/config/network.yaml b/isvctl/configs/providers/nico/config/network.yaml index f9764a3a8..658cc26eb 100644 --- a/isvctl/configs/providers/nico/config/network.yaml +++ b/isvctl/configs/providers/nico/config/network.yaml @@ -92,7 +92,6 @@ commands: timeout: 120 tests: - platform: network cluster_name: "nico-network-validation" description: "Verify NICo VPC and subnet inventory" @@ -119,7 +118,7 @@ tests: network_connectivity: step: network_connectivity checks: - StepSuccessCheck: + StepSuccessCheck-network_connectivity: test_id: "N/A" FieldValueCheck: test_id: "N/A" @@ -129,7 +128,7 @@ tests: traffic_validation: step: traffic_validation checks: - StepSuccessCheck: + StepSuccessCheck-network_traffic_validation: test_id: "N/A" FieldValueCheck: test_id: "N/A" diff --git a/isvctl/configs/providers/nico/config/observability.yaml b/isvctl/configs/providers/nico/config/observability.yaml index 07ed70878..c19b1562e 100644 --- a/isvctl/configs/providers/nico/config/observability.yaml +++ b/isvctl/configs/providers/nico/config/observability.yaml @@ -84,7 +84,6 @@ commands: timeout: 60 tests: - platform: observability cluster_name: "nico-observability-validation" description: "NICo observability validation for UFM event logs and switch log evidence" diff --git a/isvctl/configs/suites/bare_metal.yaml b/isvctl/configs/suites/bare_metal.yaml index 35276c435..553176271 100644 --- a/isvctl/configs/suites/bare_metal.yaml +++ b/isvctl/configs/suites/bare_metal.yaml @@ -51,7 +51,7 @@ tests: setup_checks: step: launch_instance checks: - InstanceStateCheck: + InstanceStateCheck-bm_setup: test_id: "N/A" labels: ["bare_metal"] expected_state: "running" @@ -59,7 +59,7 @@ tests: list_instances: step: list_instances checks: - InstanceListCheck: + InstanceListCheck-bm_list_instances: # CNP01-11 (list nodes) is a VMaaS plan item; the BMaaS plan has no equivalent list-nodes item. test_id: "N/A" labels: ["bare_metal"] @@ -68,7 +68,7 @@ tests: tag_checks: step: verify_tags checks: - InstanceTagCheck: + InstanceTagCheck-bm_tag: test_id: "CNP05-01" labels: ["bare_metal", "min_req"] required_keys: ["Name", "CreatedBy"] @@ -76,7 +76,7 @@ tests: topology_placement: step: topology_placement checks: - StepSuccessCheck: + StepSuccessCheck-bm_topology_placement: test_id: "N/A" labels: ["bare_metal"] TopologyPlacementCheck: @@ -86,7 +86,7 @@ tests: serial_console: step: serial_console checks: - SerialConsoleCheck: + SerialConsoleCheck-bm_serial_console: test_id: "CNP06-01" labels: ["bare_metal", "min_req"] SerialConsoleRetentionCheck: @@ -96,14 +96,14 @@ tests: cloud_init: step: launch_instance checks: - CloudInitCheck: + CloudInitCheck-bm_cloud_init: test_id: "BOOT02-01" labels: ["bare_metal", "min_req", "ssh"] instance_info: step: describe_instance checks: - InstanceStateCheck: + InstanceStateCheck-bm_instance_info: test_id: "N/A" labels: ["bare_metal"] expected_state: "running" @@ -111,10 +111,10 @@ tests: ssh: step: describe_instance checks: - ConnectivityCheck: + ConnectivityCheck-bm_ssh: test_id: "BMAAS03-01" labels: ["bare_metal", "ssh"] - OsCheck: + OsCheck-bm_ssh: # No BMaaS plan item for a host-OS image check (VMAAS-XX-05 is VM-only). test_id: "N/A" labels: ["bare_metal", "ssh"] @@ -123,7 +123,7 @@ tests: gpu: step: describe_instance checks: - GpuCheck: + GpuCheck-bm_gpu: test_id: "BMAAS08-01" labels: ["bare_metal", "ssh", "gpu"] expected_gpus: 8 @@ -133,14 +133,14 @@ tests: checks: # vCPU pinning / PCI passthrough / host-software are VMAAS plan items; # the BMaaS plan has no equivalent, so these are intentional gaps here. - VcpuPinningCheck: + VcpuPinningCheck-bm_host_os: test_id: "N/A" labels: ["bare_metal", "ssh"] - PciBusCheck: + PciBusCheck-bm_host_os: test_id: "N/A" labels: ["bare_metal", "ssh", "gpu"] expected_gpus: 8 - HostSoftwareCheck: + HostSoftwareCheck-bm_host_os: test_id: "N/A" labels: ["bare_metal", "ssh"] @@ -150,7 +150,7 @@ tests: host_status_logs: step: host_status_log checks: - StepSuccessCheck: + StepSuccessCheck-bm_host_status_logs: test_id: "N/A" labels: ["bare_metal"] BmHostStatusLog: @@ -211,38 +211,38 @@ tests: step: describe_instance checks: # No BMaaS plan item for a host driver/kernel check (VMAAS-XX-07 is VM-only). - DriverCheck: + DriverCheck-bm_driver_info: test_id: "N/A" labels: ["bare_metal", "ssh", "gpu"] cpu_info: step: describe_instance checks: - CpuInfoCheck: + CpuInfoCheck-bm_cpu_info: test_id: "N/A" labels: ["bare_metal", "ssh"] container_runtime: step: describe_instance checks: - ContainerRuntimeCheck: + ContainerRuntimeCheck-bm_container_runtime: test_id: "N/A" labels: ["bare_metal", "ssh", "gpu", "workload", "slow"] stop_checks: step: stop_instance checks: - InstanceStopCheck: + InstanceStopCheck-bm_stop: test_id: "CNP01-07" labels: ["bare_metal", "min_req"] start_checks: step: start_instance checks: - InstanceStartCheck: + InstanceStartCheck-bm_start: test_id: "CNP01-08" labels: ["bare_metal", "min_req"] - StableIdentifierCheck: + StableIdentifierCheck-bm_start: test_id: "CNP08-01" labels: ["bare_metal", "min_req"] reference_id: "{{steps.launch_instance.instance_id}}" @@ -255,12 +255,12 @@ tests: step: start_instance checks: # ConnectivityCheck -> BMAAS-XX-03, OsCheck -> N/A (declared in the "ssh" section) - ConnectivityCheck: + ConnectivityCheck-bm_start_ssh: test_id: "BMAAS03-01" labels: ["bare_metal", "ssh"] host: "{{steps.describe_instance.public_ip}}" key_file: "{{steps.describe_instance.key_file}}" - OsCheck: + OsCheck-bm_start_ssh: test_id: "N/A" labels: ["bare_metal", "ssh"] host: "{{steps.describe_instance.public_ip}}" @@ -271,7 +271,7 @@ tests: step: start_instance checks: # GpuCheck -> BMAAS-XX-08 (declared in the "gpu" section) - GpuCheck: + GpuCheck-bm_start_gpu: test_id: "BMAAS08-01" labels: ["bare_metal", "ssh", "gpu"] host: "{{steps.describe_instance.public_ip}}" @@ -280,12 +280,12 @@ tests: reboot_checks: step: reboot_instance checks: - InstanceRebootCheck: + InstanceRebootCheck-bm_reboot: test_id: "CNP01-05" labels: ["bare_metal", "min_req"] max_uptime: 600 # StableIdentifierCheck -> CNP08-01 (declared in the "start_checks" section) - StableIdentifierCheck: + StableIdentifierCheck-bm_reboot: test_id: "CNP08-01" labels: ["bare_metal", "min_req"] reference_id: "{{steps.launch_instance.instance_id}}" @@ -293,7 +293,7 @@ tests: reboot_state: step: reboot_instance checks: - InstanceStateCheck: + InstanceStateCheck-bm_reboot_state: test_id: "N/A" labels: ["bare_metal"] expected_state: "running" @@ -302,12 +302,12 @@ tests: step: reboot_instance checks: # ConnectivityCheck -> BMAAS-XX-03, OsCheck -> N/A (declared in the "ssh" section) - ConnectivityCheck: + ConnectivityCheck-bm_reboot_ssh: test_id: "BMAAS03-01" labels: ["bare_metal", "ssh"] host: "{{steps.describe_instance.public_ip}}" key_file: "{{steps.describe_instance.key_file}}" - OsCheck: + OsCheck-bm_reboot_ssh: test_id: "N/A" labels: ["bare_metal", "ssh"] host: "{{steps.describe_instance.public_ip}}" @@ -318,7 +318,7 @@ tests: step: reboot_instance checks: # GpuCheck -> BMAAS-XX-08 (declared in the "gpu" section) - GpuCheck: + GpuCheck-bm_reboot_gpu: test_id: "BMAAS08-01" labels: ["bare_metal", "ssh", "gpu"] host: "{{steps.describe_instance.public_ip}}" @@ -333,7 +333,7 @@ tests: labels: ["bare_metal", "min_req"] max_recovery_time: 900 # StableIdentifierCheck -> CNP08-01 (declared in the "start_checks" section) - StableIdentifierCheck: + StableIdentifierCheck-bm_power_cycle: test_id: "CNP08-01" labels: ["bare_metal", "min_req"] reference_id: "{{steps.launch_instance.instance_id}}" @@ -341,7 +341,7 @@ tests: power_cycle_state: step: power_cycle_instance checks: - InstanceStateCheck: + InstanceStateCheck-bm_power_cycle_state: test_id: "N/A" labels: ["bare_metal"] expected_state: "running" @@ -350,10 +350,10 @@ tests: step: power_cycle_instance checks: # ConnectivityCheck -> BMAAS-XX-03, OsCheck -> N/A (declared in the "ssh" section) - ConnectivityCheck: + ConnectivityCheck-bm_power_cycle_ssh: test_id: "BMAAS03-01" labels: ["bare_metal", "ssh"] - OsCheck: + OsCheck-bm_power_cycle_ssh: test_id: "N/A" labels: ["bare_metal", "ssh"] expected_os: "ubuntu" @@ -362,7 +362,7 @@ tests: step: power_cycle_instance checks: # GpuCheck -> BMAAS-XX-08 (declared in the "gpu" section) - GpuCheck: + GpuCheck-bm_power_cycle_gpu: test_id: "BMAAS08-01" labels: ["bare_metal", "ssh", "gpu"] expected_gpus: 8 @@ -371,11 +371,11 @@ tests: reinstall_state: step: reinstall_instance checks: - InstanceStateCheck: + InstanceStateCheck-bm_reinstall_state: test_id: "N/A" labels: ["bare_metal"] expected_state: "running" - StableIdentifierCheck: + StableIdentifierCheck-bm_reinstall_state: test_id: "CNP08-01" labels: ["bare_metal", "min_req"] reference_id: "{{steps.launch_instance.instance_id}}" @@ -384,10 +384,10 @@ tests: step: reinstall_instance checks: # ConnectivityCheck -> BMAAS-XX-03, OsCheck -> N/A (declared in the "ssh" section) - ConnectivityCheck: + ConnectivityCheck-bm_reinstall_ssh: test_id: "BMAAS03-01" labels: ["bare_metal", "ssh"] - OsCheck: + OsCheck-bm_reinstall_ssh: test_id: "N/A" labels: ["bare_metal", "ssh"] expected_os: "ubuntu" @@ -396,28 +396,28 @@ tests: step: reinstall_instance checks: # GpuCheck -> BMAAS-XX-08 (declared in the "gpu" section) - GpuCheck: + GpuCheck-bm_reinstall_gpu: test_id: "BMAAS08-01" labels: ["bare_metal", "ssh", "gpu"] nim_health: step: deploy_nim checks: - NimHealthCheck: + NimHealthCheck-bm_nim_health: test_id: "BMAAS10-01" labels: ["bare_metal", "gpu", "min_req", "slow", "ssh", "workload"] nim_models: step: deploy_nim checks: - NimModelCheck: + NimModelCheck-bm_nim_models: test_id: "BMAAS10-01" labels: ["bare_metal", "gpu", "min_req", "slow", "ssh", "workload"] nim_inference: step: deploy_nim checks: - NimInferenceCheck: + NimInferenceCheck-bm_nim_inference: test_id: "BMAAS10-01" labels: ["bare_metal", "gpu", "min_req", "slow", "ssh", "workload"] prompt: "What is CUDA?" @@ -426,21 +426,21 @@ tests: nim_teardown: step: teardown_nim checks: - StepSuccessCheck: + StepSuccessCheck-bm_nim_teardown: test_id: "N/A" labels: ["bare_metal"] teardown_checks: step: teardown checks: - StepSuccessCheck: + StepSuccessCheck-bm_teardown: test_id: "N/A" labels: ["bare_metal"] sanitization: step: verify_teardown checks: - StepSuccessCheck: + StepSuccessCheck-bm_sanitization: test_id: "SEC21-01" labels: ["bare_metal", "min_req", "security"] diff --git a/isvctl/configs/suites/control-plane.yaml b/isvctl/configs/suites/control-plane.yaml index 53a6ef492..a205e8be2 100644 --- a/isvctl/configs/suites/control-plane.yaml +++ b/isvctl/configs/suites/control-plane.yaml @@ -42,7 +42,6 @@ version: "1.0" tests: - platform: control_plane cluster_name: "control-plane-validation" description: "Control plane: API health, access key lifecycle, tenant lifecycle" @@ -57,17 +56,19 @@ tests: api_health: step: check_api checks: - FieldExistsCheck: + FieldExistsCheck-cp_api_health: test_id: "N/A" labels: ["control_plane"] + requires: [] fields: ["account_id", "tests"] # DATASVC-XX-01 (S3-compatible API w/ authenticated endpoints): # authenticated-endpoint half - a successful authenticated check_api # response. The data-path half is the CrudOperationsCheck in the # "s3_object_lifecycle" section, which declares the same test_id. - FieldValueCheck: + FieldValueCheck-cp_api_health: test_id: "DATASVC01-01" labels: ["control_plane", "min_req"] + requires: [] field: "success" expected: true @@ -76,10 +77,12 @@ tests: AccessKeyCreatedCheck: test_id: "CP05-01" labels: ["control_plane", "iam"] + requires: [] step: create_access_key TenantCreatedCheck: test_id: "CP07-01" labels: ["control_plane", "iam"] + requires: [] step: create_tenant access_key_lifecycle: @@ -87,14 +90,17 @@ tests: AccessKeyAuthenticatedCheck: test_id: "CP05-01" labels: ["control_plane", "iam"] + requires: [] step: test_access_key AccessKeyDisabledCheck: test_id: "CP06-01" labels: ["control_plane", "iam", "min_req"] + requires: [] step: disable_access_key AccessKeyRejectedCheck: test_id: "CP06-01" labels: ["control_plane", "iam", "min_req"] + requires: [] step: verify_key_rejected tenant_lifecycle: @@ -102,10 +108,12 @@ tests: TenantListedCheck: test_id: "CP08-01" labels: ["control_plane", "iam"] + requires: [] step: list_tenants TenantInfoCheck: test_id: "CP09-01" labels: ["control_plane", "iam"] + requires: [] step: get_tenant s3_object_lifecycle: @@ -113,16 +121,19 @@ tests: checks: # Data-path half of DATASVC-XX-01 (object put/get/delete); the # authenticated-endpoint half is the FieldValueCheck in "api_health". - StepSuccessCheck: + StepSuccessCheck-cp_s3_object_lifecycle: test_id: "N/A" labels: ["control_plane"] - FieldExistsCheck: + requires: [] + FieldExistsCheck-cp_s3_object_lifecycle: test_id: "N/A" labels: ["control_plane"] + requires: [] fields: ["bucket_name", "object_key", "operations"] - CrudOperationsCheck: + CrudOperationsCheck-cp_s3_object_lifecycle: test_id: "DATASVC01-01" labels: ["control_plane", "min_req"] + requires: [] operations: ["put", "get", "delete"] teardown_checks: @@ -132,10 +143,12 @@ tests: # CP05-01 (create) and CP06-01 (disable). test_id: "N/A" labels: ["control_plane"] + requires: [] step: delete_access_key StepSuccessCheck-delete_tenant: test_id: "CP10-01" labels: ["control_plane"] + requires: [] step: delete_tenant exclude: diff --git a/isvctl/configs/suites/iam.yaml b/isvctl/configs/suites/iam.yaml index 4ac3be7e1..981f52b95 100644 --- a/isvctl/configs/suites/iam.yaml +++ b/isvctl/configs/suites/iam.yaml @@ -37,7 +37,6 @@ version: "1.0" tests: cluster_name: "iam-validation" description: "IAM user lifecycle validation" - platform: iam settings: # Provider-specific - override in your provider config @@ -51,13 +50,15 @@ tests: setup_checks: step: create_user checks: - FieldExistsCheck: + FieldExistsCheck-iam_setup: test_id: "IAM01-01" labels: ["iam", "min_req"] + requires: [] fields: ["username", "access_key_id"] - StepSuccessCheck: + StepSuccessCheck-iam_setup: test_id: "N/A" labels: ["iam"] + requires: [] credentials: step: test_credentials @@ -65,16 +66,19 @@ tests: IamCredentialAccessCheck: test_id: "IAM03-01" labels: ["iam"] - StepSuccessCheck: + requires: [] + StepSuccessCheck-iam_credentials: test_id: "N/A" labels: ["iam"] + requires: [] teardown_checks: step: teardown checks: - StepSuccessCheck: + StepSuccessCheck-iam_teardown: test_id: "N/A" labels: ["iam"] + requires: [] exclude: labels: [] diff --git a/isvctl/configs/suites/image-registry.yaml b/isvctl/configs/suites/image-registry.yaml index c400cfb3d..9d20c8264 100644 --- a/isvctl/configs/suites/image-registry.yaml +++ b/isvctl/configs/suites/image-registry.yaml @@ -37,7 +37,6 @@ version: "1.0" tests: cluster_name: "image-registry-validation" description: "Image registry validation tests" - platform: image_registry settings: # Provider-specific - override in your provider config @@ -54,107 +53,126 @@ tests: image_upload: step: upload_image checks: - StepSuccessCheck: + StepSuccessCheck-img_image_upload: test_id: "N/A" labels: ["image_registry"] - FieldExistsCheck: + requires: [] + FieldExistsCheck-img_image_upload: test_id: "BOOT01-01" labels: ["image_registry", "min_req"] + requires: [] fields: ["image_id", "storage_bucket", "disk_ids"] # --- Image CRUD --- image_crud: step: crud_image checks: - StepSuccessCheck: + StepSuccessCheck-img_image_crud: test_id: "N/A" labels: ["image_registry"] - FieldExistsCheck: + requires: [] + FieldExistsCheck-img_image_crud: test_id: "N/A" labels: ["image_registry"] + requires: [] fields: ["image_id", "operations"] - CrudOperationsCheck: + CrudOperationsCheck-img_image_crud: test_id: "BOOT03-02" labels: ["image_registry", "min_req"] + requires: [] operations: ["get", "list", "create", "delete"] # --- VM Launch from Image --- vm_from_image: step: launch_instance checks: - StepSuccessCheck: + StepSuccessCheck-img_vm_from_image: test_id: "BOOT01-05" labels: ["image_registry", "min_req"] - FieldExistsCheck: + requires: [vm] + FieldExistsCheck-img_vm_from_image: test_id: "N/A" labels: ["image_registry"] + requires: [vm] fields: ["instance_id", "public_ip", "key_path"] - InstanceStateCheck: + InstanceStateCheck-img_vm_from_image: test_id: "N/A" labels: ["image_registry"] + requires: [vm] expected_state: "running" vm_ssh: step: launch_instance checks: - ConnectivityCheck: + ConnectivityCheck-img_vm_ssh: test_id: "N/A" labels: ["image_registry", "ssh"] - OsCheck: + requires: [vm] + OsCheck-img_vm_ssh: test_id: "N/A" labels: ["image_registry", "ssh"] + requires: [vm] expected_os: "ubuntu" # --- OS Install Config CRUD --- install_config_crud: step: crud_install_config checks: - StepSuccessCheck: + StepSuccessCheck-img_install_config_crud: test_id: "N/A" labels: ["image_registry"] - FieldExistsCheck: + requires: [] + FieldExistsCheck-img_install_config_crud: test_id: "BOOT01-02" labels: ["image_registry"] + requires: [] fields: ["config_id", "config_name", "operations"] # --- OS Image on BMaaS --- bm_from_image: step: install_image_bm checks: - StepSuccessCheck: + StepSuccessCheck-img_bm_from_image: test_id: "BOOT01-03" labels: ["image_registry", "min_req"] - FieldExistsCheck: + requires: [bare_metal] + FieldExistsCheck-img_bm_from_image: test_id: "N/A" labels: ["image_registry"] + requires: [bare_metal] fields: ["instance_id", "image_id", "instance_state"] - InstanceStateCheck: + InstanceStateCheck-img_bm_from_image: test_id: "N/A" labels: ["image_registry"] + requires: [bare_metal] expected_state: "running" # --- OS Install Config on BMaaS --- bm_from_config: step: install_config_bm checks: - StepSuccessCheck: + StepSuccessCheck-img_bm_from_config: test_id: "BOOT01-04" labels: ["image_registry", "min_req"] - FieldExistsCheck: + requires: [bare_metal] + FieldExistsCheck-img_bm_from_config: test_id: "N/A" labels: ["image_registry"] + requires: [bare_metal] fields: ["instance_id", "config_id", "instance_state"] - InstanceStateCheck: + InstanceStateCheck-img_bm_from_config: test_id: "N/A" labels: ["image_registry"] + requires: [bare_metal] expected_state: "running" teardown_checks: step: teardown checks: - StepSuccessCheck: + StepSuccessCheck-img_teardown: test_id: "N/A" labels: ["image_registry"] + requires: [] exclude: labels: [] diff --git a/isvctl/configs/suites/k8s.yaml b/isvctl/configs/suites/k8s.yaml index ea87d5f38..505e3cb09 100644 --- a/isvctl/configs/suites/k8s.yaml +++ b/isvctl/configs/suites/k8s.yaml @@ -75,7 +75,7 @@ tests: # The separate CPU and GPU create checks prove distinct pools, each with # its own instance type and node count, coexist in the same cluster. k8s_node_pools: - - K8sNodePoolCheck: + - K8sNodePoolCheck-create_cpu_pool: # Create (CPU pool): pool reaches desired replica count with its # specified CPU instance type. test_id: "K8S06-01" @@ -89,7 +89,7 @@ tests: node_type: "{{ steps.create_test_node_pool.node_type | default('cpu') }}" wait_timeout: 900 poll_interval: 10 - - K8sNodePoolCheck: + - K8sNodePoolCheck-create_gpu_pool: # Create (GPU pool): a separate pool reaches its own node count with # its specified GPU instance type, coexisting with the CPU pool. test_id: "K8S06-01" @@ -103,7 +103,7 @@ tests: node_type: "{{ steps.create_test_gpu_node_pool.node_type | default('gpu') }}" wait_timeout: 900 poll_interval: 10 - - K8sNodePoolCheck: + - K8sNodePoolCheck-update_pool: # Update (scale): same pool converges to the new count. test_id: "K8S06-02" labels: ["kubernetes", "min_req", "slow"] @@ -116,7 +116,7 @@ tests: node_type: "{{ steps.update_test_node_pool.node_type | default('cpu') }}" wait_timeout: 900 poll_interval: 10 - - K8sNodePoolCheck: + - K8sNodePoolCheck-delete_pool: # Delete: assert the throwaway pool reaches zero nodes after the # delete step. Selector comes from its create step; the label/taint/ # instance-type assertions are omitted (no nodes to check). @@ -239,207 +239,6 @@ tests: wait_timeout: 180 poll_interval: 2 - k8s_storage: - checks: - K8sCsiStorageTypesCheck: - test_id: "K8S23-04" - labels: ["kubernetes", "min_req"] - # StorageClass names by type; omit (or leave empty) to skip that type. - # Env overrides: K8S_CSI_BLOCK_SC, K8S_CSI_SHARED_FS_SC, K8S_CSI_NFS_SC. - block_storage_class: "{{ steps.setup.csi.block_storage_class | default('', true) }}" - shared_fs_storage_class: "{{ steps.setup.csi.shared_fs_storage_class | default('', true) }}" - nfs_storage_class: "{{ steps.setup.csi.nfs_storage_class | default('', true) }}" - pvc_size: "1Gi" - bind_timeout_s: 120 - namespace_prefix: "isvtest-csi-types" - K8sCsiStorageQuotaApiCheck: - test_id: "K8S23-07" - labels: ["kubernetes", "min_req"] - # Uses the block StorageClass by default; skips entirely when unset. - # per_sc_quota must exceed pvc_request so the usage PVC fits, and - # over_quota_request must exceed per_sc_quota so admission rejects it. - storage_class: "{{ steps.setup.csi.block_storage_class | default('', true) }}" - total_quota: "10Gi" - per_sc_quota: "5Gi" - pvc_request: "1Gi" - over_quota_request: "100Gi" - bind_timeout_s: 120 - quota_settle_s: 30 - namespace_prefix: "isvtest-csi-quota" - K8sCsiTenantScopedCredentialsCheck: - test_id: "K8S23-06" - labels: ["kubernetes", "min_req"] - # Pure read-only check: verifies CSI credentials are tenant-scoped - # by construction (Secret namespaces, labels/annotations, RBAC, - # and node-plugin volume topology). Skipped entirely when no - # CSIDriver objects exist. - csi_driver_namespaces: - - "kube-system" - allowed_workload_namespaces: [] - forbidden_labels: - - "shared-across-clusters=true" - K8sCsiProvisioningModesCheck: - test_id: "K8S23-05" - labels: ["kubernetes", "min_req"] - # Exercises dynamic and static CSI provisioning end-to-end with a - # BusyBox mount + canary-file write/read. Skipped entirely when - # dynamic_storage_class is unset. The static subtest is skipped - # (not failed) when static_pv.volume_handle / csi_driver are - # unset, so providers that don't pre-provision a volume can still - # run the dynamic probe. - dynamic_storage_class: "{{ steps.setup.csi.block_storage_class | default('', true) }}" - dynamic_pvc_size: "1Gi" - static_pv: - volume_handle: "{{ steps.setup.csi.static_volume_handle | default('', true) }}" - csi_driver: "{{ steps.setup.csi.static_driver_name | default('', true) }}" - # Zonal block volumes (EBS/PD/Disk) must pin their consumer pod to - # the volume's AZ; empty for zone-agnostic backends. - zone: "{{ steps.setup.csi.static_volume_az | default('', true) }}" - fs_type: "ext4" - capacity: "1Gi" - access_mode: "ReadWriteOnce" - bind_timeout_s: 180 - namespace_prefix: "isvtest-csi-prov" - K8sCsiDriverHealthCheck: - test_id: "N/A" - labels: ["kubernetes", "min_req"] - # Verifies each configured CSI driver is installed and healthy. - # Controller/node health (Deployment + DaemonSet) runs only - # when a spec supplies a `workloads` block; otherwise those subtests are - # skipped (driver registration is still checked). - # - storage_classes: ["..."] - # workloads: - # namespace: kube-system - # controller: { deployment: } - # node: { daemonset: } - drivers: - - storage_classes: - - "{{ steps.setup.csi.block_storage_class | default('', true) }}" - workloads: {} - - storage_classes: - - "{{ steps.setup.csi.shared_fs_storage_class | default('', true) }}" - workloads: {} - - storage_classes: - - "{{ steps.setup.csi.nfs_storage_class | default('', true) }}" - workloads: {} - min_controller_replicas: 1 - K8sCsiConcurrentPvcCheck: - test_id: "N/A" - labels: ["kubernetes", "min_req"] - storage_class: "{{ steps.setup.csi.block_storage_class | default('', true) }}" - pvc_count: 2 - pvc_size: "1Gi" - bind_timeout_s: 120 - namespace_prefix: "isvtest-csi-concurrent" - K8sCsiPvcExpandCheck: - test_id: "N/A" - labels: ["kubernetes", "min_req"] - storage_class: "{{ steps.setup.csi.block_storage_class | default('', true) }}" - initial_size: "1Gi" - expanded_size: "2Gi" - bind_timeout_s: 120 - expand_timeout_s: 180 - namespace_prefix: "isvtest-csi-expand" - K8sNodeKernelModulesCheck: - test_id: "HSS11-04" - labels: ["kubernetes", "storage"] - kernel_modules: [] - node_selector: {} - image: "busybox:1.36" - bind_timeout_s: 60 - namespace_prefix: "isvtest-kmod" - K8sNfsMountOptionsCheck: - test_id: "HSS11-01" - labels: ["kubernetes", "storage"] - # Env overrides: K8S_CSI_SHARED_FS_SC, K8S_CSI_NFS_SC. - shared_fs_storage_class: "{{ steps.setup.csi.shared_fs_storage_class | default('', true) }}" - nfs_storage_class: "{{ steps.setup.csi.nfs_storage_class | default('', true) }}" - image: "busybox:1.36" - node_selector: {} - pvc_size: "1Gi" - bind_timeout_s: 180 - namespace_prefix: "isvtest-fs-nfs" - # if expected values are not set, the check will be skipped - expected_version: "" - expected_nconnect: "" - expected_proto: "" - expected_read_ahead_kb: "" - - k8s_filesystem: - checks: - # Shared-filesystem POSIX-semantics checks (multi-pod, RWX PVC). - K8sFileLockingCheck: - test_id: "HSS14-02" - labels: ["kubernetes", "storage"] - shared_fs_storage_class: "{{ steps.setup.csi.shared_fs_storage_class | default('', true) }}" - nfs_storage_class: "{{ steps.setup.csi.nfs_storage_class | default('', true) }}" - image: "busybox:1.36" - node_selector: {} - pvc_size: "1Gi" - bind_timeout_s: 120 - release_timeout_s: 60 - namespace_prefix: "isvtest-fs-lock" - K8sCrossNodeWriteVisibilityCheck: - test_id: "HSS17-01" - labels: ["kubernetes", "storage"] - shared_fs_storage_class: "{{ steps.setup.csi.shared_fs_storage_class | default('', true) }}" - nfs_storage_class: "{{ steps.setup.csi.nfs_storage_class | default('', true) }}" - image: "busybox:1.36" - node_selector: {} - pvc_size: "1Gi" - bind_timeout_s: 120 - visibility_window_s: 5.0 - namespace_prefix: "isvtest-fs-vis" - K8sCrossNodeAttrConsistencyCheck: - test_id: "HSS17-02" - labels: ["kubernetes", "storage"] - shared_fs_storage_class: "{{ steps.setup.csi.shared_fs_storage_class | default('', true) }}" - nfs_storage_class: "{{ steps.setup.csi.nfs_storage_class | default('', true) }}" - image: "busybox:1.36" - node_selector: {} - pvc_size: "1Gi" - bind_timeout_s: 120 - attr_cache_window_s: 30.0 - extend_bytes: 1024 - namespace_prefix: "isvtest-fs-attr" - K8sLargeDirListingFilesCheck: - test_id: "HSS07-02" - labels: ["kubernetes", "storage"] - shared_fs_storage_class: "{{ steps.setup.csi.shared_fs_storage_class | default('', true) }}" - nfs_storage_class: "{{ steps.setup.csi.nfs_storage_class | default('', true) }}" - image: "busybox:1.36" - node_selector: {} - files_count: 10000 - pvc_size: "10Gi" - bind_timeout_s: 120 - namespace_prefix: "isvtest-fs-bigdir" - timeout: 3600 - K8sLargeDirListingDirsCheck: - test_id: "HSS07-02" - labels: ["kubernetes", "storage"] - shared_fs_storage_class: "{{ steps.setup.csi.shared_fs_storage_class | default('', true) }}" - nfs_storage_class: "{{ steps.setup.csi.nfs_storage_class | default('', true) }}" - image: "busybox:1.36" - node_selector: {} - dirs_count: 500 - pvc_size: "10Gi" - bind_timeout_s: 120 - namespace_prefix: "isvtest-fs-bigdir" - timeout: 3600 - K8sPosixComplianceCheck: - test_id: "DIR02-02" - labels: ["kubernetes", "slow", "storage"] - shared_fs_storage_class: "{{ steps.setup.csi.shared_fs_storage_class | default('', true) }}" - nfs_storage_class: "{{ steps.setup.csi.nfs_storage_class | default('', true) }}" - image: "gcc:12" - node_selector: {} - pvc_size: "5Gi" - bind_timeout_s: 120 - build_timeout_s: 600 - # tests_subdir: "" # optional: scope to a single tests/ subdir (e.g. "chmod") - namespace_prefix: "isvtest-fs-posix" - timeout: 3600 - k8s_identity: checks: K8sOidcIssuerCheck: diff --git a/isvctl/configs/suites/network.yaml b/isvctl/configs/suites/network.yaml index 181f1405b..0fe1dc152 100644 --- a/isvctl/configs/suites/network.yaml +++ b/isvctl/configs/suites/network.yaml @@ -37,7 +37,6 @@ version: "1.0" tests: cluster_name: "network-validation" description: "Network validation tests" - platform: network settings: # Provider-specific - override in your provider config @@ -53,6 +52,7 @@ tests: NetworkProvisionedCheck: test_id: "N/A" labels: ["network"] + requires: [] require_subnets: true min_subnets: 2 @@ -63,27 +63,32 @@ tests: VpcCrudCheck-create: test_id: "SDN01-01" labels: ["min_req", "network"] + requires: [] step: vpc_crud operations: ["create_vpc"] VpcCrudCheck-read: test_id: "SDN01-02" labels: ["min_req", "network"] + requires: [] step: vpc_crud operations: ["read_vpc"] VpcCrudCheck-update: test_id: "SDN01-03" labels: ["min_req", "network"] + requires: [] step: vpc_crud operations: ["update_tags", "update_dns"] VpcCrudCheck-delete: test_id: "SDN01-04" labels: ["min_req", "network"] + requires: [] step: vpc_crud operations: ["delete_vpc"] # no test id for subnet count / multi-AZ layout (SDN04-01 covers exclusive-subnet isolation, not subnet topology). SubnetConfigCheck: test_id: "N/A" labels: ["network"] + requires: [] step: subnet_config min_subnets: 4 require_multi_az: true @@ -91,25 +96,30 @@ tests: # SDN04-03 (E/W isolation) is exercised by the same check but is not separately attributable under a single test_id. test_id: "SDN04-02" labels: ["min_req", "network", "security"] + requires: [] step: vpc_isolation SgCrudCheck-create: test_id: "SDN02-01" labels: ["min_req", "network", "security"] + requires: [] step: sg_crud operations: ["create_vpc", "create_sg"] SgCrudCheck-read: test_id: "SDN02-02" labels: ["min_req", "network", "security"] + requires: [] step: sg_crud operations: ["read_sg"] SgCrudCheck-update: test_id: "SDN02-03" labels: ["min_req", "network", "security"] + requires: [] step: sg_crud operations: ["update_sg_add_rule", "update_sg_modify_rule", "update_sg_remove_rule"] SgCrudCheck-delete: test_id: "SDN02-04" labels: ["min_req", "network", "security"] + requires: [] step: sg_crud operations: ["delete_sg", "verify_deleted"] # SG/NACL enforcement + connectivity/traffic behavior have no dedicated @@ -117,22 +127,27 @@ tests: SecurityBlockingCheck: test_id: "N/A" labels: ["network", "security"] + requires: [] step: security_blocking NetworkConnectivityCheck: test_id: "N/A" labels: ["network"] + requires: [compute] step: connectivity_test TrafficFlowCheck: test_id: "N/A" labels: ["network"] + requires: [compute] step: traffic_validation VpcIpConfigCheck: test_id: "IPAM02-01" labels: ["network"] + requires: [] step: vpc_ip_config DhcpIpManagementCheck: test_id: "IPAM01-01" labels: ["network", "ssh"] + requires: [compute] step: dhcp_ip_test sg_scoping: @@ -140,22 +155,27 @@ tests: SgWorkloadScopingCheck: test_id: "SDN02-05" labels: ["min_req", "network", "security"] + requires: [] step: sg_workload_scoping SgNodeScopingCheck: test_id: "SDN02-06" labels: ["min_req", "network", "security"] + requires: [] step: sg_node_scoping SgSubnetScopingCheck: test_id: "SDN02-07" labels: ["min_req", "network", "security"] + requires: [] step: sg_subnet_scoping SgServiceScopingCheck: test_id: "SDN02-09" labels: ["min_req", "network", "security"] + requires: [] step: sg_service_scoping SgPortSecurityPolicyCheck: test_id: "SDN02-10" labels: ["min_req", "network", "security"] + requires: [] step: sg_port_security_policy sdn_policy: @@ -163,6 +183,7 @@ tests: SgPolicyPropagationTimingCheck: test_id: "SDN02-08" labels: ["min_req", "network", "security"] + requires: [] step: sg_policy_propagation max_propagation_seconds: 10 @@ -171,14 +192,17 @@ tests: SdnHardwareFaultLoggingCheck: test_id: "SDN09-01" labels: ["min_req", "network"] + requires: [] step: sdn_hardware_fault_logging SdnLatencyPerfLoggingCheck: test_id: "SDN09-02" labels: ["min_req", "network"] + requires: [] step: sdn_latency_perf_logging SdnFilterAuditTrailCheck: test_id: "SDN09-03" labels: ["min_req", "network", "security"] + requires: [] step: sdn_filter_audit_trail sdn: @@ -186,31 +210,38 @@ tests: ByoipCheck: test_id: "NET03-01" labels: ["min_req", "network"] + requires: [] step: byoip_test StablePrivateIpCheck: test_id: "SDN11-01" labels: ["min_req", "network"] + requires: [compute] step: stable_ip_test StableEgressIpCheck: test_id: "DMS05-01" labels: ["min_req", "network"] + requires: [compute] step: stable_egress_ip_test FloatingIpCheck: test_id: "SDN05-01" labels: ["min_req", "network"] + requires: [compute] step: floating_ip_test max_switch_seconds: 10 LocalizedDnsCheck: test_id: "SDN06-01" labels: ["min_req", "network"] + requires: [] step: dns_test VpcPeeringCheck: test_id: "SDN07-01" labels: ["min_req", "network"] + requires: [] step: peering_test StorageL3RoutingCheck: test_id: "SDN08-01" labels: ["min_req", "network"] + requires: [compute] step: storage_l3_routing # Read-only metadata/API checks for provider-reported backend fabric @@ -220,18 +251,21 @@ tests: BackendSwitchFabricCheck: test_id: "NET01-01" labels: ["min_req", "network"] + requires: [compute] step: backend_switch_fabric NvlinkDomainCheck: test_id: "NET02-02" labels: ["min_req", "network"] + requires: [compute] step: nvlink_domain teardown_checks: step: teardown checks: - StepSuccessCheck: + StepSuccessCheck-network_teardown: test_id: "N/A" labels: ["network"] + requires: [] exclude: labels: [] diff --git a/isvctl/configs/suites/observability.yaml b/isvctl/configs/suites/observability.yaml index db36a6f8f..9a78e48b4 100644 --- a/isvctl/configs/suites/observability.yaml +++ b/isvctl/configs/suites/observability.yaml @@ -23,7 +23,6 @@ version: "1.0" tests: cluster_name: "observability-validation" description: "Observability collector validation tests" - platform: observability settings: region: "" @@ -40,6 +39,7 @@ tests: TelemetryDeliveryLatencyCheck: test_id: "OBS05-01" labels: ["min_req", "observability"] + requires: [compute] step: telemetry_delivery_latency max_delivery_seconds: "{{max_delivery_seconds}}" @@ -48,22 +48,27 @@ tests: NorthSouthNetworkTelemetryCheck: test_id: "OBS06-01" labels: ["min_req", "network", "observability"] + requires: [compute] step: north_south_network_telemetry EastWestNetworkTelemetryCheck: test_id: "OBS07-01" labels: ["min_req", "network", "observability"] + requires: [] step: east_west_network_telemetry ManagementNetworkTelemetryCheck: test_id: "OBS08-01" labels: ["min_req", "network", "observability"] + requires: [] step: management_network_telemetry NvswitchFabricTelemetryCheck: test_id: "OBS09-01" labels: ["min_req", "network", "observability"] + requires: [] step: nvswitch_fabric_telemetry HostNicNetworkTelemetryCheck: test_id: "OBS10-01" labels: ["min_req", "network", "observability"] + requires: [compute] step: host_nic_network_telemetry network_logs: @@ -71,6 +76,7 @@ tests: VpcFlowLogsCheck: test_id: "OBS19-01" labels: ["min_req", "network", "observability"] + requires: [] step: vpc_flow_logs host_logs: @@ -78,6 +84,7 @@ tests: HostSyslogCheck: test_id: "OBS18-01" labels: ["bare_metal", "min_req", "observability"] + requires: [compute] step: host_syslogs bmc_logs: @@ -85,6 +92,7 @@ tests: BmcSelLogsCheck: test_id: "OBS17-01" labels: ["min_req", "observability", "security"] + requires: [] step: bmc_sel_logs bmc_telemetry: @@ -92,6 +100,7 @@ tests: BmcGpuTelemetryCheck: test_id: "TELEM04-01" labels: ["gpu", "min_req", "observability", "security"] + requires: [] step: bmc_gpu_telemetry storage_capacity_telemetry: @@ -99,6 +108,7 @@ tests: StorageCapacityTelemetryCheck: test_id: "TELEM05-01" labels: ["min_req", "observability"] + requires: [] step: storage_capacity_telemetry storage_performance_telemetry: @@ -106,6 +116,7 @@ tests: StoragePerformanceTelemetryCheck: test_id: "TELEM06-01" labels: ["min_req", "observability"] + requires: [compute] step: storage_performance_telemetry gpu_nvlink_telemetry: @@ -113,6 +124,7 @@ tests: GpuNvlinkTelemetryCheck: test_id: "TELEM07-01" labels: ["min_req", "observability"] + requires: [] step: gpu_nvlink_telemetry switch_nvlink_telemetry: @@ -120,6 +132,7 @@ tests: SwitchNvlinkTelemetryCheck: test_id: "TELEM08-01" labels: ["min_req", "observability"] + requires: [] step: switch_nvlink_telemetry fabric_logs: @@ -127,10 +140,12 @@ tests: FabricManagerLogsCheck: test_id: "OBS11-01" labels: ["min_req", "observability"] + requires: [] step: fabric_manager_logs UfmEventLogsCheck: test_id: "OBS13-01" labels: ["min_req", "observability"] + requires: [] step: ufm_event_logs subnet_manager_logs: @@ -138,6 +153,7 @@ tests: SubnetManagerLogsCheck: test_id: "OBS12-01" labels: ["min_req", "observability"] + requires: [] step: subnet_manager_logs switch_logs: @@ -145,14 +161,17 @@ tests: GeneralSwitchLogsCheck: test_id: "OBS14-01" labels: ["min_req", "network", "observability"] + requires: [] step: general_switch_logs SwitchSyslogCheck: test_id: "OBS15-01" labels: ["min_req", "network", "observability"] + requires: [] step: switch_syslogs SwitchKernelLogsCheck: test_id: "OBS16-01" labels: ["min_req", "network", "observability"] + requires: [] step: switch_kernel_logs exclude: diff --git a/isvctl/configs/suites/security.yaml b/isvctl/configs/suites/security.yaml index e08fe66ad..d17098ea7 100644 --- a/isvctl/configs/suites/security.yaml +++ b/isvctl/configs/suites/security.yaml @@ -36,7 +36,6 @@ version: "1.0" tests: cluster_name: "security-validation" description: "Infrastructure security hardening validation" - platform: security settings: region: "" @@ -55,6 +54,7 @@ tests: BmcManagementNetworkCheck: test_id: "SEC12-01" labels: ["min_req", "network", "security"] + requires: [] step: bmc_management_network bmc_isolation: @@ -62,6 +62,7 @@ tests: BmcTenantIsolationCheck: test_id: "SEC12-02" labels: ["min_req", "network", "security"] + requires: [] step: bmc_tenant_isolation bmc_protocol_security: @@ -69,6 +70,7 @@ tests: BmcProtocolSecurityCheck: test_id: "CNP10-01" labels: ["min_req", "network", "security"] + requires: [] step: bmc_protocol_security bmc_bastion_access: @@ -76,6 +78,7 @@ tests: BmcBastionAccessCheck: test_id: "SEC12-03" labels: ["min_req", "network", "security"] + requires: [] step: bmc_bastion_access api_security: @@ -83,6 +86,7 @@ tests: ApiEndpointIsolationCheck: test_id: "SEC14-01" labels: ["min_req", "network", "security"] + requires: [] step: api_endpoint_isolation mutual_tls: @@ -90,6 +94,7 @@ tests: MutualTlsCheck: test_id: "SEC13-01" labels: ["min_req", "network", "security"] + requires: [] step: mutual_tls_test insecure_protocols: @@ -97,6 +102,7 @@ tests: InsecureProtocolsCheck: test_id: "SEC13-02" labels: ["min_req", "network", "security"] + requires: [] step: insecure_protocols_test mfa_enforcement: @@ -104,6 +110,7 @@ tests: MfaEnforcedCheck: test_id: "SEC07-01" labels: ["iam", "min_req", "security"] + requires: [] step: mfa_enforcement cert_rotation: @@ -111,6 +118,7 @@ tests: CertRotationCycleCheck: test_id: "SEC09-01" labels: ["min_req", "security"] + requires: [] step: cert_rotation_test kms_encryption_options: @@ -118,6 +126,7 @@ tests: KmsEncryptionOptionCheck: test_id: "SEC09-02" labels: ["min_req", "security"] + requires: [] step: kms_encryption_options_test centralized_kms: @@ -125,6 +134,7 @@ tests: CentralizedKmsCheck: test_id: "SEC09-03" labels: ["min_req", "security"] + requires: [] step: centralized_kms_test customer_managed_key: @@ -132,6 +142,7 @@ tests: CustomerManagedKeyCheck: test_id: "SEC09-04" labels: ["min_req", "security", "slow", "workload"] + requires: [] step: customer_managed_key_test service_account_auth: @@ -139,6 +150,7 @@ tests: ServiceAccountCredentialCheck: test_id: "SEC03-01" labels: ["iam", "min_req", "security"] + requires: [] step: sa_credential_test user_auth_oidc: @@ -146,6 +158,7 @@ tests: OidcUserAuthCheck: test_id: "SEC01-01" labels: ["iam", "min_req", "security"] + requires: [] step: oidc_user_auth_test short_lived_credentials: @@ -153,6 +166,7 @@ tests: ShortLivedCredentialsCheck: test_id: "SEC06-01" labels: ["iam", "min_req", "security"] + requires: [] step: short_lived_credentials_test least_privilege_policy: @@ -160,6 +174,7 @@ tests: LeastPrivilegePolicyCheck: test_id: "SEC04-01" labels: ["iam", "min_req", "security"] + requires: [] step: least_privilege_test minimal_role_enforcement: @@ -167,6 +182,7 @@ tests: MinimalRoleEnforcementCheck: test_id: "SEC04-02" labels: ["iam", "min_req", "security"] + requires: [] step: least_privilege_test audit_log_entry: @@ -174,6 +190,7 @@ tests: AuditLogEntryCheck: test_id: "SEC08-01" labels: ["min_req", "security"] + requires: [] step: audit_logging_test audit_log_retention: @@ -181,6 +198,7 @@ tests: AuditLogRetentionCheck: test_id: "SEC08-02" labels: ["min_req", "security"] + requires: [] step: audit_logging_test tenant_isolation: @@ -188,6 +206,7 @@ tests: TenantIsolationCheck: test_id: "SEC11-01" labels: ["iam", "min_req", "network", "security"] + requires: [compute] step: tenant_isolation_test # Verify capacity is logically grouped and pinned to one tenant/account @@ -197,6 +216,7 @@ tests: CapacityReservationGroupingCheck: test_id: "CAP04-01" labels: ["bare_metal", "capacity", "min_req", "security"] + requires: [compute] step: capacity_reservation_grouping min_resources: "{{min_resources}}" @@ -208,14 +228,16 @@ tests: CapacityTopologyBlockAtomicAllocationCheck: test_id: "CAP04-02" labels: ["bare_metal", "capacity", "min_req", "security"] + requires: [compute] min_resources: 2 teardown_checks: step: teardown checks: - StepSuccessCheck: + StepSuccessCheck-security_teardown: test_id: "N/A" labels: ["security"] + requires: [] exclude: labels: [] diff --git a/isvctl/configs/suites/storage.yaml b/isvctl/configs/suites/storage.yaml index ef43c9464..00eefa5e7 100644 --- a/isvctl/configs/suites/storage.yaml +++ b/isvctl/configs/suites/storage.yaml @@ -53,7 +53,6 @@ version: "1.0" tests: - platform: storage cluster_name: "storage-validation" description: "Storage: block volumes and high-speed / parallel filesystem services" @@ -71,72 +70,85 @@ tests: instance_launched: step: launch_instance checks: - InstanceStateCheck: + InstanceStateCheck-storage_instance_launched: test_id: "N/A" labels: ["storage"] + requires: [compute] expected_state: "running" fixture_volume: step: create_volume checks: - StepSuccessCheck: + StepSuccessCheck-storage_fixture_volume: test_id: "N/A" labels: ["storage"] - FieldExistsCheck: + requires: [compute] + FieldExistsCheck-storage_fixture_volume: test_id: "N/A" labels: ["storage"] + requires: [compute] fields: ["volume_id", "mount_point", "operations"] - CrudOperationsCheck: + CrudOperationsCheck-storage_fixture_volume: test_id: "N/A" labels: ["storage"] + requires: [compute] operations: ["create", "attach", "format", "mount", "write_sentinel"] # ─── DATASVC-XX-02: volume snapshots (issue #321) ──────────────── snapshot_lifecycle: step: snapshot_lifecycle checks: - StepSuccessCheck: + StepSuccessCheck-storage_snapshot_lifecycle: test_id: "N/A" labels: ["storage"] - FieldExistsCheck: + requires: [compute] + FieldExistsCheck-storage_snapshot_lifecycle: test_id: "N/A" labels: ["storage"] + requires: [compute] fields: ["volume_id", "snapshot_id", "operations"] - CrudOperationsCheck: + CrudOperationsCheck-storage_snapshot_lifecycle: test_id: "DATASVC02-01" labels: ["min_req", "storage"] + requires: [compute] operations: ["create_snapshot", "restore_volume", "verify_data"] # ─── DATASVC-XX-03: volume resizing (issue #322) ───────────────── volume_resize: step: volume_resize checks: - StepSuccessCheck: + StepSuccessCheck-storage_volume_resize: test_id: "N/A" labels: ["storage"] - FieldExistsCheck: + requires: [compute] + FieldExistsCheck-storage_volume_resize: test_id: "N/A" labels: ["storage"] + requires: [compute] fields: ["volume_id", "operations"] - CrudOperationsCheck: + CrudOperationsCheck-storage_volume_resize: test_id: "DATASVC03-01" labels: ["min_req", "storage"] + requires: [compute] operations: ["modify_volume", "grow_partition", "resize_filesystem", "verify_size"] # ─── DATASVC-XX-04: persistence across restarts (issue #323) ───── volume_persistence: step: volume_persistence checks: - StepSuccessCheck: + StepSuccessCheck-storage_volume_persistence: test_id: "N/A" labels: ["storage"] - FieldExistsCheck: + requires: [compute] + FieldExistsCheck-storage_volume_persistence: test_id: "N/A" labels: ["storage"] + requires: [compute] fields: ["volume_id", "operations"] - CrudOperationsCheck: + CrudOperationsCheck-storage_volume_persistence: test_id: "DATASVC04-01" labels: ["min_req", "storage"] + requires: [compute] operations: ["stop", "start", "verify_attached", "verify_data"] # ─── Home directory storage ────────────────────────────────────── @@ -146,12 +158,15 @@ tests: DirectoryFilesystemQuotaCheck: test_id: "DIR01-01" labels: ["min_req", "storage"] + requires: [compute] DirectoryUsageAccountingCheck: test_id: "DIR01-02" labels: ["min_req", "storage"] + requires: [compute] DirectoryNfsAvailabilityCheck: test_id: "DIR02-01" labels: ["min_req", "storage"] + requires: [compute] # ─── High-speed storage: SDS controller ────────────────────────── sds_controller: @@ -159,18 +174,22 @@ tests: HssStorageProvisioningCheck: test_id: "HSS01-01" labels: ["min_req", "storage"] + requires: [] step: provision_storage HssQosThroughputCheck: test_id: "HSS02-01" labels: ["min_req", "storage"] + requires: [compute] step: qos_throughput HssNonDisruptiveUpgradeCheck: test_id: "HSS05-01" labels: ["min_req", "storage"] + requires: [] step: non_disruptive_upgrade HssRdmaMemoryProtectionCheck: test_id: "HSS06-01" labels: ["min_req", "storage"] + requires: [compute] step: rdma_memory_protection # ─── High-speed storage: parallel file system services ─────────── @@ -179,43 +198,271 @@ tests: HssParallelFsProvisioningCheck: test_id: "HSS07-01" labels: ["min_req", "storage"] + requires: [compute] step: provision_parallel_fs HssMultipleFilesystemsCheck: test_id: "HSS09-01" labels: ["min_req", "storage"] + requires: [] step: multiple_filesystems HssLiveExpansionCheck: test_id: "HSS10-01" labels: ["min_req", "storage"] + requires: [] step: live_expansion HssQuotaEnforcementCheck: test_id: "HSS12-01" labels: ["min_req", "storage"] + requires: [compute] step: quota_enforcement HssRootSquashCheck: test_id: "HSS13-01" labels: ["min_req", "storage"] + requires: [] step: root_squash HssFlockMountCheck: test_id: "HSS14-01" labels: ["min_req", "storage"] + requires: [compute] step: flock_mount HssChangelogAuditCheck: test_id: "HSS15-01" labels: ["min_req", "storage"] + requires: [compute] step: changelog_audit HssMultipathCheck: test_id: "HSS18-01" labels: ["min_req", "storage"] + requires: [compute] step: multipath # ─── Teardown ──────────────────────────────────────────────────── teardown_checks: step: teardown_volume checks: - StepSuccessCheck: + StepSuccessCheck-storage_teardown: test_id: "N/A" labels: ["storage"] + requires: [] + + # Kubernetes storage checks use a normal provider-owned cluster fixture. + k8s_storage: + step: ensure_cluster + checks: + K8sCsiStorageTypesCheck: + test_id: "K8S23-04" + labels: ["kubernetes", "min_req", "storage"] + requires: [kubernetes] + # StorageClass names by type; omit (or leave empty) to skip that type. + # Env overrides: K8S_CSI_BLOCK_SC, K8S_CSI_SHARED_FS_SC, K8S_CSI_NFS_SC. + block_storage_class: "{{ steps.ensure_cluster.csi.block_storage_class | default('', true) }}" + shared_fs_storage_class: "{{ steps.ensure_cluster.csi.shared_fs_storage_class | default('', true) }}" + nfs_storage_class: "{{ steps.ensure_cluster.csi.nfs_storage_class | default('', true) }}" + pvc_size: "1Gi" + bind_timeout_s: 120 + namespace_prefix: "isvtest-csi-types" + K8sCsiStorageQuotaApiCheck: + test_id: "K8S23-07" + labels: ["kubernetes", "min_req", "storage"] + requires: [kubernetes] + # Uses the block StorageClass by default; skips entirely when unset. + # per_sc_quota must exceed pvc_request so the usage PVC fits, and + # over_quota_request must exceed per_sc_quota so admission rejects it. + storage_class: "{{ steps.ensure_cluster.csi.block_storage_class | default('', true) }}" + total_quota: "10Gi" + per_sc_quota: "5Gi" + pvc_request: "1Gi" + over_quota_request: "100Gi" + bind_timeout_s: 120 + quota_settle_s: 30 + namespace_prefix: "isvtest-csi-quota" + K8sCsiTenantScopedCredentialsCheck: + test_id: "K8S23-06" + labels: ["kubernetes", "min_req", "storage"] + requires: [kubernetes] + # Pure read-only check: verifies CSI credentials are tenant-scoped + # by construction (Secret namespaces, labels/annotations, RBAC, + # and node-plugin volume topology). Skipped entirely when no + # CSIDriver objects exist. + csi_driver_namespaces: + - "kube-system" + allowed_workload_namespaces: [] + forbidden_labels: + - "shared-across-clusters=true" + K8sCsiProvisioningModesCheck: + test_id: "K8S23-05" + labels: ["kubernetes", "min_req", "storage"] + requires: [kubernetes] + # Exercises dynamic and static CSI provisioning end-to-end with a + # BusyBox mount + canary-file write/read. Skipped entirely when + # dynamic_storage_class is unset. The static subtest is skipped + # (not failed) when static_pv.volume_handle / csi_driver are + # unset, so providers that don't pre-provision a volume can still + # run the dynamic probe. + dynamic_storage_class: "{{ steps.ensure_cluster.csi.block_storage_class | default('', true) }}" + dynamic_pvc_size: "1Gi" + static_pv: + volume_handle: "{{ steps.ensure_cluster.csi.static_volume_handle | default('', true) }}" + csi_driver: "{{ steps.ensure_cluster.csi.static_driver_name | default('', true) }}" + # Zonal block volumes (EBS/PD/Disk) must pin their consumer pod to + # the volume's AZ; empty for zone-agnostic backends. + zone: "{{ steps.ensure_cluster.csi.static_volume_az | default('', true) }}" + fs_type: "ext4" + capacity: "1Gi" + access_mode: "ReadWriteOnce" + bind_timeout_s: 180 + namespace_prefix: "isvtest-csi-prov" + K8sCsiDriverHealthCheck: + test_id: "N/A" + labels: ["kubernetes", "min_req", "storage"] + requires: [kubernetes] + # Verifies each configured CSI driver is installed and healthy. + # Controller/node health (Deployment + DaemonSet) runs only + # when a spec supplies a `workloads` block; otherwise those subtests are + # skipped (driver registration is still checked). + # - storage_classes: ["..."] + # workloads: + # namespace: kube-system + # controller: { deployment: } + # node: { daemonset: } + drivers: + - storage_classes: + - "{{ steps.ensure_cluster.csi.block_storage_class | default('', true) }}" + workloads: {} + - storage_classes: + - "{{ steps.ensure_cluster.csi.shared_fs_storage_class | default('', true) }}" + workloads: {} + - storage_classes: + - "{{ steps.ensure_cluster.csi.nfs_storage_class | default('', true) }}" + workloads: {} + min_controller_replicas: 1 + K8sCsiConcurrentPvcCheck: + test_id: "N/A" + labels: ["kubernetes", "min_req", "storage"] + requires: [kubernetes] + storage_class: "{{ steps.ensure_cluster.csi.block_storage_class | default('', true) }}" + pvc_count: 2 + pvc_size: "1Gi" + bind_timeout_s: 120 + namespace_prefix: "isvtest-csi-concurrent" + K8sCsiPvcExpandCheck: + test_id: "N/A" + labels: ["kubernetes", "min_req", "storage"] + requires: [kubernetes] + storage_class: "{{ steps.ensure_cluster.csi.block_storage_class | default('', true) }}" + initial_size: "1Gi" + expanded_size: "2Gi" + bind_timeout_s: 120 + expand_timeout_s: 180 + namespace_prefix: "isvtest-csi-expand" + K8sNodeKernelModulesCheck: + test_id: "HSS11-04" + labels: ["kubernetes", "storage"] + requires: [kubernetes] + kernel_modules: [] + node_selector: {} + image: "busybox:1.36" + bind_timeout_s: 60 + namespace_prefix: "isvtest-kmod" + K8sNfsMountOptionsCheck: + test_id: "HSS11-01" + labels: ["kubernetes", "storage"] + requires: [kubernetes] + # Env overrides: K8S_CSI_SHARED_FS_SC, K8S_CSI_NFS_SC. + shared_fs_storage_class: "{{ steps.ensure_cluster.csi.shared_fs_storage_class | default('', true) }}" + nfs_storage_class: "{{ steps.ensure_cluster.csi.nfs_storage_class | default('', true) }}" + image: "busybox:1.36" + node_selector: {} + pvc_size: "1Gi" + bind_timeout_s: 180 + namespace_prefix: "isvtest-fs-nfs" + # if expected values are not set, the check will be skipped + expected_version: "" + expected_nconnect: "" + expected_proto: "" + expected_read_ahead_kb: "" + + k8s_filesystem: + step: ensure_cluster + checks: + # Shared-filesystem POSIX-semantics checks (multi-pod, RWX PVC). + K8sFileLockingCheck: + test_id: "HSS14-02" + labels: ["kubernetes", "storage"] + requires: [kubernetes] + shared_fs_storage_class: "{{ steps.ensure_cluster.csi.shared_fs_storage_class | default('', true) }}" + nfs_storage_class: "{{ steps.ensure_cluster.csi.nfs_storage_class | default('', true) }}" + image: "busybox:1.36" + node_selector: {} + pvc_size: "1Gi" + bind_timeout_s: 120 + release_timeout_s: 60 + namespace_prefix: "isvtest-fs-lock" + K8sCrossNodeWriteVisibilityCheck: + test_id: "HSS17-01" + labels: ["kubernetes", "storage"] + requires: [kubernetes] + shared_fs_storage_class: "{{ steps.ensure_cluster.csi.shared_fs_storage_class | default('', true) }}" + nfs_storage_class: "{{ steps.ensure_cluster.csi.nfs_storage_class | default('', true) }}" + image: "busybox:1.36" + node_selector: {} + pvc_size: "1Gi" + bind_timeout_s: 120 + visibility_window_s: 5.0 + namespace_prefix: "isvtest-fs-vis" + K8sCrossNodeAttrConsistencyCheck: + test_id: "HSS17-02" + labels: ["kubernetes", "storage"] + requires: [kubernetes] + shared_fs_storage_class: "{{ steps.ensure_cluster.csi.shared_fs_storage_class | default('', true) }}" + nfs_storage_class: "{{ steps.ensure_cluster.csi.nfs_storage_class | default('', true) }}" + image: "busybox:1.36" + node_selector: {} + pvc_size: "1Gi" + bind_timeout_s: 120 + attr_cache_window_s: 30.0 + extend_bytes: 1024 + namespace_prefix: "isvtest-fs-attr" + K8sLargeDirListingFilesCheck: + test_id: "HSS07-02" + labels: ["kubernetes", "storage"] + requires: [kubernetes] + shared_fs_storage_class: "{{ steps.ensure_cluster.csi.shared_fs_storage_class | default('', true) }}" + nfs_storage_class: "{{ steps.ensure_cluster.csi.nfs_storage_class | default('', true) }}" + image: "busybox:1.36" + node_selector: {} + files_count: 10000 + pvc_size: "10Gi" + bind_timeout_s: 120 + namespace_prefix: "isvtest-fs-bigdir" + timeout: 3600 + K8sLargeDirListingDirsCheck: + test_id: "HSS07-02" + labels: ["kubernetes", "storage"] + requires: [kubernetes] + shared_fs_storage_class: "{{ steps.ensure_cluster.csi.shared_fs_storage_class | default('', true) }}" + nfs_storage_class: "{{ steps.ensure_cluster.csi.nfs_storage_class | default('', true) }}" + image: "busybox:1.36" + node_selector: {} + dirs_count: 500 + pvc_size: "10Gi" + bind_timeout_s: 120 + namespace_prefix: "isvtest-fs-bigdir" + timeout: 3600 + K8sPosixComplianceCheck: + test_id: "DIR02-02" + labels: ["kubernetes", "slow", "storage"] + requires: [kubernetes] + shared_fs_storage_class: "{{ steps.ensure_cluster.csi.shared_fs_storage_class | default('', true) }}" + nfs_storage_class: "{{ steps.ensure_cluster.csi.nfs_storage_class | default('', true) }}" + image: "gcc:12" + node_selector: {} + pvc_size: "5Gi" + bind_timeout_s: 120 + build_timeout_s: 600 + # tests_subdir: "" # optional: scope to a single tests/ subdir (e.g. "chmod") + namespace_prefix: "isvtest-fs-posix" + timeout: 3600 exclude: labels: [] diff --git a/isvctl/configs/suites/vm.yaml b/isvctl/configs/suites/vm.yaml index b7ee93dc4..5b96ebee6 100644 --- a/isvctl/configs/suites/vm.yaml +++ b/isvctl/configs/suites/vm.yaml @@ -58,7 +58,7 @@ tests: setup_checks: step: launch_instance checks: - InstanceStateCheck: + InstanceStateCheck-vm_setup: test_id: "N/A" labels: ["vm"] expected_state: "running" @@ -87,7 +87,7 @@ tests: list_instances: step: list_instances checks: - InstanceListCheck: + InstanceListCheck-vm_list_instances: test_id: "CNP01-11" labels: ["min_req", "vm"] min_count: 1 @@ -95,7 +95,7 @@ tests: tag_checks: step: verify_tags checks: - InstanceTagCheck: + InstanceTagCheck-vm_tag: test_id: "CNP05-02" labels: ["min_req", "vm"] required_keys: ["Name", "CreatedBy"] @@ -103,7 +103,7 @@ tests: serial_console: step: serial_console checks: - SerialConsoleCheck: + SerialConsoleCheck-vm_serial_console: test_id: "CNP06-03" labels: ["min_req", "vm"] @@ -124,7 +124,7 @@ tests: cloud_init: step: launch_instance checks: - CloudInitCheck: + CloudInitCheck-vm_cloud_init: test_id: "BOOT02-02" labels: ["min_req", "ssh", "vm"] @@ -134,7 +134,7 @@ tests: instance_info: step: describe_instance checks: - InstanceStateCheck: + InstanceStateCheck-vm_instance_info: test_id: "N/A" labels: ["vm"] expected_state: "running" @@ -142,10 +142,10 @@ tests: ssh: step: describe_instance checks: - ConnectivityCheck: + ConnectivityCheck-vm_ssh: test_id: "VMAAS01-01" labels: ["vm", "ssh"] - OsCheck: + OsCheck-vm_ssh: test_id: "VMAAS05-01" labels: ["vm", "ssh"] expected_os: "ubuntu" @@ -153,7 +153,7 @@ tests: gpu: step: describe_instance checks: - GpuCheck: + GpuCheck-vm_gpu: test_id: "VMAAS06-01" labels: ["vm", "ssh", "gpu"] expected_gpus: 1 @@ -161,35 +161,35 @@ tests: host_os: step: describe_instance checks: - VcpuPinningCheck: + VcpuPinningCheck-vm_host_os: test_id: "VMAAS09-01" labels: ["gpu", "ssh", "vm"] - PciBusCheck: + PciBusCheck-vm_host_os: test_id: "VMAAS09-01" labels: ["vm", "ssh", "gpu"] expected_gpus: 1 - HostSoftwareCheck: + HostSoftwareCheck-vm_host_os: test_id: "VMAAS07-01" labels: ["gpu", "ssh", "vm"] driver_info: step: describe_instance checks: - DriverCheck: + DriverCheck-vm_driver_info: test_id: "VMAAS07-01" labels: ["vm", "ssh", "gpu"] cpu_info: step: describe_instance checks: - CpuInfoCheck: + CpuInfoCheck-vm_cpu_info: test_id: "N/A" labels: ["vm", "ssh"] container_runtime: step: describe_instance checks: - ContainerRuntimeCheck: + ContainerRuntimeCheck-vm_container_runtime: test_id: "N/A" labels: ["vm", "ssh", "gpu", "workload", "slow"] @@ -197,17 +197,17 @@ tests: stop_checks: step: stop_instance checks: - InstanceStopCheck: + InstanceStopCheck-vm_stop: test_id: "CNP01-14" labels: ["min_req", "vm"] start_checks: step: start_instance checks: - InstanceStartCheck: + InstanceStartCheck-vm_start: test_id: "CNP01-15" labels: ["min_req", "vm"] - StableIdentifierCheck: + StableIdentifierCheck-vm_start: test_id: "CNP08-03" labels: ["min_req", "vm"] reference_id: "{{steps.launch_instance.instance_id}}" @@ -216,10 +216,10 @@ tests: step: start_instance checks: # ConnectivityCheck -> VMAAS-XX-01, OsCheck -> VMAAS-XX-05 (declared in the "ssh" section) - ConnectivityCheck: + ConnectivityCheck-vm_start_ssh: test_id: "VMAAS01-01" labels: ["vm", "ssh"] - OsCheck: + OsCheck-vm_start_ssh: test_id: "VMAAS05-01" labels: ["vm", "ssh"] expected_os: "ubuntu" @@ -228,7 +228,7 @@ tests: step: start_instance checks: # GpuCheck -> VMAAS-XX-06 (declared in the "gpu" section) - GpuCheck: + GpuCheck-vm_start_gpu: test_id: "VMAAS06-01" labels: ["vm", "ssh", "gpu"] expected_gpus: 1 @@ -236,12 +236,12 @@ tests: reboot_checks: step: reboot_instance checks: - InstanceRebootCheck: + InstanceRebootCheck-vm_reboot: test_id: "CNP01-13" labels: ["vm"] max_uptime: 600 # StableIdentifierCheck -> CNP08-03 (declared in the "start_checks" section) - StableIdentifierCheck: + StableIdentifierCheck-vm_reboot: test_id: "CNP08-03" labels: ["min_req", "vm"] reference_id: "{{steps.launch_instance.instance_id}}" @@ -249,7 +249,7 @@ tests: reboot_state: step: reboot_instance checks: - InstanceStateCheck: + InstanceStateCheck-vm_reboot_state: test_id: "N/A" labels: ["vm"] expected_state: "running" @@ -258,10 +258,10 @@ tests: step: reboot_instance checks: # ConnectivityCheck -> VMAAS-XX-01, OsCheck -> VMAAS-XX-05 (declared in the "ssh" section) - ConnectivityCheck: + ConnectivityCheck-vm_reboot_ssh: test_id: "VMAAS01-01" labels: ["vm", "ssh"] - OsCheck: + OsCheck-vm_reboot_ssh: test_id: "VMAAS05-01" labels: ["vm", "ssh"] expected_os: "ubuntu" @@ -270,7 +270,7 @@ tests: step: reboot_instance checks: # GpuCheck -> VMAAS-XX-06 (declared in the "gpu" section) - GpuCheck: + GpuCheck-vm_reboot_gpu: test_id: "VMAAS06-01" labels: ["vm", "ssh", "gpu"] expected_gpus: 1 @@ -279,21 +279,21 @@ tests: nim_health: step: deploy_nim checks: - NimHealthCheck: + NimHealthCheck-vm_nim_health: test_id: "VMAAS12-01" labels: ["gpu", "slow", "ssh", "vm", "workload"] nim_models: step: deploy_nim checks: - NimModelCheck: + NimModelCheck-vm_nim_models: test_id: "VMAAS12-01" labels: ["gpu", "slow", "ssh", "vm", "workload"] nim_inference: step: deploy_nim checks: - NimInferenceCheck: + NimInferenceCheck-vm_nim_inference: test_id: "VMAAS12-01" labels: ["vm", "ssh", "gpu", "workload", "slow"] prompt: "What is CUDA?" @@ -303,14 +303,14 @@ tests: nim_teardown: step: teardown_nim checks: - StepSuccessCheck: + StepSuccessCheck-vm_nim_teardown: test_id: "N/A" labels: ["vm"] teardown_checks: step: teardown checks: - StepSuccessCheck: + StepSuccessCheck-vm_teardown: test_id: "N/A" labels: ["vm"] From 7d50d7a3731192b8c72b301af227ac9e1770c8f7 Mon Sep 17 00:00:00 2001 From: Alexandre Begnoche Date: Wed, 22 Jul 2026 13:31:54 -0400 Subject: [PATCH 06/34] test: update provider merge and catalog expectations Align merger, nico provider, and capacity config tests with plain-suite capability resolution and catalog schema v2 entry shape. Signed-off-by: Alexandre Begnoche --- .../providers/nico/test_nico_provider.py | 23 +++++++++++-------- isvctl/tests/test_capacity_config.py | 2 ++ isvctl/tests/test_merger.py | 16 +++++++------ 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/isvctl/tests/providers/nico/test_nico_provider.py b/isvctl/tests/providers/nico/test_nico_provider.py index 06d7aa21c..45c49234b 100644 --- a/isvctl/tests/providers/nico/test_nico_provider.py +++ b/isvctl/tests/providers/nico/test_nico_provider.py @@ -461,11 +461,12 @@ def _assert_steps_use_nico_api_base(steps: dict[str, dict[str, Any]]) -> None: assert "{{nico_api_base}}" in step["args"] -def test_nico_control_plane_config_platform_matches_command_group() -> None: - """The orchestrator uses tests.platform to look up the control-plane commands group.""" +def test_nico_control_plane_plain_suite_has_one_command_group() -> None: + """A plain suite derives execution identity from its sole command group.""" merged, _steps = _merged_nico_config_steps("control-plane.yaml", "control_plane") - assert merged["tests"]["platform"] == "control_plane" + assert "platform" not in merged["tests"] + assert list(merged["commands"]) == ["control_plane"] def test_nico_control_plane_config_wires_api_health() -> None: @@ -551,11 +552,12 @@ def fake_forge_get( ] -def test_nico_iam_config_platform_matches_command_group() -> None: - """The orchestrator uses tests.platform to look up the IAM commands group.""" +def test_nico_iam_plain_suite_has_one_command_group() -> None: + """A plain suite derives execution identity from its sole command group.""" merged, _steps = _merged_nico_config_steps("iam.yaml", "iam") - assert merged["tests"]["platform"] == "iam" + assert "platform" not in merged["tests"] + assert list(merged["commands"]) == ["iam"] def test_nico_iam_config_wires_credential_readiness() -> None: @@ -568,7 +570,7 @@ def test_nico_iam_config_wires_credential_readiness() -> None: validations = merged["tests"]["validations"] assert merged["tests"]["settings"]["nico_api_base"] == "{{env.NICO_API_BASE}}" assert validations["credential_readiness"]["step"] == "check_credentials" - assert validations["credential_readiness"]["checks"]["FieldExistsCheck"]["fields"] == [ + assert validations["credential_readiness"]["checks"]["FieldExistsCheck-iam_credential_readiness"]["fields"] == [ "account_id", "authenticated", "tests", @@ -836,11 +838,12 @@ def test_nico_instance_inventory_scripts_skip_when_site_has_no_instances( assert "No instances found" in describe_payload["skip_reason"] -def test_nico_network_config_platform_matches_command_group() -> None: - """The orchestrator uses tests.platform to look up the network commands group.""" +def test_nico_network_plain_suite_has_one_command_group() -> None: + """A plain suite derives execution identity from its sole command group.""" merged, _steps = _merged_nico_config_steps("network.yaml", "network") - assert merged["tests"]["platform"] == "network" + assert "platform" not in merged["tests"] + assert list(merged["commands"]) == ["network"] def test_nico_network_config_wires_network_inventory_probes() -> None: diff --git a/isvctl/tests/test_capacity_config.py b/isvctl/tests/test_capacity_config.py index b249804d9..319b17ccc 100644 --- a/isvctl/tests/test_capacity_config.py +++ b/isvctl/tests/test_capacity_config.py @@ -24,6 +24,7 @@ def test_security_suite_defines_capacity_validations() -> None: "CapacityReservationGroupingCheck": { "test_id": "CAP04-01", "labels": ["bare_metal", "capacity", "min_req", "security"], + "requires": ["compute"], "step": "capacity_reservation_grouping", "min_resources": "{{min_resources}}", } @@ -36,6 +37,7 @@ def test_security_suite_defines_capacity_validations() -> None: "CapacityTopologyBlockAtomicAllocationCheck": { "test_id": "CAP04-02", "labels": ["bare_metal", "capacity", "min_req", "security"], + "requires": ["compute"], "min_resources": 2, } } diff --git a/isvctl/tests/test_merger.py b/isvctl/tests/test_merger.py index ce9bf919f..950a59063 100644 --- a/isvctl/tests/test_merger.py +++ b/isvctl/tests/test_merger.py @@ -437,23 +437,25 @@ def test_aws_iam_inherits_test_validations(self) -> None: assert "credentials" in validations assert "teardown_checks" in validations assert result["tests"]["cluster_name"] == "aws-iam-validation" - assert result["tests"]["platform"] == "iam" + assert "platform" not in result["tests"] + assert list(result["commands"]) == ["iam"] - def test_my_isv_observability_declares_raw_platform_for_report_upload(self) -> None: - """Raw observability config exposes platform for upload paths that skip imports.""" + def test_my_isv_observability_is_a_plain_suite(self) -> None: + """Plain-suite provider configs do not recreate a platform axis.""" config_path = self.CONFIGS_DIR / "providers" / "my-isv" / "config" / "observability.yaml" raw_config = yaml.safe_load(config_path.read_text()) or {} - assert raw_config.get("tests", {}).get("platform") == "observability" + assert "platform" not in raw_config.get("tests", {}) result = merge_yaml_files([config_path]) - assert result["tests"]["platform"] == "observability" + assert "platform" not in result["tests"] + assert list(result["commands"]) == ["observability"] def test_aws_observability_inherits_supported_validations(self) -> None: """AWS observability imports the canonical suite and wires supported steps.""" result = merge_yaml_files([self.CONFIGS_DIR / "providers" / "aws" / "config" / "observability.yaml"]) - assert result["tests"]["platform"] == "observability" + assert "platform" not in result["tests"] assert result["tests"]["cluster_name"] == "aws-observability-validation" steps = result["commands"]["observability"]["steps"] @@ -594,7 +596,7 @@ def test_aws_bare_metal_overrides_serial_console_retention_check(self) -> None: result = merge_yaml_files([self.CONFIGS_DIR / "providers" / "aws" / "config" / "bare_metal.yaml"]) checks = result["tests"]["validations"]["serial_console"]["checks"] - assert checks == [{"SerialConsoleCheck": {}}] + assert checks == [{"SerialConsoleCheck-bm_serial_console": {}}] def test_microk8s_inherits_k8s_validations(self) -> None: """providers/microk8s.yaml imports suites/k8s.yaml and adds overrides.""" From 9ed3b96a13a3e8c78322b036d93258e1d5e0996e Mon Sep 17 00:00:00 2001 From: Alexandre Begnoche Date: Wed, 22 Jul 2026 13:31:54 -0400 Subject: [PATCH 07/34] fix(make): filter storage demo to vm capability context my-isv storage demo exercises compute-path checks only; pass --capabilities vm so kubernetes requires checks are skipped with explicit capability_requirement reasons. Signed-off-by: Alexandre Begnoche --- Makefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 56fcacae2..5a7063a08 100644 --- a/Makefile +++ b/Makefile @@ -89,9 +89,10 @@ define run_demo @echo "==========================================" @echo "Demo test: $(1)" @echo "==========================================" - @echo "Running cmd: ISVCTL_DEMO_MODE=1 uv run isvctl test run -f isvctl/configs/providers/my-isv/config/$(1).yaml" + @echo "Running cmd: ISVCTL_DEMO_MODE=1 uv run isvctl test run -f isvctl/configs/providers/my-isv/config/$(1).yaml$(if $(filter storage,$(1)), --capabilities vm,)" @ISVCTL_DEMO_MODE=1 uv run isvctl test run \ - -f isvctl/configs/providers/my-isv/config/$(1).yaml + -f isvctl/configs/providers/my-isv/config/$(1).yaml \ + $(if $(filter storage,$(1)),--capabilities vm,) endef demo-test: demo-all ## Alias for demo-all (backward compat) From 102e822bdedb034522e4b2326af37f031c726433 Mon Sep 17 00:00:00 2001 From: Alexandre Begnoche Date: Wed, 22 Jul 2026 13:33:48 -0400 Subject: [PATCH 08/34] fix(catalog): align upload envelope with backend platforms/suites Emit declarable capabilities as `platforms` and plain suite names as `suites` in catalog documents and upload payloads so backend catalog ingestion matches the feat/test-structure contract. --- isvctl/src/isvctl/cli/catalog.py | 3 +- isvctl/tests/test_catalog_cli.py | 3 +- isvreporter/src/isvreporter/client.py | 9 ++++-- isvreporter/tests/test_catalog_upload.py | 13 +++++---- isvtest/src/isvtest/catalog.py | 36 +++++++++++++++++++----- isvtest/tests/test_catalog.py | 15 ++++++++-- 6 files changed, 60 insertions(+), 19 deletions(-) diff --git a/isvctl/src/isvctl/cli/catalog.py b/isvctl/src/isvctl/cli/catalog.py index 7a55940f5..e9478d41d 100644 --- a/isvctl/src/isvctl/cli/catalog.py +++ b/isvctl/src/isvctl/cli/catalog.py @@ -224,7 +224,8 @@ def push( isv_test_version=catalog_version, entries=catalog_entries, schema_version=document["schemaVersion"], - capabilities=document["capabilities"], + platforms=document["platforms"], + suites=document["suites"], ): print_progress(typer.style("[OK]", fg=typer.colors.GREEN) + " Catalog push complete") else: diff --git a/isvctl/tests/test_catalog_cli.py b/isvctl/tests/test_catalog_cli.py index 93f6a32ae..def100bc5 100644 --- a/isvctl/tests/test_catalog_cli.py +++ b/isvctl/tests/test_catalog_cli.py @@ -80,7 +80,8 @@ def test_catalog_list_json() -> None: assert payload["schemaVersion"] == 2 assert payload["isvTestVersion"] == "1.2.3" assert payload["entries"] == _FAKE_ENTRIES - assert "kubernetes" in payload["capabilities"] + assert "kubernetes" in payload["platforms"] + assert "storage" in payload["suites"] def test_catalog_labels_table() -> None: diff --git a/isvreporter/src/isvreporter/client.py b/isvreporter/src/isvreporter/client.py index bf7e70c4f..869ef33e2 100644 --- a/isvreporter/src/isvreporter/client.py +++ b/isvreporter/src/isvreporter/client.py @@ -283,7 +283,8 @@ def upload_test_catalog( entries: list[dict[str, Any]], *, schema_version: int = 2, - capabilities: list[str] | None = None, + platforms: list[str] | None = None, + suites: list[str] | None = None, ) -> bool: """Upload test catalog for a suite version (idempotent per version). @@ -298,7 +299,8 @@ def upload_test_catalog( entries: List of catalog entry dicts with keys: name, description, labels, source, suite, platform, requires, test_ids schema_version: Catalog document schema version. - capabilities: Declarable capability vocabulary. + platforms: Declarable capability vocabulary (platform suites). + suites: Plain suite names declared by the catalog. Returns: True if catalog was uploaded or already exists, False on error @@ -320,7 +322,8 @@ def upload_test_catalog( payload = { "schemaVersion": schema_version, "isvTestVersion": isv_test_version, - "capabilities": capabilities or [], + "platforms": platforms or [], + "suites": suites or [], "entries": [ { "name": e["name"], diff --git a/isvreporter/tests/test_catalog_upload.py b/isvreporter/tests/test_catalog_upload.py index 29e8c7684..f57f3e5a4 100644 --- a/isvreporter/tests/test_catalog_upload.py +++ b/isvreporter/tests/test_catalog_upload.py @@ -82,7 +82,8 @@ def test_successful_upload(self, mock_urlopen: MagicMock) -> None: payload = json.loads(request.data.decode()) assert payload["isvTestVersion"] == "1.2.3" assert payload["schemaVersion"] == 2 - assert payload["capabilities"] == [] + assert payload["platforms"] == [] + assert payload["suites"] == [] assert len(payload["entries"]) == 2 assert payload["entries"][0]["name"] == "TestA" assert payload["entries"][0]["labels"] == ["k8s"] @@ -198,8 +199,8 @@ def test_empty_optional_fields_use_defaults(self, mock_urlopen: MagicMock) -> No assert entry["test_ids"] == [] @patch("isvreporter.client.urlopen") - def test_forwards_capability_vocabulary(self, mock_urlopen: MagicMock) -> None: - """Schema version and declarable capabilities are sent in the envelope.""" + def test_forwards_catalog_axis_vocabulary(self, mock_urlopen: MagicMock) -> None: + """Schema version and catalog axis lists are sent in the envelope.""" get_response = MagicMock() get_response.read.return_value = json.dumps([]).encode() get_response.__enter__ = MagicMock(return_value=get_response) @@ -216,13 +217,15 @@ def test_forwards_capability_vocabulary(self, mock_urlopen: MagicMock) -> None: isv_test_version="1.2.3", entries=[{"name": "TestA"}], schema_version=2, - capabilities=["kubernetes", "vm"], + platforms=["kubernetes", "vm"], + suites=["storage", "iam"], ) request = mock_urlopen.call_args_list[1][0][0] payload = json.loads(request.data.decode()) assert payload["schemaVersion"] == 2 - assert payload["capabilities"] == ["kubernetes", "vm"] + assert payload["platforms"] == ["kubernetes", "vm"] + assert payload["suites"] == ["storage", "iam"] @patch("isvreporter.client.urlopen") def test_markers_field_is_not_forwarded(self, mock_urlopen: MagicMock) -> None: diff --git a/isvtest/src/isvtest/catalog.py b/isvtest/src/isvtest/catalog.py index fb54e4f30..675addd38 100644 --- a/isvtest/src/isvtest/catalog.py +++ b/isvtest/src/isvtest/catalog.py @@ -32,7 +32,7 @@ from isvreporter.version import get_version from isvtest.core.discovery import discover_all_tests -from isvtest.core.resolution import resolve_class_key +from isvtest.core.resolution import DECLARABLE_CAPABILITIES, resolve_class_key from isvtest.release_manifest import INCLUDE_UNRELEASED_ENV, load_released_test_filter logger = logging.getLogger(__name__) @@ -305,21 +305,43 @@ def build_catalog(*, released_only: bool = True) -> list[dict[str, Any]]: def build_capability_vocabulary() -> list[str]: """Return declarable capabilities derived from platform suite YAML.""" suite_map = _build_suite_map() - return sorted({entry["platform"] for entry in suite_map.values() if entry["platform"]}) + return sorted( + platform + for platform in {entry["platform"] for entry in suite_map.values() if entry["platform"]} + if platform in DECLARABLE_CAPABILITIES + ) + + +def build_suite_vocabulary() -> list[str]: + """Return plain suite names declared by canonical suite YAML.""" + configs_dir = _find_configs_dir() + if not configs_dir: + return [] + + suites: set[str] = set() + for config_path in sorted((configs_dir / "suites").glob("*.yaml")): + data = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {} + tests = data.get("tests") or {} + platform = tests.get("platform") if isinstance(tests, dict) else None + if isinstance(platform, str) and platform in DECLARABLE_CAPABILITIES: + continue + suites.add(config_path.stem.replace("-", "_")) + return sorted(suites) def catalog_document(entries: list[dict[str, Any]], version: str) -> dict[str, Any]: """Wrap catalog ``entries`` in the versioned upload/artifact envelope. - Adds the schema version, the isvtest package version, and the declarable - capability vocabulary. The per-entry ``labels`` are - intentionally not summarized at the top level - a consumer can derive the - label universe from the entries when needed. + Adds the schema version, the isvtest package version, and the catalog axis + vocabulary expected by the backend upload contract. The per-entry + ``labels`` are intentionally not summarized at the top level - a consumer + can derive the label universe from the entries when needed. """ return { "schemaVersion": CATALOG_SCHEMA_VERSION, "isvTestVersion": version, - "capabilities": build_capability_vocabulary(), + "platforms": build_capability_vocabulary(), + "suites": build_suite_vocabulary(), "entries": entries, } diff --git a/isvtest/tests/test_catalog.py b/isvtest/tests/test_catalog.py index 209dae01d..1e7ad6e2f 100644 --- a/isvtest/tests/test_catalog.py +++ b/isvtest/tests/test_catalog.py @@ -21,6 +21,7 @@ CATALOG_SCHEMA_VERSION, build_capability_vocabulary, build_catalog, + build_suite_vocabulary, catalog_document, get_catalog_version, ) @@ -44,14 +45,24 @@ def test_derives_capabilities_from_platform_suites(self) -> None: """Only real platform suite keys are declarable capabilities.""" assert build_capability_vocabulary() == ["bare_metal", "kubernetes", "slurm", "vm"] + def test_derives_suite_vocabulary_from_plain_suites(self) -> None: + """Plain suite YAML files are listed separately from platform suites.""" + suites = build_suite_vocabulary() + assert "iam" in suites + assert "storage" in suites + assert "kubernetes" not in suites + assert "vm" not in suites + def test_catalog_document_wraps_entries_with_metadata(self) -> None: - """The envelope carries schema version, package version, and capabilities.""" + """The envelope carries schema version, package version, and axis lists.""" entries = [{"name": "X", "labels": ["iam"]}] doc = catalog_document(entries, "1.2.3") assert doc["schemaVersion"] == CATALOG_SCHEMA_VERSION assert doc["isvTestVersion"] == "1.2.3" assert doc["entries"] == entries - assert doc["capabilities"] == build_capability_vocabulary() + assert doc["platforms"] == build_capability_vocabulary() + assert doc["suites"] == build_suite_vocabulary() + assert "capabilities" not in doc # The label universe is intentionally not summarized at the top level. assert "labels" not in doc From d3d114f4ebfcb001cc373708369970c954ad5772 Mon Sep 17 00:00:00 2001 From: Alexandre Begnoche Date: Fri, 17 Jul 2026 11:06:12 -0400 Subject: [PATCH 09/34] feat: add --dry-run alias for catalog push --no-upload Both flags now build and save the catalog locally without uploading, using --dry-run as the primary name for consistency with other commands. Signed-off-by: Alexandre Begnoche (cherry picked from commit 15506977a3a7fdc98b127d3a39dea26ead451a01) --- isvctl/src/isvctl/cli/catalog.py | 14 +++++++++----- isvctl/tests/test_catalog_cli.py | 22 ++++++++++++++++++++++ 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/isvctl/src/isvctl/cli/catalog.py b/isvctl/src/isvctl/cli/catalog.py index e9478d41d..2d0790a44 100644 --- a/isvctl/src/isvctl/cli/catalog.py +++ b/isvctl/src/isvctl/cli/catalog.py @@ -169,9 +169,13 @@ def push( bool, typer.Option("--verbose", "-v", help="Enable verbose logging"), ] = False, - no_upload: Annotated[ + dry_run: Annotated[ bool, - typer.Option("--no-upload", help="Build and save locally without uploading"), + typer.Option( + "--dry-run", + "--no-upload", + help="Build and save locally without uploading", + ), ] = False, ) -> None: """Build the test catalog and upload it to ISV Lab Service. @@ -183,7 +187,7 @@ def push( Examples: isvctl catalog push - isvctl catalog push --no-upload + isvctl catalog push --dry-run """ setup_logging(verbose) @@ -198,8 +202,8 @@ def push( catalog_path.write_text(json.dumps(document, indent=2)) print_progress(f" Saved to: {catalog_path}") - if no_upload: - print_progress("Skipping upload (--no-upload)") + if dry_run: + print_progress("Dry run: saved catalog locally (upload skipped)") return from isvctl.reporting import check_upload_credentials, get_environment_config diff --git a/isvctl/tests/test_catalog_cli.py b/isvctl/tests/test_catalog_cli.py index def100bc5..43f24b9d0 100644 --- a/isvctl/tests/test_catalog_cli.py +++ b/isvctl/tests/test_catalog_cli.py @@ -16,8 +16,10 @@ """Unit tests for the catalog CLI subcommand.""" import json +from pathlib import Path from unittest.mock import patch +import pytest from typer.testing import CliRunner from isvctl.cli.catalog import app @@ -160,3 +162,23 @@ def test_catalog_list_unreleased_json() -> None: build_catalog.assert_called_once_with(released_only=False) payload = json.loads(result.output) assert payload["entries"] == [_FAKE_ENTRIES[1]] + + +@pytest.mark.parametrize("flag", ["--dry-run", "--no-upload"]) +def test_catalog_push_dry_run_saves_without_upload(flag: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """`catalog push --dry-run` / `--no-upload` saves locally and skips upload.""" + monkeypatch.setattr("isvctl.cli.catalog.get_output_dir", lambda: tmp_path) + with ( + patch("isvctl.cli.catalog.build_catalog", return_value=_FAKE_ENTRIES), + patch("isvctl.cli.catalog.get_catalog_version", return_value="1.2.3"), + patch("isvctl.reporting.check_upload_credentials") as check_creds, + ): + result = runner.invoke(app, ["push", flag]) + + assert result.exit_code == 0, result.output + check_creds.assert_not_called() + catalog_path = tmp_path / "test_catalog.json" + assert catalog_path.exists() + payload = json.loads(catalog_path.read_text(encoding="utf-8")) + assert payload["entries"] == _FAKE_ENTRIES + assert "Dry run: saved catalog locally" in result.output From da36b178296e1b87463ca77dd2e1bc1218d91c4d Mon Sep 17 00:00:00 2001 From: Alexandre Begnoche Date: Wed, 22 Jul 2026 13:53:46 -0400 Subject: [PATCH 10/34] fix(docs): sync test-plan labels with storage suite wiring K8S23-04..07 gained the storage label in the suite YAML; the test-plan coverage guardrail requires the doc to carry the same label union. Co-Authored-By: Claude Fable 5 Signed-off-by: Alexandre Begnoche --- docs/test-plan.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/test-plan.yaml b/docs/test-plan.yaml index 5913c35d4..78f4b6aa8 100644 --- a/docs/test-plan.yaml +++ b/docs/test-plan.yaml @@ -3243,6 +3243,7 @@ domains: labels: - kubernetes - min_req + - storage priority: P0 milestone: M5 notes: "" @@ -3255,6 +3256,7 @@ domains: labels: - kubernetes - min_req + - storage priority: P0 milestone: M5 notes: "" @@ -3267,6 +3269,7 @@ domains: labels: - kubernetes - min_req + - storage priority: P0 milestone: M5 notes: "" @@ -3279,6 +3282,7 @@ domains: labels: - kubernetes - min_req + - storage priority: P0 milestone: M5 notes: "" From a7e72546b66b10d3da9e5d722bd5472b7f9b3469 Mon Sep 17 00:00:00 2001 From: Alexandre Begnoche Date: Wed, 22 Jul 2026 13:53:46 -0400 Subject: [PATCH 11/34] chore(make): say suites, not domains, in demo output Co-Authored-By: Claude Fable 5 Signed-off-by: Alexandre Begnoche --- Makefile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 5a7063a08..7722f55b2 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ -MY_ISV_DOMAINS := bare_metal control-plane iam image-registry network observability security storage vm -DEMO_TARGETS := $(addprefix demo-,$(MY_ISV_DOMAINS)) +MY_ISV_SUITES := bare_metal control-plane iam image-registry network observability security storage vm +DEMO_TARGETS := $(addprefix demo-,$(MY_ISV_SUITES)) .PHONY: help pre-commit build test coverage clean lint format install bump-patch bump-fix bump-minor bump-feat bump-major bump bump-check \ security-trivy security-trivy-detail security-trufflehog ci-security demo-test demo-all $(DEMO_TARGETS) plan plan-coverage validate-suites \ @@ -97,14 +97,14 @@ endef demo-test: demo-all ## Alias for demo-all (backward compat) -demo-all: ## Run all my-isv living examples (or demo- for one, e.g. demo-security) +demo-all: ## Run all my-isv living examples (or demo- for one, e.g. demo-security) @echo "Running my-isv living examples in demo mode..." - @for domain in $(MY_ISV_DOMAINS); do \ - $(MAKE) --no-print-directory demo-$$domain || exit 1; \ + @for suite in $(MY_ISV_SUITES); do \ + $(MAKE) --no-print-directory demo-$$suite || exit 1; \ done @echo "" @echo "✅ All my-isv living examples passed in demo mode!" - @echo "Domains: $(MY_ISV_DOMAINS)" + @echo "Suites: $(MY_ISV_SUITES)" $(DEMO_TARGETS): demo-%: $(call run_demo,$*) From 6d4e62e12f1b11f062341b8f6a659ca172d7dcc8 Mon Sep 17 00:00:00 2001 From: Alexandre Begnoche Date: Wed, 22 Jul 2026 15:40:07 -0400 Subject: [PATCH 12/34] refactor(suites): use any-match requirement semantics Signed-off-by: Alexandre Begnoche --- isvctl/configs/suites/network.yaml | 18 ++++---- isvctl/configs/suites/observability.yaml | 10 ++--- isvctl/configs/suites/security.yaml | 6 +-- isvctl/configs/suites/storage.yaml | 46 ++++++++++---------- isvctl/src/isvctl/cli/test.py | 14 +++--- isvctl/src/isvctl/config/schema.py | 9 +--- isvctl/src/isvctl/config/suite_resolution.py | 13 ++---- isvctl/src/isvctl/orchestrator/loop.py | 5 +-- isvctl/tests/test_capacity_config.py | 4 +- isvctl/tests/test_catalog_cli.py | 2 +- isvctl/tests/test_suite_resolution.py | 8 ++-- isvreporter/tests/test_catalog_upload.py | 2 +- isvtest/src/isvtest/core/resolution.py | 15 +++---- isvtest/tests/test_catalog.py | 4 +- isvtest/tests/test_resolution.py | 11 ++--- scripts/validate_suite_wiring.py | 11 ++--- 16 files changed, 80 insertions(+), 98 deletions(-) diff --git a/isvctl/configs/suites/network.yaml b/isvctl/configs/suites/network.yaml index 0fe1dc152..0d32fd03c 100644 --- a/isvctl/configs/suites/network.yaml +++ b/isvctl/configs/suites/network.yaml @@ -132,12 +132,12 @@ tests: NetworkConnectivityCheck: test_id: "N/A" labels: ["network"] - requires: [compute] + requires: [vm, bare_metal] step: connectivity_test TrafficFlowCheck: test_id: "N/A" labels: ["network"] - requires: [compute] + requires: [vm, bare_metal] step: traffic_validation VpcIpConfigCheck: test_id: "IPAM02-01" @@ -147,7 +147,7 @@ tests: DhcpIpManagementCheck: test_id: "IPAM01-01" labels: ["network", "ssh"] - requires: [compute] + requires: [vm, bare_metal] step: dhcp_ip_test sg_scoping: @@ -215,17 +215,17 @@ tests: StablePrivateIpCheck: test_id: "SDN11-01" labels: ["min_req", "network"] - requires: [compute] + requires: [vm, bare_metal] step: stable_ip_test StableEgressIpCheck: test_id: "DMS05-01" labels: ["min_req", "network"] - requires: [compute] + requires: [vm, bare_metal] step: stable_egress_ip_test FloatingIpCheck: test_id: "SDN05-01" labels: ["min_req", "network"] - requires: [compute] + requires: [vm, bare_metal] step: floating_ip_test max_switch_seconds: 10 LocalizedDnsCheck: @@ -241,7 +241,7 @@ tests: StorageL3RoutingCheck: test_id: "SDN08-01" labels: ["min_req", "network"] - requires: [compute] + requires: [vm, bare_metal] step: storage_l3_routing # Read-only metadata/API checks for provider-reported backend fabric @@ -251,12 +251,12 @@ tests: BackendSwitchFabricCheck: test_id: "NET01-01" labels: ["min_req", "network"] - requires: [compute] + requires: [vm, bare_metal] step: backend_switch_fabric NvlinkDomainCheck: test_id: "NET02-02" labels: ["min_req", "network"] - requires: [compute] + requires: [vm, bare_metal] step: nvlink_domain teardown_checks: diff --git a/isvctl/configs/suites/observability.yaml b/isvctl/configs/suites/observability.yaml index 9a78e48b4..e67d0f615 100644 --- a/isvctl/configs/suites/observability.yaml +++ b/isvctl/configs/suites/observability.yaml @@ -39,7 +39,7 @@ tests: TelemetryDeliveryLatencyCheck: test_id: "OBS05-01" labels: ["min_req", "observability"] - requires: [compute] + requires: [vm, bare_metal] step: telemetry_delivery_latency max_delivery_seconds: "{{max_delivery_seconds}}" @@ -48,7 +48,7 @@ tests: NorthSouthNetworkTelemetryCheck: test_id: "OBS06-01" labels: ["min_req", "network", "observability"] - requires: [compute] + requires: [vm, bare_metal] step: north_south_network_telemetry EastWestNetworkTelemetryCheck: test_id: "OBS07-01" @@ -68,7 +68,7 @@ tests: HostNicNetworkTelemetryCheck: test_id: "OBS10-01" labels: ["min_req", "network", "observability"] - requires: [compute] + requires: [vm, bare_metal] step: host_nic_network_telemetry network_logs: @@ -84,7 +84,7 @@ tests: HostSyslogCheck: test_id: "OBS18-01" labels: ["bare_metal", "min_req", "observability"] - requires: [compute] + requires: [vm, bare_metal] step: host_syslogs bmc_logs: @@ -116,7 +116,7 @@ tests: StoragePerformanceTelemetryCheck: test_id: "TELEM06-01" labels: ["min_req", "observability"] - requires: [compute] + requires: [vm, bare_metal] step: storage_performance_telemetry gpu_nvlink_telemetry: diff --git a/isvctl/configs/suites/security.yaml b/isvctl/configs/suites/security.yaml index d17098ea7..94bb565c1 100644 --- a/isvctl/configs/suites/security.yaml +++ b/isvctl/configs/suites/security.yaml @@ -206,7 +206,7 @@ tests: TenantIsolationCheck: test_id: "SEC11-01" labels: ["iam", "min_req", "network", "security"] - requires: [compute] + requires: [vm, bare_metal] step: tenant_isolation_test # Verify capacity is logically grouped and pinned to one tenant/account @@ -216,7 +216,7 @@ tests: CapacityReservationGroupingCheck: test_id: "CAP04-01" labels: ["bare_metal", "capacity", "min_req", "security"] - requires: [compute] + requires: [vm, bare_metal] step: capacity_reservation_grouping min_resources: "{{min_resources}}" @@ -228,7 +228,7 @@ tests: CapacityTopologyBlockAtomicAllocationCheck: test_id: "CAP04-02" labels: ["bare_metal", "capacity", "min_req", "security"] - requires: [compute] + requires: [vm, bare_metal] min_resources: 2 teardown_checks: diff --git a/isvctl/configs/suites/storage.yaml b/isvctl/configs/suites/storage.yaml index 00eefa5e7..94b44d2c8 100644 --- a/isvctl/configs/suites/storage.yaml +++ b/isvctl/configs/suites/storage.yaml @@ -73,7 +73,7 @@ tests: InstanceStateCheck-storage_instance_launched: test_id: "N/A" labels: ["storage"] - requires: [compute] + requires: [vm, bare_metal] expected_state: "running" fixture_volume: @@ -82,16 +82,16 @@ tests: StepSuccessCheck-storage_fixture_volume: test_id: "N/A" labels: ["storage"] - requires: [compute] + requires: [vm, bare_metal] FieldExistsCheck-storage_fixture_volume: test_id: "N/A" labels: ["storage"] - requires: [compute] + requires: [vm, bare_metal] fields: ["volume_id", "mount_point", "operations"] CrudOperationsCheck-storage_fixture_volume: test_id: "N/A" labels: ["storage"] - requires: [compute] + requires: [vm, bare_metal] operations: ["create", "attach", "format", "mount", "write_sentinel"] # ─── DATASVC-XX-02: volume snapshots (issue #321) ──────────────── @@ -101,16 +101,16 @@ tests: StepSuccessCheck-storage_snapshot_lifecycle: test_id: "N/A" labels: ["storage"] - requires: [compute] + requires: [vm, bare_metal] FieldExistsCheck-storage_snapshot_lifecycle: test_id: "N/A" labels: ["storage"] - requires: [compute] + requires: [vm, bare_metal] fields: ["volume_id", "snapshot_id", "operations"] CrudOperationsCheck-storage_snapshot_lifecycle: test_id: "DATASVC02-01" labels: ["min_req", "storage"] - requires: [compute] + requires: [vm, bare_metal] operations: ["create_snapshot", "restore_volume", "verify_data"] # ─── DATASVC-XX-03: volume resizing (issue #322) ───────────────── @@ -120,16 +120,16 @@ tests: StepSuccessCheck-storage_volume_resize: test_id: "N/A" labels: ["storage"] - requires: [compute] + requires: [vm, bare_metal] FieldExistsCheck-storage_volume_resize: test_id: "N/A" labels: ["storage"] - requires: [compute] + requires: [vm, bare_metal] fields: ["volume_id", "operations"] CrudOperationsCheck-storage_volume_resize: test_id: "DATASVC03-01" labels: ["min_req", "storage"] - requires: [compute] + requires: [vm, bare_metal] operations: ["modify_volume", "grow_partition", "resize_filesystem", "verify_size"] # ─── DATASVC-XX-04: persistence across restarts (issue #323) ───── @@ -139,16 +139,16 @@ tests: StepSuccessCheck-storage_volume_persistence: test_id: "N/A" labels: ["storage"] - requires: [compute] + requires: [vm, bare_metal] FieldExistsCheck-storage_volume_persistence: test_id: "N/A" labels: ["storage"] - requires: [compute] + requires: [vm, bare_metal] fields: ["volume_id", "operations"] CrudOperationsCheck-storage_volume_persistence: test_id: "DATASVC04-01" labels: ["min_req", "storage"] - requires: [compute] + requires: [vm, bare_metal] operations: ["stop", "start", "verify_attached", "verify_data"] # ─── Home directory storage ────────────────────────────────────── @@ -158,15 +158,15 @@ tests: DirectoryFilesystemQuotaCheck: test_id: "DIR01-01" labels: ["min_req", "storage"] - requires: [compute] + requires: [vm, bare_metal] DirectoryUsageAccountingCheck: test_id: "DIR01-02" labels: ["min_req", "storage"] - requires: [compute] + requires: [vm, bare_metal] DirectoryNfsAvailabilityCheck: test_id: "DIR02-01" labels: ["min_req", "storage"] - requires: [compute] + requires: [vm, bare_metal] # ─── High-speed storage: SDS controller ────────────────────────── sds_controller: @@ -179,7 +179,7 @@ tests: HssQosThroughputCheck: test_id: "HSS02-01" labels: ["min_req", "storage"] - requires: [compute] + requires: [vm, bare_metal] step: qos_throughput HssNonDisruptiveUpgradeCheck: test_id: "HSS05-01" @@ -189,7 +189,7 @@ tests: HssRdmaMemoryProtectionCheck: test_id: "HSS06-01" labels: ["min_req", "storage"] - requires: [compute] + requires: [vm, bare_metal] step: rdma_memory_protection # ─── High-speed storage: parallel file system services ─────────── @@ -198,7 +198,7 @@ tests: HssParallelFsProvisioningCheck: test_id: "HSS07-01" labels: ["min_req", "storage"] - requires: [compute] + requires: [vm, bare_metal] step: provision_parallel_fs HssMultipleFilesystemsCheck: test_id: "HSS09-01" @@ -213,7 +213,7 @@ tests: HssQuotaEnforcementCheck: test_id: "HSS12-01" labels: ["min_req", "storage"] - requires: [compute] + requires: [vm, bare_metal] step: quota_enforcement HssRootSquashCheck: test_id: "HSS13-01" @@ -223,17 +223,17 @@ tests: HssFlockMountCheck: test_id: "HSS14-01" labels: ["min_req", "storage"] - requires: [compute] + requires: [vm, bare_metal] step: flock_mount HssChangelogAuditCheck: test_id: "HSS15-01" labels: ["min_req", "storage"] - requires: [compute] + requires: [vm, bare_metal] step: changelog_audit HssMultipathCheck: test_id: "HSS18-01" labels: ["min_req", "storage"] - requires: [compute] + requires: [vm, bare_metal] step: multipath # ─── Teardown ──────────────────────────────────────────────────── diff --git a/isvctl/src/isvctl/cli/test.py b/isvctl/src/isvctl/cli/test.py index 002f0df2b..214783c0c 100644 --- a/isvctl/src/isvctl/cli/test.py +++ b/isvctl/src/isvctl/cli/test.py @@ -28,7 +28,7 @@ import typer import yaml from isvtest.catalog import build_catalog, catalog_document, get_catalog_version -from isvtest.core.resolution import expand_capabilities, parse_validations +from isvtest.core.resolution import parse_validations, requirements_satisfied from isvtest.release_manifest import load_released_test_filter from isvctl.cli import setup_logging @@ -50,7 +50,6 @@ from isvctl.config.schema import RunConfig from isvctl.config.suite_resolution import SuiteResolutionError, parse_capabilities, resolve_suite from isvctl.orchestrator.loop import Orchestrator, Phase -from isvctl.redaction import redact_dict from isvctl.reporting import check_upload_credentials, create_test_run, get_environment_config, update_test_run logger = logging.getLogger(__name__) @@ -122,7 +121,6 @@ def _human_readable_dry_run(config: RunConfig, capabilities: set[str] | None) -> platform = config.tests.platform if config.tests and config.tests.platform else None suite_type = f"platform ({platform})" if platform else "plain" context = "not filtered" if capabilities is None else ", ".join(sorted(capabilities)) or "(none)" - expanded = expand_capabilities(capabilities or ()) if capabilities is not None else None validations = config.tests.validations if config.tests else {} entries = parse_validations(validations) @@ -134,7 +132,7 @@ def _human_readable_dry_run(config: RunConfig, capabilities: set[str] | None) -> ] for entry in entries: requirement = ", ".join(entry.requires) or "core" - if expanded is not None and not set(entry.requires).issubset(expanded): + if capabilities is not None and not requirements_satisfied(entry.requires, capabilities): declared = ", ".join(sorted(capabilities or ())) or "(none)" lines.append(f" [SKIP] {entry.name}: requires {requirement} (context: {declared})") else: @@ -472,7 +470,9 @@ def run( # Create test run before running tests if upload_results and lab_id: print_progress("Creating test run in ISV Lab Service...") - platform = config.tests.platform if config.tests and config.tests.platform else next(iter(config.commands), "unknown") + platform = ( + config.tests.platform if config.tests and config.tests.platform else next(iter(config.commands), "unknown") + ) test_run_id = create_test_run( lab_id=lab_id, platform=platform, @@ -515,9 +515,7 @@ def run( catalog_version = get_catalog_version() print_progress(f"Built test catalog: {len(catalog_entries)} tests (version: {catalog_version})") catalog_path = output_dir / "test_catalog.json" - catalog_path.write_text( - json.dumps(catalog_document(catalog_entries, catalog_version), indent=2) - ) + catalog_path.write_text(json.dumps(catalog_document(catalog_entries, catalog_version), indent=2)) print_progress(f" Saved test catalog to: {catalog_path}") except Exception as e: logger.warning("Failed to build test catalog: %s", e) diff --git a/isvctl/src/isvctl/config/schema.py b/isvctl/src/isvctl/config/schema.py index 75eda669f..698c8ccd6 100644 --- a/isvctl/src/isvctl/config/schema.py +++ b/isvctl/src/isvctl/config/schema.py @@ -24,9 +24,8 @@ from typing import Any -from pydantic import BaseModel, ConfigDict, Field, model_validator - from isvtest.core.resolution import DECLARABLE_CAPABILITIES, parse_validations +from pydantic import BaseModel, ConfigDict, Field, model_validator class LabConfig(BaseModel): @@ -382,12 +381,8 @@ def validate_suite_shape(self) -> "ValidationConfig": """Reject legacy axes and requirements on platform suite checks.""" if self.model_extra and "module" in self.model_extra: raise ValueError("tests.module is no longer supported; plain suites have no axis key") - if self.platform == "compute": - raise ValueError("compute is requirement-only and cannot be declared as tests.platform") if self.platform and self.platform not in DECLARABLE_CAPABILITIES: - raise ValueError( - f"tests.platform must be one of: {', '.join(sorted(DECLARABLE_CAPABILITIES))}" - ) + raise ValueError(f"tests.platform must be one of: {', '.join(sorted(DECLARABLE_CAPABILITIES))}") entries = parse_validations(self.validations) if self.platform and any("requires" in entry.params_template for entry in entries): raise ValueError("requires is not allowed in platform suites") diff --git a/isvctl/src/isvctl/config/suite_resolution.py b/isvctl/src/isvctl/config/suite_resolution.py index 3bc69c57c..84dd283ec 100644 --- a/isvctl/src/isvctl/config/suite_resolution.py +++ b/isvctl/src/isvctl/config/suite_resolution.py @@ -46,7 +46,7 @@ def platform_vocabulary(configs_root: Path) -> frozenset[str]: def parse_capabilities(value: str | None, configs_root: Path) -> set[str] | None: - """Parse a comma-separated capability context, rejecting aliases like compute.""" + """Parse a comma-separated capability context using platform suite names.""" if value is None: return None capabilities = {_normalize_name(item) for item in value.split(",") if item.strip()} @@ -100,22 +100,17 @@ def resolve_suite(provider: str, suite: str, *, configs_root: Path) -> ResolvedS requested = _normalize_name(suite) declarable = platform_vocabulary(configs_root) - classified = [ - (path, *_suite_name(path, declarable)) - for path in sorted(config_dir.glob("*.yaml")) - ] + classified = [(path, *_suite_name(path, declarable)) for path in sorted(config_dir.glob("*.yaml"))] matches = [item for item in classified if item[1] == requested] if not matches: available = sorted({name for _, name, _ in classified}) raise SuiteResolutionError( - f"Provider {provider!r} has no {requested!r} suite. " - f"Available suites: {', '.join(available) or '(none)'}." + f"Provider {provider!r} has no {requested!r} suite. Available suites: {', '.join(available) or '(none)'}." ) if len(matches) > 1: paths = ", ".join(str(path) for path, _, _ in matches) raise SuiteResolutionError( - f"Provider {provider!r} has multiple {requested!r} suite configs ({paths}). " - "Disambiguate with --config/-f." + f"Provider {provider!r} has multiple {requested!r} suite configs ({paths}). Disambiguate with --config/-f." ) path, name, platform = matches[0] return ResolvedSuite(config_path=path, name=name, platform=platform) diff --git a/isvctl/src/isvctl/orchestrator/loop.py b/isvctl/src/isvctl/orchestrator/loop.py index a8c15cd7a..37acaa754 100644 --- a/isvctl/src/isvctl/orchestrator/loop.py +++ b/isvctl/src/isvctl/orchestrator/loop.py @@ -35,9 +35,9 @@ SkipReason, State, ValidationEntry, - expand_capabilities, get_entry_phase, parse_validations, + requirements_satisfied, resolve_class_key, resolve_entries, ) @@ -343,11 +343,10 @@ def _apply_capability_step_gates( """Skip a step when every validation bound to it is requirement-filtered.""" if capabilities is None: return steps - expanded = expand_capabilities(capabilities) gated_steps: list[Any] = [] for step in steps: bound_entries = [entry for entry in validation_entries if entry.step == step.name] - if bound_entries and all(not set(entry.requires).issubset(expanded) for entry in bound_entries): + if bound_entries and all(not requirements_satisfied(entry.requires, capabilities) for entry in bound_entries): logger.info("Skipping step '%s' because all bound validations are capability-filtered", step.name) gated_steps.append(step.model_copy(update={"skip": True})) else: diff --git a/isvctl/tests/test_capacity_config.py b/isvctl/tests/test_capacity_config.py index 319b17ccc..8466513b8 100644 --- a/isvctl/tests/test_capacity_config.py +++ b/isvctl/tests/test_capacity_config.py @@ -24,7 +24,7 @@ def test_security_suite_defines_capacity_validations() -> None: "CapacityReservationGroupingCheck": { "test_id": "CAP04-01", "labels": ["bare_metal", "capacity", "min_req", "security"], - "requires": ["compute"], + "requires": ["vm", "bare_metal"], "step": "capacity_reservation_grouping", "min_resources": "{{min_resources}}", } @@ -37,7 +37,7 @@ def test_security_suite_defines_capacity_validations() -> None: "CapacityTopologyBlockAtomicAllocationCheck": { "test_id": "CAP04-02", "labels": ["bare_metal", "capacity", "min_req", "security"], - "requires": ["compute"], + "requires": ["vm", "bare_metal"], "min_resources": 2, } } diff --git a/isvctl/tests/test_catalog_cli.py b/isvctl/tests/test_catalog_cli.py index 43f24b9d0..9915ca1b1 100644 --- a/isvctl/tests/test_catalog_cli.py +++ b/isvctl/tests/test_catalog_cli.py @@ -43,7 +43,7 @@ "source": "isvtest.validations.beta", "suite": "storage", "platform": None, - "requires": ["compute"], + "requires": ["vm", "bare_metal"], }, ] diff --git a/isvctl/tests/test_suite_resolution.py b/isvctl/tests/test_suite_resolution.py index 7c990341a..08bc606e6 100644 --- a/isvctl/tests/test_suite_resolution.py +++ b/isvctl/tests/test_suite_resolution.py @@ -41,8 +41,8 @@ def test_one_suite_flag_resolves_platform_and_plain_suites(tmp_path: Path) -> No assert plain.platform is None -def test_capabilities_use_catalog_vocabulary_and_reject_compute(tmp_path: Path) -> None: - """Compute remains requirement-only while omitted context disables filtering.""" +def test_capabilities_use_catalog_vocabulary(tmp_path: Path) -> None: + """Unknown capabilities are rejected while omitted context disables filtering.""" _write_catalog(tmp_path) assert parse_capabilities(None, tmp_path) is None @@ -51,11 +51,11 @@ def test_capabilities_use_catalog_vocabulary_and_reject_compute(tmp_path: Path) parse_capabilities("compute", tmp_path) -def test_platform_suites_reject_requires_and_compute_context() -> None: +def test_platform_suites_reject_requires_and_unknown_platforms() -> None: """Platform placement is its obligation; it cannot declare check requirements.""" validation = {"sample": {"checks": {"PlainCheck": {"requires": []}}}} with pytest.raises(ValidationError, match="requires is not allowed in platform suites"): RunConfig.model_validate({"tests": {"platform": "kubernetes", "validations": validation}}) - with pytest.raises(ValidationError, match="compute is requirement-only"): + with pytest.raises(ValidationError, match=r"tests\.platform must be one of"): RunConfig.model_validate({"tests": {"platform": "compute", "validations": {}}}) diff --git a/isvreporter/tests/test_catalog_upload.py b/isvreporter/tests/test_catalog_upload.py index f57f3e5a4..80fb9d5ff 100644 --- a/isvreporter/tests/test_catalog_upload.py +++ b/isvreporter/tests/test_catalog_upload.py @@ -60,7 +60,7 @@ def test_successful_upload(self, mock_urlopen: MagicMock) -> None: "source": "mod.b", "suite": "storage", "platform": None, - "requires": ["compute"], + "requires": ["vm", "bare_metal"], }, ] diff --git a/isvtest/src/isvtest/core/resolution.py b/isvtest/src/isvtest/core/resolution.py index d0d9267d3..4fc8db196 100644 --- a/isvtest/src/isvtest/core/resolution.py +++ b/isvtest/src/isvtest/core/resolution.py @@ -34,7 +34,7 @@ ADAPTER_HANDLED_CATEGORIES = {"reframe"} DEFAULT_VALIDATION_PHASE = "test" DECLARABLE_CAPABILITIES = frozenset({"vm", "bare_metal", "kubernetes", "slurm"}) -REQUIREMENT_VOCABULARY = DECLARABLE_CAPABILITIES | {"compute"} +REQUIREMENT_VOCABULARY = DECLARABLE_CAPABILITIES class State(StrEnum): @@ -122,12 +122,10 @@ def _wiring_requires(params_template: Any) -> tuple[str, ...]: return tuple(item for item in value if isinstance(item, str)) -def expand_capabilities(declared: Iterable[str]) -> frozenset[str]: - """Expand declarable capabilities with requirement-only aliases.""" - expanded = set(declared) - if expanded & {"vm", "bare_metal"}: - expanded.add("compute") - return frozenset(expanded) +def requirements_satisfied(requires: Iterable[str], capabilities: AbstractSet[str]) -> bool: + """Return whether any required platform is present in the capability context.""" + required = set(requires) + return not required or not required.isdisjoint(capabilities) def parse_validations(raw_config: Mapping[str, Any]) -> list[ValidationEntry]: @@ -212,7 +210,6 @@ def resolve_entries( """ resolved: list[ResolvedEntry] = [] env = _create_jinja_env() - expanded_capabilities = expand_capabilities(capabilities or ()) if capabilities is not None else None for entry in entries: config_error = _validate_entry_shape(entry) @@ -237,7 +234,7 @@ def resolve_entries( resolved.append(_skip(entry, SkipReason.EXCLUDED, f"validation '{entry.name}' is excluded by name")) continue - if expanded_capabilities is not None and not set(entry.requires).issubset(expanded_capabilities): + if capabilities is not None and not requirements_satisfied(entry.requires, capabilities): requirement_list = ", ".join(entry.requires) or "(none)" context_list = ", ".join(sorted(capabilities or ())) or "(none)" resolved.append( diff --git a/isvtest/tests/test_catalog.py b/isvtest/tests/test_catalog.py index 1e7ad6e2f..2e27121a0 100644 --- a/isvtest/tests/test_catalog.py +++ b/isvtest/tests/test_catalog.py @@ -185,7 +185,7 @@ def test_catalog_emits_explicit_labels(self) -> None: "ExplicitLabelCatalogCheck": { "suite": "demo", "platform": None, - "requires": ["compute"], + "requires": ["vm", "bare_metal"], } }, ), @@ -207,7 +207,7 @@ def test_catalog_emits_explicit_labels(self) -> None: "source": __name__, "suite": "demo", "platform": None, - "requires": ["compute"], + "requires": ["vm", "bare_metal"], } ] diff --git a/isvtest/tests/test_resolution.py b/isvtest/tests/test_resolution.py index 8cb0c7f0e..40eb2a7a0 100644 --- a/isvtest/tests/test_resolution.py +++ b/isvtest/tests/test_resolution.py @@ -108,21 +108,22 @@ def _resolve( return results[0] -def test_compute_requirement_expands_from_vm_or_bare_metal() -> None: - """Either concrete compute capability satisfies a compute prerequisite.""" - entry = _entry(requires=("compute",)) +def test_any_declared_requirement_satisfies_capability_filter() -> None: + """Orthogonal platform requirements use OR semantics.""" + entry = _entry(requires=("vm", "bare_metal")) assert _resolve(entry, capabilities={"vm"}).is_ready assert _resolve(entry, capabilities={"bare_metal"}).is_ready + assert _resolve(_entry(requires=()), capabilities={"kubernetes"}).is_ready def test_capability_filter_has_explicit_skip_reason() -> None: """An unmet prerequisite reports both the requirement and active context.""" - resolved = _resolve(_entry(requires=("compute",)), capabilities={"kubernetes"}) + resolved = _resolve(_entry(requires=("vm", "bare_metal")), capabilities={"kubernetes"}) assert resolved.state == State.SKIPPED assert resolved.skip_reason == SkipReason.CAPABILITY_REQUIREMENT - assert resolved.message == "requires compute (context: kubernetes)" + assert resolved.message == "requires vm, bare_metal (context: kubernetes)" def test_omitted_capabilities_disables_requirement_filtering() -> None: diff --git a/scripts/validate_suite_wiring.py b/scripts/validate_suite_wiring.py index c7091c160..6ec3c78f3 100644 --- a/scripts/validate_suite_wiring.py +++ b/scripts/validate_suite_wiring.py @@ -41,11 +41,12 @@ from typing import Any import yaml + REPO_ROOT = Path(__file__).resolve().parent.parent SUITES_DIR = REPO_ROOT / "isvctl" / "configs" / "suites" _NEXT_CATEGORY_LINE = re.compile(r"^ \S") DECLARABLE_CAPABILITIES = {"vm", "bare_metal", "kubernetes", "slurm"} -REQUIREMENT_VOCABULARY = DECLARABLE_CAPABILITIES | {"compute"} +REQUIREMENT_VOCABULARY = DECLARABLE_CAPABILITIES def _check_line_patterns(check_name: str) -> tuple[re.Pattern[str], ...]: @@ -159,9 +160,7 @@ def wiring_errors(suites_dir: Path = SUITES_DIR) -> list[str]: if module is not None: errors.append(f"{path}: tests.module is no longer supported") if platform is not None and platform not in DECLARABLE_CAPABILITIES: - errors.append( - f"{path}: tests.platform must be one of: {', '.join(sorted(DECLARABLE_CAPABILITIES))}" - ) + errors.append(f"{path}: tests.platform must be one of: {', '.join(sorted(DECLARABLE_CAPABILITIES))}") for category, name, params in checks: key = (path, category, name) line_numbers = find_check_line_numbers(lines, category, name) @@ -227,9 +226,7 @@ def main(argv: list[str] | None = None) -> int: print(message) return 0 - ok = ( - f"OK: all wired checks in {SUITES_DIR.relative_to(REPO_ROOT)} declare valid suite metadata." - ) + ok = f"OK: all wired checks in {SUITES_DIR.relative_to(REPO_ROOT)} declare valid suite metadata." print(ok) return 0 From 607c12199390a1c9a04d35fd9dadb3308a447e7f Mon Sep 17 00:00:00 2001 From: Alexandre Begnoche Date: Wed, 22 Jul 2026 15:56:12 -0400 Subject: [PATCH 13/34] fix(storage): tear down cluster fixture Signed-off-by: Alexandre Begnoche --- .../configs/providers/aws/config/storage.yaml | 25 +++++++- .../providers/my-isv/config/storage.yaml | 4 +- .../{ensure_cluster.py => setup_cluster.py} | 8 +-- isvctl/configs/suites/storage.yaml | 58 +++++++++---------- 4 files changed, 57 insertions(+), 38 deletions(-) rename isvctl/configs/providers/my-isv/scripts/storage/{ensure_cluster.py => setup_cluster.py} (83%) diff --git a/isvctl/configs/providers/aws/config/storage.yaml b/isvctl/configs/providers/aws/config/storage.yaml index d88620ca3..b2d8e5dcc 100644 --- a/isvctl/configs/providers/aws/config/storage.yaml +++ b/isvctl/configs/providers/aws/config/storage.yaml @@ -16,6 +16,7 @@ # AWS Storage Validation Configuration # # Imports suites/storage.yaml and wires AWS implementations for: +# - Kubernetes CSI (EKS): setup_cluster / teardown_cluster via eks scripts # - Block volume (EBS): launch/create/snapshot/resize/persistence/teardown # - Home directory (FSx for OpenZFS): NFSv4, volume quota, uid/gid accounting # - High-speed storage (FSx for Lustre): provisioning, multiple filesystems, @@ -25,6 +26,9 @@ # Block-volume steps reuse the VM scripts for launch/teardown; volume steps # live in scripts/storage/. FSx steps are self-contained (create → assert → # cleanup) and create real PERSISTENT_2 filesystems (~10-15 min each, cost). +# The EKS fixture reuses Terraform state when present but will provision a +# cluster when none exists; teardown_cluster always runs so that path cannot +# leak. Set AWS_SKIP_TEARDOWN=true to preserve the cluster across runs. # HSS checks ship unreleased; run with ISVTEST_INCLUDE_UNRELEASED=1. # # Usage: @@ -51,12 +55,17 @@ commands: storage: phases: ["setup", "test", "teardown"] steps: - # Idempotent reference implementation: the EKS setup reuses Terraform - # state and emits kubeconfig + CSI StorageClass names. - - name: ensure_cluster + # Kubernetes CSI fixture: provisions (or reuses) the shared EKS Terraform + # stack and emits kubeconfig + CSI StorageClass names. Paired with + # teardown_cluster below so a standalone storage run does not leak the + # cluster. Honors AWS_SKIP_TEARDOWN like eks.yaml. + - name: setup_cluster phase: setup command: "../scripts/eks/setup.sh" timeout: 5400 + env: + TF_AUTO_APPROVE: "true" + TF_VAR_region: "{{region}}" # ─── Block volume (EBS) ──────────────────────────────────────── - name: launch_instance @@ -219,6 +228,16 @@ commands: - "{{teardown_flag}}" timeout: 600 + # Destroy the EKS stack from setup_cluster (including the out-of-band + # static-CSI EBS volume tagged by setup.sh). Keep this last so CSI + # checks finish before the cluster disappears. + - name: teardown_cluster + phase: teardown + command: "../scripts/eks/teardown.sh" + timeout: 1800 + env: + TF_AUTO_APPROVE: "true" + tests: cluster_name: "aws-storage-validation" description: "AWS storage validation (EBS block volume + FSx for Lustre)" diff --git a/isvctl/configs/providers/my-isv/config/storage.yaml b/isvctl/configs/providers/my-isv/config/storage.yaml index d5579e454..c0efca903 100644 --- a/isvctl/configs/providers/my-isv/config/storage.yaml +++ b/isvctl/configs/providers/my-isv/config/storage.yaml @@ -49,9 +49,9 @@ commands: storage: phases: ["setup", "test", "teardown"] steps: - - name: ensure_cluster + - name: setup_cluster phase: setup - command: "python ../scripts/storage/ensure_cluster.py" + command: "python ../scripts/storage/setup_cluster.py" timeout: 60 - name: launch_instance diff --git a/isvctl/configs/providers/my-isv/scripts/storage/ensure_cluster.py b/isvctl/configs/providers/my-isv/scripts/storage/setup_cluster.py similarity index 83% rename from isvctl/configs/providers/my-isv/scripts/storage/ensure_cluster.py rename to isvctl/configs/providers/my-isv/scripts/storage/setup_cluster.py index 12441400f..e1d0525cc 100644 --- a/isvctl/configs/providers/my-isv/scripts/storage/ensure_cluster.py +++ b/isvctl/configs/providers/my-isv/scripts/storage/setup_cluster.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Ensure a Kubernetes cluster with the provider's CSI drivers is available.""" +"""Set up a Kubernetes cluster with the provider's CSI drivers.""" import json import os @@ -28,8 +28,8 @@ def main() -> int: }, } - # TODO: Idempotently reuse or create a cluster with your CSI drivers, - # then return its kubeconfig and StorageClass names using this contract. + # TODO: Provision (or reuse) a cluster with your CSI drivers, then return + # its kubeconfig and StorageClass names using this contract. if DEMO_MODE: result.update( { @@ -46,7 +46,7 @@ def main() -> int: } ) else: - result["error"] = "Not implemented - ensure a Kubernetes cluster with CSI installed" + result["error"] = "Not implemented - set up a Kubernetes cluster with CSI installed" print(json.dumps(result, indent=2)) return 0 if result["success"] else 1 diff --git a/isvctl/configs/suites/storage.yaml b/isvctl/configs/suites/storage.yaml index 94b44d2c8..2852be80a 100644 --- a/isvctl/configs/suites/storage.yaml +++ b/isvctl/configs/suites/storage.yaml @@ -247,7 +247,7 @@ tests: # Kubernetes storage checks use a normal provider-owned cluster fixture. k8s_storage: - step: ensure_cluster + step: setup_cluster checks: K8sCsiStorageTypesCheck: test_id: "K8S23-04" @@ -255,9 +255,9 @@ tests: requires: [kubernetes] # StorageClass names by type; omit (or leave empty) to skip that type. # Env overrides: K8S_CSI_BLOCK_SC, K8S_CSI_SHARED_FS_SC, K8S_CSI_NFS_SC. - block_storage_class: "{{ steps.ensure_cluster.csi.block_storage_class | default('', true) }}" - shared_fs_storage_class: "{{ steps.ensure_cluster.csi.shared_fs_storage_class | default('', true) }}" - nfs_storage_class: "{{ steps.ensure_cluster.csi.nfs_storage_class | default('', true) }}" + block_storage_class: "{{ steps.setup_cluster.csi.block_storage_class | default('', true) }}" + shared_fs_storage_class: "{{ steps.setup_cluster.csi.shared_fs_storage_class | default('', true) }}" + nfs_storage_class: "{{ steps.setup_cluster.csi.nfs_storage_class | default('', true) }}" pvc_size: "1Gi" bind_timeout_s: 120 namespace_prefix: "isvtest-csi-types" @@ -268,7 +268,7 @@ tests: # Uses the block StorageClass by default; skips entirely when unset. # per_sc_quota must exceed pvc_request so the usage PVC fits, and # over_quota_request must exceed per_sc_quota so admission rejects it. - storage_class: "{{ steps.ensure_cluster.csi.block_storage_class | default('', true) }}" + storage_class: "{{ steps.setup_cluster.csi.block_storage_class | default('', true) }}" total_quota: "10Gi" per_sc_quota: "5Gi" pvc_request: "1Gi" @@ -299,14 +299,14 @@ tests: # (not failed) when static_pv.volume_handle / csi_driver are # unset, so providers that don't pre-provision a volume can still # run the dynamic probe. - dynamic_storage_class: "{{ steps.ensure_cluster.csi.block_storage_class | default('', true) }}" + dynamic_storage_class: "{{ steps.setup_cluster.csi.block_storage_class | default('', true) }}" dynamic_pvc_size: "1Gi" static_pv: - volume_handle: "{{ steps.ensure_cluster.csi.static_volume_handle | default('', true) }}" - csi_driver: "{{ steps.ensure_cluster.csi.static_driver_name | default('', true) }}" + volume_handle: "{{ steps.setup_cluster.csi.static_volume_handle | default('', true) }}" + csi_driver: "{{ steps.setup_cluster.csi.static_driver_name | default('', true) }}" # Zonal block volumes (EBS/PD/Disk) must pin their consumer pod to # the volume's AZ; empty for zone-agnostic backends. - zone: "{{ steps.ensure_cluster.csi.static_volume_az | default('', true) }}" + zone: "{{ steps.setup_cluster.csi.static_volume_az | default('', true) }}" fs_type: "ext4" capacity: "1Gi" access_mode: "ReadWriteOnce" @@ -327,20 +327,20 @@ tests: # node: { daemonset: } drivers: - storage_classes: - - "{{ steps.ensure_cluster.csi.block_storage_class | default('', true) }}" + - "{{ steps.setup_cluster.csi.block_storage_class | default('', true) }}" workloads: {} - storage_classes: - - "{{ steps.ensure_cluster.csi.shared_fs_storage_class | default('', true) }}" + - "{{ steps.setup_cluster.csi.shared_fs_storage_class | default('', true) }}" workloads: {} - storage_classes: - - "{{ steps.ensure_cluster.csi.nfs_storage_class | default('', true) }}" + - "{{ steps.setup_cluster.csi.nfs_storage_class | default('', true) }}" workloads: {} min_controller_replicas: 1 K8sCsiConcurrentPvcCheck: test_id: "N/A" labels: ["kubernetes", "min_req", "storage"] requires: [kubernetes] - storage_class: "{{ steps.ensure_cluster.csi.block_storage_class | default('', true) }}" + storage_class: "{{ steps.setup_cluster.csi.block_storage_class | default('', true) }}" pvc_count: 2 pvc_size: "1Gi" bind_timeout_s: 120 @@ -349,7 +349,7 @@ tests: test_id: "N/A" labels: ["kubernetes", "min_req", "storage"] requires: [kubernetes] - storage_class: "{{ steps.ensure_cluster.csi.block_storage_class | default('', true) }}" + storage_class: "{{ steps.setup_cluster.csi.block_storage_class | default('', true) }}" initial_size: "1Gi" expanded_size: "2Gi" bind_timeout_s: 120 @@ -369,8 +369,8 @@ tests: labels: ["kubernetes", "storage"] requires: [kubernetes] # Env overrides: K8S_CSI_SHARED_FS_SC, K8S_CSI_NFS_SC. - shared_fs_storage_class: "{{ steps.ensure_cluster.csi.shared_fs_storage_class | default('', true) }}" - nfs_storage_class: "{{ steps.ensure_cluster.csi.nfs_storage_class | default('', true) }}" + shared_fs_storage_class: "{{ steps.setup_cluster.csi.shared_fs_storage_class | default('', true) }}" + nfs_storage_class: "{{ steps.setup_cluster.csi.nfs_storage_class | default('', true) }}" image: "busybox:1.36" node_selector: {} pvc_size: "1Gi" @@ -383,15 +383,15 @@ tests: expected_read_ahead_kb: "" k8s_filesystem: - step: ensure_cluster + step: setup_cluster checks: # Shared-filesystem POSIX-semantics checks (multi-pod, RWX PVC). K8sFileLockingCheck: test_id: "HSS14-02" labels: ["kubernetes", "storage"] requires: [kubernetes] - shared_fs_storage_class: "{{ steps.ensure_cluster.csi.shared_fs_storage_class | default('', true) }}" - nfs_storage_class: "{{ steps.ensure_cluster.csi.nfs_storage_class | default('', true) }}" + shared_fs_storage_class: "{{ steps.setup_cluster.csi.shared_fs_storage_class | default('', true) }}" + nfs_storage_class: "{{ steps.setup_cluster.csi.nfs_storage_class | default('', true) }}" image: "busybox:1.36" node_selector: {} pvc_size: "1Gi" @@ -402,8 +402,8 @@ tests: test_id: "HSS17-01" labels: ["kubernetes", "storage"] requires: [kubernetes] - shared_fs_storage_class: "{{ steps.ensure_cluster.csi.shared_fs_storage_class | default('', true) }}" - nfs_storage_class: "{{ steps.ensure_cluster.csi.nfs_storage_class | default('', true) }}" + shared_fs_storage_class: "{{ steps.setup_cluster.csi.shared_fs_storage_class | default('', true) }}" + nfs_storage_class: "{{ steps.setup_cluster.csi.nfs_storage_class | default('', true) }}" image: "busybox:1.36" node_selector: {} pvc_size: "1Gi" @@ -414,8 +414,8 @@ tests: test_id: "HSS17-02" labels: ["kubernetes", "storage"] requires: [kubernetes] - shared_fs_storage_class: "{{ steps.ensure_cluster.csi.shared_fs_storage_class | default('', true) }}" - nfs_storage_class: "{{ steps.ensure_cluster.csi.nfs_storage_class | default('', true) }}" + shared_fs_storage_class: "{{ steps.setup_cluster.csi.shared_fs_storage_class | default('', true) }}" + nfs_storage_class: "{{ steps.setup_cluster.csi.nfs_storage_class | default('', true) }}" image: "busybox:1.36" node_selector: {} pvc_size: "1Gi" @@ -427,8 +427,8 @@ tests: test_id: "HSS07-02" labels: ["kubernetes", "storage"] requires: [kubernetes] - shared_fs_storage_class: "{{ steps.ensure_cluster.csi.shared_fs_storage_class | default('', true) }}" - nfs_storage_class: "{{ steps.ensure_cluster.csi.nfs_storage_class | default('', true) }}" + shared_fs_storage_class: "{{ steps.setup_cluster.csi.shared_fs_storage_class | default('', true) }}" + nfs_storage_class: "{{ steps.setup_cluster.csi.nfs_storage_class | default('', true) }}" image: "busybox:1.36" node_selector: {} files_count: 10000 @@ -440,8 +440,8 @@ tests: test_id: "HSS07-02" labels: ["kubernetes", "storage"] requires: [kubernetes] - shared_fs_storage_class: "{{ steps.ensure_cluster.csi.shared_fs_storage_class | default('', true) }}" - nfs_storage_class: "{{ steps.ensure_cluster.csi.nfs_storage_class | default('', true) }}" + shared_fs_storage_class: "{{ steps.setup_cluster.csi.shared_fs_storage_class | default('', true) }}" + nfs_storage_class: "{{ steps.setup_cluster.csi.nfs_storage_class | default('', true) }}" image: "busybox:1.36" node_selector: {} dirs_count: 500 @@ -453,8 +453,8 @@ tests: test_id: "DIR02-02" labels: ["kubernetes", "slow", "storage"] requires: [kubernetes] - shared_fs_storage_class: "{{ steps.ensure_cluster.csi.shared_fs_storage_class | default('', true) }}" - nfs_storage_class: "{{ steps.ensure_cluster.csi.nfs_storage_class | default('', true) }}" + shared_fs_storage_class: "{{ steps.setup_cluster.csi.shared_fs_storage_class | default('', true) }}" + nfs_storage_class: "{{ steps.setup_cluster.csi.nfs_storage_class | default('', true) }}" image: "gcc:12" node_selector: {} pvc_size: "5Gi" From 6013c68e106824ef31cbfc6fc7f6dd3e7d43202e Mon Sep 17 00:00:00 2001 From: Alexandre Begnoche Date: Wed, 22 Jul 2026 16:07:13 -0400 Subject: [PATCH 14/34] refactor(suites): defer unique wiring names Signed-off-by: Alexandre Begnoche --- .../providers/aws/config/bare_metal.yaml | 16 ++-- .../providers/nico/config/control-plane.yaml | 2 +- isvctl/configs/providers/nico/config/iam.yaml | 4 +- .../providers/nico/config/network.yaml | 4 +- isvctl/configs/suites/bare_metal.yaml | 90 +++++++++---------- isvctl/configs/suites/control-plane.yaml | 10 +-- isvctl/configs/suites/iam.yaml | 8 +- isvctl/configs/suites/image-registry.yaml | 38 ++++---- isvctl/configs/suites/k8s.yaml | 8 +- isvctl/configs/suites/network.yaml | 2 +- isvctl/configs/suites/security.yaml | 2 +- isvctl/configs/suites/storage.yaml | 28 +++--- isvctl/configs/suites/vm.yaml | 64 ++++++------- .../providers/nico/test_nico_provider.py | 2 +- isvctl/tests/test_merger.py | 2 +- isvtest/src/isvtest/catalog.py | 18 ++-- isvtest/tests/test_catalog.py | 6 +- scripts/validate_suite_wiring.py | 8 +- 18 files changed, 161 insertions(+), 151 deletions(-) diff --git a/isvctl/configs/providers/aws/config/bare_metal.yaml b/isvctl/configs/providers/aws/config/bare_metal.yaml index c38a4aaea..6fe86a349 100644 --- a/isvctl/configs/providers/aws/config/bare_metal.yaml +++ b/isvctl/configs/providers/aws/config/bare_metal.yaml @@ -322,7 +322,7 @@ tests: serial_console: step: serial_console checks: - - SerialConsoleCheck-bm_serial_console: {} + - SerialConsoleCheck: {} # GPU stress - AWS metal has 8 GPUs gpu_stress: @@ -352,14 +352,14 @@ tests: start_gpu: step: start_instance checks: - GpuCheck-bm_start_gpu: + GpuCheck: expected_gpus: 8 # Power-cycle GPU - AWS metal has 8 GPUs power_cycle_gpu: step: power_cycle_instance checks: - GpuCheck-bm_power_cycle_gpu: + GpuCheck: expected_gpus: 8 # NVLink - AWS metal has 8 GPUs @@ -373,16 +373,16 @@ tests: image_installed: step: verify_image checks: - StepSuccessCheck-bm_image_installed: {} - FieldExistsCheck-bm_image_installed: + StepSuccessCheck: {} + FieldExistsCheck: fields: ["instance_id", "image_id", "image_name", "instance_state"] - InstanceStateCheck-bm_image_installed: + InstanceStateCheck: expected_state: "running" # AWS-specific: verify install config can provision BM config_installable: step: verify_config checks: - StepSuccessCheck-bm_config_installable: {} - FieldExistsCheck-bm_config_installable: + StepSuccessCheck: {} + FieldExistsCheck: fields: ["config_id", "config_name", "dry_run_passed"] diff --git a/isvctl/configs/providers/nico/config/control-plane.yaml b/isvctl/configs/providers/nico/config/control-plane.yaml index b70369c2f..5300ea95a 100644 --- a/isvctl/configs/providers/nico/config/control-plane.yaml +++ b/isvctl/configs/providers/nico/config/control-plane.yaml @@ -57,7 +57,7 @@ tests: api_health: step: check_api checks: - FieldExistsCheck-cp_api_health: + FieldExistsCheck: test_id: "N/A" fields: ["account_id", "tests"] FieldValueCheck: diff --git a/isvctl/configs/providers/nico/config/iam.yaml b/isvctl/configs/providers/nico/config/iam.yaml index 18afdc621..4711568ee 100644 --- a/isvctl/configs/providers/nico/config/iam.yaml +++ b/isvctl/configs/providers/nico/config/iam.yaml @@ -56,10 +56,10 @@ tests: credential_readiness: step: check_credentials checks: - FieldExistsCheck-iam_credential_readiness: + FieldExistsCheck: test_id: "N/A" fields: ["account_id", "authenticated", "tests"] - StepSuccessCheck-iam_credential_readiness: + StepSuccessCheck: test_id: "N/A" exclude: diff --git a/isvctl/configs/providers/nico/config/network.yaml b/isvctl/configs/providers/nico/config/network.yaml index 658cc26eb..80a345ad4 100644 --- a/isvctl/configs/providers/nico/config/network.yaml +++ b/isvctl/configs/providers/nico/config/network.yaml @@ -118,7 +118,7 @@ tests: network_connectivity: step: network_connectivity checks: - StepSuccessCheck-network_connectivity: + StepSuccessCheck: test_id: "N/A" FieldValueCheck: test_id: "N/A" @@ -128,7 +128,7 @@ tests: traffic_validation: step: traffic_validation checks: - StepSuccessCheck-network_traffic_validation: + StepSuccessCheck: test_id: "N/A" FieldValueCheck: test_id: "N/A" diff --git a/isvctl/configs/suites/bare_metal.yaml b/isvctl/configs/suites/bare_metal.yaml index 553176271..35276c435 100644 --- a/isvctl/configs/suites/bare_metal.yaml +++ b/isvctl/configs/suites/bare_metal.yaml @@ -51,7 +51,7 @@ tests: setup_checks: step: launch_instance checks: - InstanceStateCheck-bm_setup: + InstanceStateCheck: test_id: "N/A" labels: ["bare_metal"] expected_state: "running" @@ -59,7 +59,7 @@ tests: list_instances: step: list_instances checks: - InstanceListCheck-bm_list_instances: + InstanceListCheck: # CNP01-11 (list nodes) is a VMaaS plan item; the BMaaS plan has no equivalent list-nodes item. test_id: "N/A" labels: ["bare_metal"] @@ -68,7 +68,7 @@ tests: tag_checks: step: verify_tags checks: - InstanceTagCheck-bm_tag: + InstanceTagCheck: test_id: "CNP05-01" labels: ["bare_metal", "min_req"] required_keys: ["Name", "CreatedBy"] @@ -76,7 +76,7 @@ tests: topology_placement: step: topology_placement checks: - StepSuccessCheck-bm_topology_placement: + StepSuccessCheck: test_id: "N/A" labels: ["bare_metal"] TopologyPlacementCheck: @@ -86,7 +86,7 @@ tests: serial_console: step: serial_console checks: - SerialConsoleCheck-bm_serial_console: + SerialConsoleCheck: test_id: "CNP06-01" labels: ["bare_metal", "min_req"] SerialConsoleRetentionCheck: @@ -96,14 +96,14 @@ tests: cloud_init: step: launch_instance checks: - CloudInitCheck-bm_cloud_init: + CloudInitCheck: test_id: "BOOT02-01" labels: ["bare_metal", "min_req", "ssh"] instance_info: step: describe_instance checks: - InstanceStateCheck-bm_instance_info: + InstanceStateCheck: test_id: "N/A" labels: ["bare_metal"] expected_state: "running" @@ -111,10 +111,10 @@ tests: ssh: step: describe_instance checks: - ConnectivityCheck-bm_ssh: + ConnectivityCheck: test_id: "BMAAS03-01" labels: ["bare_metal", "ssh"] - OsCheck-bm_ssh: + OsCheck: # No BMaaS plan item for a host-OS image check (VMAAS-XX-05 is VM-only). test_id: "N/A" labels: ["bare_metal", "ssh"] @@ -123,7 +123,7 @@ tests: gpu: step: describe_instance checks: - GpuCheck-bm_gpu: + GpuCheck: test_id: "BMAAS08-01" labels: ["bare_metal", "ssh", "gpu"] expected_gpus: 8 @@ -133,14 +133,14 @@ tests: checks: # vCPU pinning / PCI passthrough / host-software are VMAAS plan items; # the BMaaS plan has no equivalent, so these are intentional gaps here. - VcpuPinningCheck-bm_host_os: + VcpuPinningCheck: test_id: "N/A" labels: ["bare_metal", "ssh"] - PciBusCheck-bm_host_os: + PciBusCheck: test_id: "N/A" labels: ["bare_metal", "ssh", "gpu"] expected_gpus: 8 - HostSoftwareCheck-bm_host_os: + HostSoftwareCheck: test_id: "N/A" labels: ["bare_metal", "ssh"] @@ -150,7 +150,7 @@ tests: host_status_logs: step: host_status_log checks: - StepSuccessCheck-bm_host_status_logs: + StepSuccessCheck: test_id: "N/A" labels: ["bare_metal"] BmHostStatusLog: @@ -211,38 +211,38 @@ tests: step: describe_instance checks: # No BMaaS plan item for a host driver/kernel check (VMAAS-XX-07 is VM-only). - DriverCheck-bm_driver_info: + DriverCheck: test_id: "N/A" labels: ["bare_metal", "ssh", "gpu"] cpu_info: step: describe_instance checks: - CpuInfoCheck-bm_cpu_info: + CpuInfoCheck: test_id: "N/A" labels: ["bare_metal", "ssh"] container_runtime: step: describe_instance checks: - ContainerRuntimeCheck-bm_container_runtime: + ContainerRuntimeCheck: test_id: "N/A" labels: ["bare_metal", "ssh", "gpu", "workload", "slow"] stop_checks: step: stop_instance checks: - InstanceStopCheck-bm_stop: + InstanceStopCheck: test_id: "CNP01-07" labels: ["bare_metal", "min_req"] start_checks: step: start_instance checks: - InstanceStartCheck-bm_start: + InstanceStartCheck: test_id: "CNP01-08" labels: ["bare_metal", "min_req"] - StableIdentifierCheck-bm_start: + StableIdentifierCheck: test_id: "CNP08-01" labels: ["bare_metal", "min_req"] reference_id: "{{steps.launch_instance.instance_id}}" @@ -255,12 +255,12 @@ tests: step: start_instance checks: # ConnectivityCheck -> BMAAS-XX-03, OsCheck -> N/A (declared in the "ssh" section) - ConnectivityCheck-bm_start_ssh: + ConnectivityCheck: test_id: "BMAAS03-01" labels: ["bare_metal", "ssh"] host: "{{steps.describe_instance.public_ip}}" key_file: "{{steps.describe_instance.key_file}}" - OsCheck-bm_start_ssh: + OsCheck: test_id: "N/A" labels: ["bare_metal", "ssh"] host: "{{steps.describe_instance.public_ip}}" @@ -271,7 +271,7 @@ tests: step: start_instance checks: # GpuCheck -> BMAAS-XX-08 (declared in the "gpu" section) - GpuCheck-bm_start_gpu: + GpuCheck: test_id: "BMAAS08-01" labels: ["bare_metal", "ssh", "gpu"] host: "{{steps.describe_instance.public_ip}}" @@ -280,12 +280,12 @@ tests: reboot_checks: step: reboot_instance checks: - InstanceRebootCheck-bm_reboot: + InstanceRebootCheck: test_id: "CNP01-05" labels: ["bare_metal", "min_req"] max_uptime: 600 # StableIdentifierCheck -> CNP08-01 (declared in the "start_checks" section) - StableIdentifierCheck-bm_reboot: + StableIdentifierCheck: test_id: "CNP08-01" labels: ["bare_metal", "min_req"] reference_id: "{{steps.launch_instance.instance_id}}" @@ -293,7 +293,7 @@ tests: reboot_state: step: reboot_instance checks: - InstanceStateCheck-bm_reboot_state: + InstanceStateCheck: test_id: "N/A" labels: ["bare_metal"] expected_state: "running" @@ -302,12 +302,12 @@ tests: step: reboot_instance checks: # ConnectivityCheck -> BMAAS-XX-03, OsCheck -> N/A (declared in the "ssh" section) - ConnectivityCheck-bm_reboot_ssh: + ConnectivityCheck: test_id: "BMAAS03-01" labels: ["bare_metal", "ssh"] host: "{{steps.describe_instance.public_ip}}" key_file: "{{steps.describe_instance.key_file}}" - OsCheck-bm_reboot_ssh: + OsCheck: test_id: "N/A" labels: ["bare_metal", "ssh"] host: "{{steps.describe_instance.public_ip}}" @@ -318,7 +318,7 @@ tests: step: reboot_instance checks: # GpuCheck -> BMAAS-XX-08 (declared in the "gpu" section) - GpuCheck-bm_reboot_gpu: + GpuCheck: test_id: "BMAAS08-01" labels: ["bare_metal", "ssh", "gpu"] host: "{{steps.describe_instance.public_ip}}" @@ -333,7 +333,7 @@ tests: labels: ["bare_metal", "min_req"] max_recovery_time: 900 # StableIdentifierCheck -> CNP08-01 (declared in the "start_checks" section) - StableIdentifierCheck-bm_power_cycle: + StableIdentifierCheck: test_id: "CNP08-01" labels: ["bare_metal", "min_req"] reference_id: "{{steps.launch_instance.instance_id}}" @@ -341,7 +341,7 @@ tests: power_cycle_state: step: power_cycle_instance checks: - InstanceStateCheck-bm_power_cycle_state: + InstanceStateCheck: test_id: "N/A" labels: ["bare_metal"] expected_state: "running" @@ -350,10 +350,10 @@ tests: step: power_cycle_instance checks: # ConnectivityCheck -> BMAAS-XX-03, OsCheck -> N/A (declared in the "ssh" section) - ConnectivityCheck-bm_power_cycle_ssh: + ConnectivityCheck: test_id: "BMAAS03-01" labels: ["bare_metal", "ssh"] - OsCheck-bm_power_cycle_ssh: + OsCheck: test_id: "N/A" labels: ["bare_metal", "ssh"] expected_os: "ubuntu" @@ -362,7 +362,7 @@ tests: step: power_cycle_instance checks: # GpuCheck -> BMAAS-XX-08 (declared in the "gpu" section) - GpuCheck-bm_power_cycle_gpu: + GpuCheck: test_id: "BMAAS08-01" labels: ["bare_metal", "ssh", "gpu"] expected_gpus: 8 @@ -371,11 +371,11 @@ tests: reinstall_state: step: reinstall_instance checks: - InstanceStateCheck-bm_reinstall_state: + InstanceStateCheck: test_id: "N/A" labels: ["bare_metal"] expected_state: "running" - StableIdentifierCheck-bm_reinstall_state: + StableIdentifierCheck: test_id: "CNP08-01" labels: ["bare_metal", "min_req"] reference_id: "{{steps.launch_instance.instance_id}}" @@ -384,10 +384,10 @@ tests: step: reinstall_instance checks: # ConnectivityCheck -> BMAAS-XX-03, OsCheck -> N/A (declared in the "ssh" section) - ConnectivityCheck-bm_reinstall_ssh: + ConnectivityCheck: test_id: "BMAAS03-01" labels: ["bare_metal", "ssh"] - OsCheck-bm_reinstall_ssh: + OsCheck: test_id: "N/A" labels: ["bare_metal", "ssh"] expected_os: "ubuntu" @@ -396,28 +396,28 @@ tests: step: reinstall_instance checks: # GpuCheck -> BMAAS-XX-08 (declared in the "gpu" section) - GpuCheck-bm_reinstall_gpu: + GpuCheck: test_id: "BMAAS08-01" labels: ["bare_metal", "ssh", "gpu"] nim_health: step: deploy_nim checks: - NimHealthCheck-bm_nim_health: + NimHealthCheck: test_id: "BMAAS10-01" labels: ["bare_metal", "gpu", "min_req", "slow", "ssh", "workload"] nim_models: step: deploy_nim checks: - NimModelCheck-bm_nim_models: + NimModelCheck: test_id: "BMAAS10-01" labels: ["bare_metal", "gpu", "min_req", "slow", "ssh", "workload"] nim_inference: step: deploy_nim checks: - NimInferenceCheck-bm_nim_inference: + NimInferenceCheck: test_id: "BMAAS10-01" labels: ["bare_metal", "gpu", "min_req", "slow", "ssh", "workload"] prompt: "What is CUDA?" @@ -426,21 +426,21 @@ tests: nim_teardown: step: teardown_nim checks: - StepSuccessCheck-bm_nim_teardown: + StepSuccessCheck: test_id: "N/A" labels: ["bare_metal"] teardown_checks: step: teardown checks: - StepSuccessCheck-bm_teardown: + StepSuccessCheck: test_id: "N/A" labels: ["bare_metal"] sanitization: step: verify_teardown checks: - StepSuccessCheck-bm_sanitization: + StepSuccessCheck: test_id: "SEC21-01" labels: ["bare_metal", "min_req", "security"] diff --git a/isvctl/configs/suites/control-plane.yaml b/isvctl/configs/suites/control-plane.yaml index a205e8be2..bf8e15e53 100644 --- a/isvctl/configs/suites/control-plane.yaml +++ b/isvctl/configs/suites/control-plane.yaml @@ -56,7 +56,7 @@ tests: api_health: step: check_api checks: - FieldExistsCheck-cp_api_health: + FieldExistsCheck: test_id: "N/A" labels: ["control_plane"] requires: [] @@ -65,7 +65,7 @@ tests: # authenticated-endpoint half - a successful authenticated check_api # response. The data-path half is the CrudOperationsCheck in the # "s3_object_lifecycle" section, which declares the same test_id. - FieldValueCheck-cp_api_health: + FieldValueCheck: test_id: "DATASVC01-01" labels: ["control_plane", "min_req"] requires: [] @@ -121,16 +121,16 @@ tests: checks: # Data-path half of DATASVC-XX-01 (object put/get/delete); the # authenticated-endpoint half is the FieldValueCheck in "api_health". - StepSuccessCheck-cp_s3_object_lifecycle: + StepSuccessCheck: test_id: "N/A" labels: ["control_plane"] requires: [] - FieldExistsCheck-cp_s3_object_lifecycle: + FieldExistsCheck: test_id: "N/A" labels: ["control_plane"] requires: [] fields: ["bucket_name", "object_key", "operations"] - CrudOperationsCheck-cp_s3_object_lifecycle: + CrudOperationsCheck: test_id: "DATASVC01-01" labels: ["control_plane", "min_req"] requires: [] diff --git a/isvctl/configs/suites/iam.yaml b/isvctl/configs/suites/iam.yaml index 981f52b95..bd8442fd5 100644 --- a/isvctl/configs/suites/iam.yaml +++ b/isvctl/configs/suites/iam.yaml @@ -50,12 +50,12 @@ tests: setup_checks: step: create_user checks: - FieldExistsCheck-iam_setup: + FieldExistsCheck: test_id: "IAM01-01" labels: ["iam", "min_req"] requires: [] fields: ["username", "access_key_id"] - StepSuccessCheck-iam_setup: + StepSuccessCheck: test_id: "N/A" labels: ["iam"] requires: [] @@ -67,7 +67,7 @@ tests: test_id: "IAM03-01" labels: ["iam"] requires: [] - StepSuccessCheck-iam_credentials: + StepSuccessCheck: test_id: "N/A" labels: ["iam"] requires: [] @@ -75,7 +75,7 @@ tests: teardown_checks: step: teardown checks: - StepSuccessCheck-iam_teardown: + StepSuccessCheck: test_id: "N/A" labels: ["iam"] requires: [] diff --git a/isvctl/configs/suites/image-registry.yaml b/isvctl/configs/suites/image-registry.yaml index 9d20c8264..e2c6d263c 100644 --- a/isvctl/configs/suites/image-registry.yaml +++ b/isvctl/configs/suites/image-registry.yaml @@ -53,11 +53,11 @@ tests: image_upload: step: upload_image checks: - StepSuccessCheck-img_image_upload: + StepSuccessCheck: test_id: "N/A" labels: ["image_registry"] requires: [] - FieldExistsCheck-img_image_upload: + FieldExistsCheck: test_id: "BOOT01-01" labels: ["image_registry", "min_req"] requires: [] @@ -67,16 +67,16 @@ tests: image_crud: step: crud_image checks: - StepSuccessCheck-img_image_crud: + StepSuccessCheck: test_id: "N/A" labels: ["image_registry"] requires: [] - FieldExistsCheck-img_image_crud: + FieldExistsCheck: test_id: "N/A" labels: ["image_registry"] requires: [] fields: ["image_id", "operations"] - CrudOperationsCheck-img_image_crud: + CrudOperationsCheck: test_id: "BOOT03-02" labels: ["image_registry", "min_req"] requires: [] @@ -86,16 +86,16 @@ tests: vm_from_image: step: launch_instance checks: - StepSuccessCheck-img_vm_from_image: + StepSuccessCheck: test_id: "BOOT01-05" labels: ["image_registry", "min_req"] requires: [vm] - FieldExistsCheck-img_vm_from_image: + FieldExistsCheck: test_id: "N/A" labels: ["image_registry"] requires: [vm] fields: ["instance_id", "public_ip", "key_path"] - InstanceStateCheck-img_vm_from_image: + InstanceStateCheck: test_id: "N/A" labels: ["image_registry"] requires: [vm] @@ -104,11 +104,11 @@ tests: vm_ssh: step: launch_instance checks: - ConnectivityCheck-img_vm_ssh: + ConnectivityCheck: test_id: "N/A" labels: ["image_registry", "ssh"] requires: [vm] - OsCheck-img_vm_ssh: + OsCheck: test_id: "N/A" labels: ["image_registry", "ssh"] requires: [vm] @@ -118,11 +118,11 @@ tests: install_config_crud: step: crud_install_config checks: - StepSuccessCheck-img_install_config_crud: + StepSuccessCheck: test_id: "N/A" labels: ["image_registry"] requires: [] - FieldExistsCheck-img_install_config_crud: + FieldExistsCheck: test_id: "BOOT01-02" labels: ["image_registry"] requires: [] @@ -132,16 +132,16 @@ tests: bm_from_image: step: install_image_bm checks: - StepSuccessCheck-img_bm_from_image: + StepSuccessCheck: test_id: "BOOT01-03" labels: ["image_registry", "min_req"] requires: [bare_metal] - FieldExistsCheck-img_bm_from_image: + FieldExistsCheck: test_id: "N/A" labels: ["image_registry"] requires: [bare_metal] fields: ["instance_id", "image_id", "instance_state"] - InstanceStateCheck-img_bm_from_image: + InstanceStateCheck: test_id: "N/A" labels: ["image_registry"] requires: [bare_metal] @@ -151,16 +151,16 @@ tests: bm_from_config: step: install_config_bm checks: - StepSuccessCheck-img_bm_from_config: + StepSuccessCheck: test_id: "BOOT01-04" labels: ["image_registry", "min_req"] requires: [bare_metal] - FieldExistsCheck-img_bm_from_config: + FieldExistsCheck: test_id: "N/A" labels: ["image_registry"] requires: [bare_metal] fields: ["instance_id", "config_id", "instance_state"] - InstanceStateCheck-img_bm_from_config: + InstanceStateCheck: test_id: "N/A" labels: ["image_registry"] requires: [bare_metal] @@ -169,7 +169,7 @@ tests: teardown_checks: step: teardown checks: - StepSuccessCheck-img_teardown: + StepSuccessCheck: test_id: "N/A" labels: ["image_registry"] requires: [] diff --git a/isvctl/configs/suites/k8s.yaml b/isvctl/configs/suites/k8s.yaml index 505e3cb09..489456533 100644 --- a/isvctl/configs/suites/k8s.yaml +++ b/isvctl/configs/suites/k8s.yaml @@ -75,7 +75,7 @@ tests: # The separate CPU and GPU create checks prove distinct pools, each with # its own instance type and node count, coexist in the same cluster. k8s_node_pools: - - K8sNodePoolCheck-create_cpu_pool: + - K8sNodePoolCheck: # Create (CPU pool): pool reaches desired replica count with its # specified CPU instance type. test_id: "K8S06-01" @@ -89,7 +89,7 @@ tests: node_type: "{{ steps.create_test_node_pool.node_type | default('cpu') }}" wait_timeout: 900 poll_interval: 10 - - K8sNodePoolCheck-create_gpu_pool: + - K8sNodePoolCheck: # Create (GPU pool): a separate pool reaches its own node count with # its specified GPU instance type, coexisting with the CPU pool. test_id: "K8S06-01" @@ -103,7 +103,7 @@ tests: node_type: "{{ steps.create_test_gpu_node_pool.node_type | default('gpu') }}" wait_timeout: 900 poll_interval: 10 - - K8sNodePoolCheck-update_pool: + - K8sNodePoolCheck: # Update (scale): same pool converges to the new count. test_id: "K8S06-02" labels: ["kubernetes", "min_req", "slow"] @@ -116,7 +116,7 @@ tests: node_type: "{{ steps.update_test_node_pool.node_type | default('cpu') }}" wait_timeout: 900 poll_interval: 10 - - K8sNodePoolCheck-delete_pool: + - K8sNodePoolCheck: # Delete: assert the throwaway pool reaches zero nodes after the # delete step. Selector comes from its create step; the label/taint/ # instance-type assertions are omitted (no nodes to check). diff --git a/isvctl/configs/suites/network.yaml b/isvctl/configs/suites/network.yaml index 0d32fd03c..e9b19f11d 100644 --- a/isvctl/configs/suites/network.yaml +++ b/isvctl/configs/suites/network.yaml @@ -262,7 +262,7 @@ tests: teardown_checks: step: teardown checks: - StepSuccessCheck-network_teardown: + StepSuccessCheck: test_id: "N/A" labels: ["network"] requires: [] diff --git a/isvctl/configs/suites/security.yaml b/isvctl/configs/suites/security.yaml index 94bb565c1..fdfc1cff5 100644 --- a/isvctl/configs/suites/security.yaml +++ b/isvctl/configs/suites/security.yaml @@ -234,7 +234,7 @@ tests: teardown_checks: step: teardown checks: - StepSuccessCheck-security_teardown: + StepSuccessCheck: test_id: "N/A" labels: ["security"] requires: [] diff --git a/isvctl/configs/suites/storage.yaml b/isvctl/configs/suites/storage.yaml index 2852be80a..e17897ac6 100644 --- a/isvctl/configs/suites/storage.yaml +++ b/isvctl/configs/suites/storage.yaml @@ -70,7 +70,7 @@ tests: instance_launched: step: launch_instance checks: - InstanceStateCheck-storage_instance_launched: + InstanceStateCheck: test_id: "N/A" labels: ["storage"] requires: [vm, bare_metal] @@ -79,16 +79,16 @@ tests: fixture_volume: step: create_volume checks: - StepSuccessCheck-storage_fixture_volume: + StepSuccessCheck: test_id: "N/A" labels: ["storage"] requires: [vm, bare_metal] - FieldExistsCheck-storage_fixture_volume: + FieldExistsCheck: test_id: "N/A" labels: ["storage"] requires: [vm, bare_metal] fields: ["volume_id", "mount_point", "operations"] - CrudOperationsCheck-storage_fixture_volume: + CrudOperationsCheck: test_id: "N/A" labels: ["storage"] requires: [vm, bare_metal] @@ -98,16 +98,16 @@ tests: snapshot_lifecycle: step: snapshot_lifecycle checks: - StepSuccessCheck-storage_snapshot_lifecycle: + StepSuccessCheck: test_id: "N/A" labels: ["storage"] requires: [vm, bare_metal] - FieldExistsCheck-storage_snapshot_lifecycle: + FieldExistsCheck: test_id: "N/A" labels: ["storage"] requires: [vm, bare_metal] fields: ["volume_id", "snapshot_id", "operations"] - CrudOperationsCheck-storage_snapshot_lifecycle: + CrudOperationsCheck: test_id: "DATASVC02-01" labels: ["min_req", "storage"] requires: [vm, bare_metal] @@ -117,16 +117,16 @@ tests: volume_resize: step: volume_resize checks: - StepSuccessCheck-storage_volume_resize: + StepSuccessCheck: test_id: "N/A" labels: ["storage"] requires: [vm, bare_metal] - FieldExistsCheck-storage_volume_resize: + FieldExistsCheck: test_id: "N/A" labels: ["storage"] requires: [vm, bare_metal] fields: ["volume_id", "operations"] - CrudOperationsCheck-storage_volume_resize: + CrudOperationsCheck: test_id: "DATASVC03-01" labels: ["min_req", "storage"] requires: [vm, bare_metal] @@ -136,16 +136,16 @@ tests: volume_persistence: step: volume_persistence checks: - StepSuccessCheck-storage_volume_persistence: + StepSuccessCheck: test_id: "N/A" labels: ["storage"] requires: [vm, bare_metal] - FieldExistsCheck-storage_volume_persistence: + FieldExistsCheck: test_id: "N/A" labels: ["storage"] requires: [vm, bare_metal] fields: ["volume_id", "operations"] - CrudOperationsCheck-storage_volume_persistence: + CrudOperationsCheck: test_id: "DATASVC04-01" labels: ["min_req", "storage"] requires: [vm, bare_metal] @@ -240,7 +240,7 @@ tests: teardown_checks: step: teardown_volume checks: - StepSuccessCheck-storage_teardown: + StepSuccessCheck: test_id: "N/A" labels: ["storage"] requires: [] diff --git a/isvctl/configs/suites/vm.yaml b/isvctl/configs/suites/vm.yaml index 5b96ebee6..b7ee93dc4 100644 --- a/isvctl/configs/suites/vm.yaml +++ b/isvctl/configs/suites/vm.yaml @@ -58,7 +58,7 @@ tests: setup_checks: step: launch_instance checks: - InstanceStateCheck-vm_setup: + InstanceStateCheck: test_id: "N/A" labels: ["vm"] expected_state: "running" @@ -87,7 +87,7 @@ tests: list_instances: step: list_instances checks: - InstanceListCheck-vm_list_instances: + InstanceListCheck: test_id: "CNP01-11" labels: ["min_req", "vm"] min_count: 1 @@ -95,7 +95,7 @@ tests: tag_checks: step: verify_tags checks: - InstanceTagCheck-vm_tag: + InstanceTagCheck: test_id: "CNP05-02" labels: ["min_req", "vm"] required_keys: ["Name", "CreatedBy"] @@ -103,7 +103,7 @@ tests: serial_console: step: serial_console checks: - SerialConsoleCheck-vm_serial_console: + SerialConsoleCheck: test_id: "CNP06-03" labels: ["min_req", "vm"] @@ -124,7 +124,7 @@ tests: cloud_init: step: launch_instance checks: - CloudInitCheck-vm_cloud_init: + CloudInitCheck: test_id: "BOOT02-02" labels: ["min_req", "ssh", "vm"] @@ -134,7 +134,7 @@ tests: instance_info: step: describe_instance checks: - InstanceStateCheck-vm_instance_info: + InstanceStateCheck: test_id: "N/A" labels: ["vm"] expected_state: "running" @@ -142,10 +142,10 @@ tests: ssh: step: describe_instance checks: - ConnectivityCheck-vm_ssh: + ConnectivityCheck: test_id: "VMAAS01-01" labels: ["vm", "ssh"] - OsCheck-vm_ssh: + OsCheck: test_id: "VMAAS05-01" labels: ["vm", "ssh"] expected_os: "ubuntu" @@ -153,7 +153,7 @@ tests: gpu: step: describe_instance checks: - GpuCheck-vm_gpu: + GpuCheck: test_id: "VMAAS06-01" labels: ["vm", "ssh", "gpu"] expected_gpus: 1 @@ -161,35 +161,35 @@ tests: host_os: step: describe_instance checks: - VcpuPinningCheck-vm_host_os: + VcpuPinningCheck: test_id: "VMAAS09-01" labels: ["gpu", "ssh", "vm"] - PciBusCheck-vm_host_os: + PciBusCheck: test_id: "VMAAS09-01" labels: ["vm", "ssh", "gpu"] expected_gpus: 1 - HostSoftwareCheck-vm_host_os: + HostSoftwareCheck: test_id: "VMAAS07-01" labels: ["gpu", "ssh", "vm"] driver_info: step: describe_instance checks: - DriverCheck-vm_driver_info: + DriverCheck: test_id: "VMAAS07-01" labels: ["vm", "ssh", "gpu"] cpu_info: step: describe_instance checks: - CpuInfoCheck-vm_cpu_info: + CpuInfoCheck: test_id: "N/A" labels: ["vm", "ssh"] container_runtime: step: describe_instance checks: - ContainerRuntimeCheck-vm_container_runtime: + ContainerRuntimeCheck: test_id: "N/A" labels: ["vm", "ssh", "gpu", "workload", "slow"] @@ -197,17 +197,17 @@ tests: stop_checks: step: stop_instance checks: - InstanceStopCheck-vm_stop: + InstanceStopCheck: test_id: "CNP01-14" labels: ["min_req", "vm"] start_checks: step: start_instance checks: - InstanceStartCheck-vm_start: + InstanceStartCheck: test_id: "CNP01-15" labels: ["min_req", "vm"] - StableIdentifierCheck-vm_start: + StableIdentifierCheck: test_id: "CNP08-03" labels: ["min_req", "vm"] reference_id: "{{steps.launch_instance.instance_id}}" @@ -216,10 +216,10 @@ tests: step: start_instance checks: # ConnectivityCheck -> VMAAS-XX-01, OsCheck -> VMAAS-XX-05 (declared in the "ssh" section) - ConnectivityCheck-vm_start_ssh: + ConnectivityCheck: test_id: "VMAAS01-01" labels: ["vm", "ssh"] - OsCheck-vm_start_ssh: + OsCheck: test_id: "VMAAS05-01" labels: ["vm", "ssh"] expected_os: "ubuntu" @@ -228,7 +228,7 @@ tests: step: start_instance checks: # GpuCheck -> VMAAS-XX-06 (declared in the "gpu" section) - GpuCheck-vm_start_gpu: + GpuCheck: test_id: "VMAAS06-01" labels: ["vm", "ssh", "gpu"] expected_gpus: 1 @@ -236,12 +236,12 @@ tests: reboot_checks: step: reboot_instance checks: - InstanceRebootCheck-vm_reboot: + InstanceRebootCheck: test_id: "CNP01-13" labels: ["vm"] max_uptime: 600 # StableIdentifierCheck -> CNP08-03 (declared in the "start_checks" section) - StableIdentifierCheck-vm_reboot: + StableIdentifierCheck: test_id: "CNP08-03" labels: ["min_req", "vm"] reference_id: "{{steps.launch_instance.instance_id}}" @@ -249,7 +249,7 @@ tests: reboot_state: step: reboot_instance checks: - InstanceStateCheck-vm_reboot_state: + InstanceStateCheck: test_id: "N/A" labels: ["vm"] expected_state: "running" @@ -258,10 +258,10 @@ tests: step: reboot_instance checks: # ConnectivityCheck -> VMAAS-XX-01, OsCheck -> VMAAS-XX-05 (declared in the "ssh" section) - ConnectivityCheck-vm_reboot_ssh: + ConnectivityCheck: test_id: "VMAAS01-01" labels: ["vm", "ssh"] - OsCheck-vm_reboot_ssh: + OsCheck: test_id: "VMAAS05-01" labels: ["vm", "ssh"] expected_os: "ubuntu" @@ -270,7 +270,7 @@ tests: step: reboot_instance checks: # GpuCheck -> VMAAS-XX-06 (declared in the "gpu" section) - GpuCheck-vm_reboot_gpu: + GpuCheck: test_id: "VMAAS06-01" labels: ["vm", "ssh", "gpu"] expected_gpus: 1 @@ -279,21 +279,21 @@ tests: nim_health: step: deploy_nim checks: - NimHealthCheck-vm_nim_health: + NimHealthCheck: test_id: "VMAAS12-01" labels: ["gpu", "slow", "ssh", "vm", "workload"] nim_models: step: deploy_nim checks: - NimModelCheck-vm_nim_models: + NimModelCheck: test_id: "VMAAS12-01" labels: ["gpu", "slow", "ssh", "vm", "workload"] nim_inference: step: deploy_nim checks: - NimInferenceCheck-vm_nim_inference: + NimInferenceCheck: test_id: "VMAAS12-01" labels: ["vm", "ssh", "gpu", "workload", "slow"] prompt: "What is CUDA?" @@ -303,14 +303,14 @@ tests: nim_teardown: step: teardown_nim checks: - StepSuccessCheck-vm_nim_teardown: + StepSuccessCheck: test_id: "N/A" labels: ["vm"] teardown_checks: step: teardown checks: - StepSuccessCheck-vm_teardown: + StepSuccessCheck: test_id: "N/A" labels: ["vm"] diff --git a/isvctl/tests/providers/nico/test_nico_provider.py b/isvctl/tests/providers/nico/test_nico_provider.py index 45c49234b..cf9870d6c 100644 --- a/isvctl/tests/providers/nico/test_nico_provider.py +++ b/isvctl/tests/providers/nico/test_nico_provider.py @@ -570,7 +570,7 @@ def test_nico_iam_config_wires_credential_readiness() -> None: validations = merged["tests"]["validations"] assert merged["tests"]["settings"]["nico_api_base"] == "{{env.NICO_API_BASE}}" assert validations["credential_readiness"]["step"] == "check_credentials" - assert validations["credential_readiness"]["checks"]["FieldExistsCheck-iam_credential_readiness"]["fields"] == [ + assert validations["credential_readiness"]["checks"]["FieldExistsCheck"]["fields"] == [ "account_id", "authenticated", "tests", diff --git a/isvctl/tests/test_merger.py b/isvctl/tests/test_merger.py index 950a59063..fd40701a6 100644 --- a/isvctl/tests/test_merger.py +++ b/isvctl/tests/test_merger.py @@ -596,7 +596,7 @@ def test_aws_bare_metal_overrides_serial_console_retention_check(self) -> None: result = merge_yaml_files([self.CONFIGS_DIR / "providers" / "aws" / "config" / "bare_metal.yaml"]) checks = result["tests"]["validations"]["serial_console"]["checks"] - assert checks == [{"SerialConsoleCheck-bm_serial_console": {}}] + assert checks == [{"SerialConsoleCheck": {}}] def test_microk8s_inherits_k8s_validations(self) -> None: """providers/microk8s.yaml imports suites/k8s.yaml and adds overrides.""" diff --git a/isvtest/src/isvtest/catalog.py b/isvtest/src/isvtest/catalog.py index 675addd38..ba1f0e5a7 100644 --- a/isvtest/src/isvtest/catalog.py +++ b/isvtest/src/isvtest/catalog.py @@ -24,6 +24,7 @@ """ import logging +import os from collections.abc import Callable, Iterator from pathlib import Path from typing import Any @@ -192,12 +193,17 @@ def build_label_file_map() -> dict[str, set[str]]: def _build_suite_map() -> dict[str, dict[str, Any]]: - """Map globally unique wiring names to suite placement and requirements.""" + """Map suite wiring names to suite placement and requirements. + + Duplicate wiring names currently last-wins. Global uniqueness enforcement + is deferred to a follow-up PR (``ISVCTL_ENFORCE_UNIQUE_WIRING=1``). + """ configs_dir = _find_configs_dir() if not configs_dir: logger.warning("Could not locate isvctl/configs/ directory") return {} + enforce_unique = os.environ.get("ISVCTL_ENFORCE_UNIQUE_WIRING") == "1" suite_map: dict[str, dict[str, Any]] = {} for config_path in sorted((configs_dir / "suites").glob("*.yaml")): data = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {} @@ -205,7 +211,7 @@ def _build_suite_map() -> dict[str, dict[str, Any]]: platform = tests.get("platform") if isinstance(tests, dict) else None suite = str(platform) if isinstance(platform, str) and platform else config_path.stem.replace("-", "_") for check_name, params in iter_config_checks(config_path): - if check_name in suite_map: + if check_name in suite_map and enforce_unique: raise ValueError(f"Suite wiring name {check_name!r} is not globally unique") requires = params.get("requires", []) suite_map[check_name] = { @@ -219,8 +225,8 @@ def _build_suite_map() -> dict[str, dict[str, Any]]: def build_catalog(*, released_only: bool = True) -> list[dict[str, Any]]: """Discover all validation tests and return structured catalog entries. - Each entry is one globally unique canonical suite wiring. Plain suites - carry ``requires`` while platform suites carry their ``platform`` key. + Each entry is one suite wiring name. Plain suites carry ``requires`` while + platform suites carry their ``platform`` key. Args: released_only: When True, omit tests that are not in the committed @@ -291,9 +297,7 @@ def build_catalog(*, released_only: bool = True) -> list[dict[str, Any]]: omitted_names = sorted( entry["name"] for entry in catalog if resolve_class_key(entry["name"], released_tests) is None ) - catalog = [ - entry for entry in catalog if resolve_class_key(entry["name"], released_tests) is not None - ] + catalog = [entry for entry in catalog if resolve_class_key(entry["name"], released_tests) is not None] if omitted_names: logger.info("Omitted %d unreleased tests from catalog", len(omitted_names)) logger.debug("Unreleased tests omitted from catalog: %s", ", ".join(omitted_names)) diff --git a/isvtest/tests/test_catalog.py b/isvtest/tests/test_catalog.py index 2e27121a0..8523beeca 100644 --- a/isvtest/tests/test_catalog.py +++ b/isvtest/tests/test_catalog.py @@ -71,7 +71,7 @@ class TestBuildCatalog: """Tests for build_catalog function.""" def test_entries_have_suite_contract(self) -> None: - """Every catalog row is one globally unique canonical suite wiring.""" + """Catalog rows expose suite placement and requirement metadata.""" catalog = build_catalog(released_only=False) names = [entry["name"] for entry in catalog] assert catalog @@ -165,8 +165,8 @@ def test_unreleased_env_includes_full_catalog(self) -> None: catalog = build_catalog() names = {e["name"] for e in catalog} - assert "StepSuccessCheck-storage_fixture_volume" in names - assert "FieldExistsCheck-storage_fixture_volume" in names + assert "StepSuccessCheck" in names + assert "FieldExistsCheck" in names def test_labels_are_lists_of_strings(self) -> None: """Test that labels are lists of strings.""" diff --git a/scripts/validate_suite_wiring.py b/scripts/validate_suite_wiring.py index 6ec3c78f3..6171b34ab 100644 --- a/scripts/validate_suite_wiring.py +++ b/scripts/validate_suite_wiring.py @@ -33,6 +33,7 @@ from __future__ import annotations import argparse +import os import re import sys from collections import defaultdict @@ -47,6 +48,8 @@ _NEXT_CATEGORY_LINE = re.compile(r"^ \S") DECLARABLE_CAPABILITIES = {"vm", "bare_metal", "kubernetes", "slurm"} REQUIREMENT_VOCABULARY = DECLARABLE_CAPABILITIES +# Opt-in until unique wiring names land in a dedicated PR. +ENFORCE_UNIQUE_WIRING = os.environ.get("ISVCTL_ENFORCE_UNIQUE_WIRING") == "1" def _check_line_patterns(check_name: str) -> tuple[re.Pattern[str], ...]: @@ -172,8 +175,11 @@ def wiring_errors(suites_dir: Path = SUITES_DIR) -> list[str]: labels = _normalize_labels(params.get("labels")) required_label = required_suite_label(path) previous_location = wiring_locations.get(name) + # Uniqueness enforcement is intentionally deferred to a follow-up + # PR. Keep the check so it can be re-enabled without rediscovery. if previous_location: - errors.append(f"{location}: wiring name is not globally unique (also at {previous_location})") + if ENFORCE_UNIQUE_WIRING: + errors.append(f"{location}: wiring name is not globally unique (also at {previous_location})") else: wiring_locations[name] = location if test_id is None: From b4d7c74bd2371de99a99b5169beb6029c1e2b605 Mon Sep 17 00:00:00 2001 From: Alexandre Begnoche Date: Wed, 22 Jul 2026 16:30:16 -0400 Subject: [PATCH 15/34] fix(cli): reject unknown test options Signed-off-by: Alexandre Begnoche --- isvctl/src/isvctl/cli/test.py | 19 +++++++++--- isvctl/tests/test_test_cli_labels.py | 46 ++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 5 deletions(-) diff --git a/isvctl/src/isvctl/cli/test.py b/isvctl/src/isvctl/cli/test.py index 214783c0c..d03b267c2 100644 --- a/isvctl/src/isvctl/cli/test.py +++ b/isvctl/src/isvctl/cli/test.py @@ -131,16 +131,23 @@ def _human_readable_dry_run(config: RunConfig, capabilities: set[str] | None) -> f" Checks: {len(entries)}", ] for entry in entries: - requirement = ", ".join(entry.requires) or "core" if capabilities is not None and not requirements_satisfied(entry.requires, capabilities): + requirement = ", ".join(entry.requires) declared = ", ".join(sorted(capabilities or ())) or "(none)" lines.append(f" [SKIP] {entry.name}: requires {requirement} (context: {declared})") + elif entry.requires: + lines.append(f" [RUN] {entry.name}: requires {', '.join(entry.requires)}") else: - lines.append(f" [RUN] {entry.name}: requires {requirement}") + lines.append(f" [RUN] {entry.name}") return "\n".join(lines) -@app.command("run", context_settings={"allow_extra_args": True, "ignore_unknown_options": True}) +@app.command( + "run", + # Allow pytest args after `--`, but reject unknown options before it so + # stale flags like `--platform` fail loudly instead of being forwarded. + context_settings={"allow_extra_args": True, "ignore_unknown_options": False}, +) def run( ctx: typer.Context, config_files: Annotated[ @@ -159,7 +166,7 @@ def run( str | None, typer.Option( "--provider", - help="Provider name for label discovery when no --config/-f files are supplied.", + help="Provider name for --suite selection or --label discovery when no --config/-f is supplied.", ), ] = None, suite: Annotated[ @@ -289,6 +296,8 @@ def run( Use -- to pass additional arguments to pytest/isvtest. Examples: + isvctl test run --provider aws --suite k8s + isvctl test run --provider aws --label network isvctl test run -f lab.yaml -f commands.yaml -f suites/k8s.yaml isvctl test run -f config.yaml --set context.node_count=8 isvctl test run -f config.yaml --phase setup @@ -328,7 +337,7 @@ def run( print_error("--provider discovery cannot be combined with --config/-f.") raise typer.Exit(code=1) if not labels: - print_error("--provider requires at least one --label/-l for discovery.") + print_error("--provider requires either --suite NAME or at least one --label/-l.") raise typer.Exit(code=1) known_providers = list_providers(CONFIGS_ROOT) diff --git a/isvctl/tests/test_test_cli_labels.py b/isvctl/tests/test_test_cli_labels.py index 866292ee1..ebbf71028 100644 --- a/isvctl/tests/test_test_cli_labels.py +++ b/isvctl/tests/test_test_cli_labels.py @@ -256,3 +256,49 @@ def test_provider_label_discovery_dry_run_prints_plan_without_running( assert plan["labels"] == ["network"] assert [Path(item["config"]).name for item in plan["configs"]] == ["network.yaml", "observability.yaml"] assert [item["matched_checks"][0]["name"] for item in plan["configs"]] == ["NetworkCheck", "VpcFlowLogsCheck"] + + +def test_provider_without_suite_or_label_mentions_both_options(monkeypatch: pytest.MonkeyPatch) -> None: + """`--provider` alone tells the user to pick `--suite` or `--label`.""" + _FakeOrchestrator.calls = [] + monkeypatch.setattr(test_cli, "Orchestrator", _FakeOrchestrator) + + result = runner.invoke(test_cli.app, ["run", "--provider", "aws", "--dry-run", "--no-upload"]) + + assert result.exit_code == 1, result.output + assert "--suite NAME" in result.output + assert "--label/-l" in result.output + assert _FakeOrchestrator.calls == [] + + +def test_unknown_option_before_separator_is_rejected(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """Stale flags like `--platform` fail before they can be forwarded to pytest.""" + config = _write_config(tmp_path) + _FakeOrchestrator.calls = [] + monkeypatch.setattr(test_cli, "Orchestrator", _FakeOrchestrator) + + result = runner.invoke( + test_cli.app, + ["run", "-f", str(config), "--platform", "k8s", "--no-upload", "--dry-run"], + ) + + assert result.exit_code != 0, result.output + assert "No such option" in result.output or "no such option" in result.output.lower() + assert "--platform" in result.output + assert _FakeOrchestrator.calls == [] + + +def test_pytest_args_after_separator_are_forwarded(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """Args after `--` still reach the orchestrator as pytest extras.""" + config = _write_config(tmp_path) + _FakeOrchestrator.captured = {} + _FakeOrchestrator.calls = [] + monkeypatch.setattr(test_cli, "Orchestrator", _FakeOrchestrator) + + result = runner.invoke( + test_cli.app, + ["run", "-f", str(config), "--no-upload", "--", "-k", "K8sNodeCountCheck"], + ) + + assert result.exit_code == 0, result.output + assert _FakeOrchestrator.captured["extra_pytest_args"] == ["-k", "K8sNodeCountCheck"] From 5133891cf6fc44ccb2da572d73c33f40a185e772 Mon Sep 17 00:00:00 2001 From: Alexandre Begnoche Date: Fri, 24 Jul 2026 10:57:07 -0400 Subject: [PATCH 16/34] refactor(cli): make --capability single-valued MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The four capabilities (kubernetes, slurm, vm, bare_metal) are mutually exclusive execution environments — you run on one, never a combination — so a comma-separated multi-value context never made sense. - Rename the flag --capabilities -> --capability (single value); reject comma-separated input with an explanatory error. - parse_capabilities -> parse_capability (returns str | None). - requirements_satisfied / resolve_entries take one capability instead of a set: `not requires or capability in requires`. - Thread a single capability through the orchestrator and dry-run. `requires` (per-check) stays a list: any-match applicability across the isolated contexts, not a request to combine them. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Alexandre Begnoche --- Makefile | 4 ++-- isvctl/src/isvctl/cli/test.py | 25 ++++++++++---------- isvctl/src/isvctl/config/suite_resolution.py | 23 ++++++++++++------ isvctl/src/isvctl/orchestrator/loop.py | 18 +++++++------- isvctl/tests/test_suite_resolution.py | 16 +++++++------ isvtest/src/isvtest/core/resolution.py | 19 +++++++++------ isvtest/tests/test_resolution.py | 16 ++++++------- 7 files changed, 68 insertions(+), 53 deletions(-) diff --git a/Makefile b/Makefile index 7722f55b2..282ee4a62 100644 --- a/Makefile +++ b/Makefile @@ -89,10 +89,10 @@ define run_demo @echo "==========================================" @echo "Demo test: $(1)" @echo "==========================================" - @echo "Running cmd: ISVCTL_DEMO_MODE=1 uv run isvctl test run -f isvctl/configs/providers/my-isv/config/$(1).yaml$(if $(filter storage,$(1)), --capabilities vm,)" + @echo "Running cmd: ISVCTL_DEMO_MODE=1 uv run isvctl test run -f isvctl/configs/providers/my-isv/config/$(1).yaml$(if $(filter storage,$(1)), --capability vm,)" @ISVCTL_DEMO_MODE=1 uv run isvctl test run \ -f isvctl/configs/providers/my-isv/config/$(1).yaml \ - $(if $(filter storage,$(1)),--capabilities vm,) + $(if $(filter storage,$(1)),--capability vm,) endef demo-test: demo-all ## Alias for demo-all (backward compat) diff --git a/isvctl/src/isvctl/cli/test.py b/isvctl/src/isvctl/cli/test.py index d03b267c2..423b43d74 100644 --- a/isvctl/src/isvctl/cli/test.py +++ b/isvctl/src/isvctl/cli/test.py @@ -48,7 +48,7 @@ ) from isvctl.config.merger import merge_yaml_files from isvctl.config.schema import RunConfig -from isvctl.config.suite_resolution import SuiteResolutionError, parse_capabilities, resolve_suite +from isvctl.config.suite_resolution import SuiteResolutionError, parse_capability, resolve_suite from isvctl.orchestrator.loop import Orchestrator, Phase from isvctl.reporting import check_upload_credentials, create_test_run, get_environment_config, update_test_run @@ -116,25 +116,24 @@ def _junitxml_for_discovered_config(junitxml: Path, match: ProviderConfigMatch, return junitxml.with_name(f"{junitxml.stem}-{match.config_path.stem}{junitxml.suffix}") -def _human_readable_dry_run(config: RunConfig, capabilities: set[str] | None) -> str: +def _human_readable_dry_run(config: RunConfig, capability: str | None) -> str: """Render the validation requirement plan without executing lifecycle steps.""" platform = config.tests.platform if config.tests and config.tests.platform else None suite_type = f"platform ({platform})" if platform else "plain" - context = "not filtered" if capabilities is None else ", ".join(sorted(capabilities)) or "(none)" + context = "not filtered" if capability is None else capability validations = config.tests.validations if config.tests else {} entries = parse_validations(validations) lines = [ "Dry-run plan", f" Suite type: {suite_type}", - f" Capabilities: {context}", + f" Capability: {context}", f" Checks: {len(entries)}", ] for entry in entries: - if capabilities is not None and not requirements_satisfied(entry.requires, capabilities): + if capability is not None and not requirements_satisfied(entry.requires, capability): requirement = ", ".join(entry.requires) - declared = ", ".join(sorted(capabilities or ())) or "(none)" - lines.append(f" [SKIP] {entry.name}: requires {requirement} (context: {declared})") + lines.append(f" [SKIP] {entry.name}: requires {requirement} (context: {capability})") elif entry.requires: lines.append(f" [RUN] {entry.name}: requires {', '.join(entry.requires)}") else: @@ -176,11 +175,11 @@ def run( help="Run one platform or plain suite from the selected provider.", ), ] = None, - capabilities: Annotated[ + capability: Annotated[ str | None, typer.Option( - "--capabilities", - help="Comma-separated capability context used to filter check requirements.", + "--capability", + help="Single capability context (one of the platform suites) used to filter check requirements.", ), ] = None, set_values: Annotated[ @@ -308,7 +307,7 @@ def run( apply_user_config(no_user_config) try: - capability_context = parse_capabilities(capabilities, CONFIGS_ROOT) + capability_context = parse_capability(capability, CONFIGS_ROOT) except SuiteResolutionError as exc: print_error(str(exc)) raise typer.Exit(code=1) @@ -370,7 +369,7 @@ def run( config_files=[match.config_path], provider=None, suite=None, - capabilities=capabilities, + capability=capability, set_values=set_values, phase=phase, labels=labels, @@ -514,7 +513,7 @@ def run( extra_pytest_args=extra_pytest_args, include_labels=labels, exclude_labels=exclude_labels, - capabilities=capability_context, + capability=capability_context, verbose=verbose, junitxml=str(junitxml), ) diff --git a/isvctl/src/isvctl/config/suite_resolution.py b/isvctl/src/isvctl/config/suite_resolution.py index 84dd283ec..14b838bf3 100644 --- a/isvctl/src/isvctl/config/suite_resolution.py +++ b/isvctl/src/isvctl/config/suite_resolution.py @@ -45,19 +45,28 @@ def platform_vocabulary(configs_root: Path) -> frozenset[str]: return frozenset(platforms) -def parse_capabilities(value: str | None, configs_root: Path) -> set[str] | None: - """Parse a comma-separated capability context using platform suite names.""" +def parse_capability(value: str | None, configs_root: Path) -> str | None: + """Parse the single capability context (one platform suite name). + + The four capabilities are mutually exclusive execution environments (you run + on kubernetes OR slurm OR vm OR bare_metal, never a combination), so this + takes exactly one value. + """ if value is None: return None - capabilities = {_normalize_name(item) for item in value.split(",") if item.strip()} + if "," in value: + raise SuiteResolutionError( + f"--capability takes a single platform (got {value!r}); the capabilities " + "are mutually exclusive execution environments, so only one runs at a time." + ) + capability = _normalize_name(value) allowed = platform_vocabulary(configs_root) - unknown = sorted(capabilities - allowed) - if unknown: + if capability not in allowed: raise SuiteResolutionError( - f"Unknown or non-declarable capabilities: {', '.join(unknown)}. " + f"Unknown or non-declarable capability: {value}. " f"Available capabilities: {', '.join(sorted(allowed)) or '(none)'}." ) - return capabilities + return capability def _raw_imports(config_path: Path) -> list[str]: diff --git a/isvctl/src/isvctl/orchestrator/loop.py b/isvctl/src/isvctl/orchestrator/loop.py index 37acaa754..d247d9361 100644 --- a/isvctl/src/isvctl/orchestrator/loop.py +++ b/isvctl/src/isvctl/orchestrator/loop.py @@ -338,15 +338,15 @@ def _apply_step_validation_gates(steps: list[Any], released_tests: set[str] | No def _apply_capability_step_gates( steps: list[Any], validation_entries: list[ValidationEntry], - capabilities: set[str] | None, + capability: str | None, ) -> list[Any]: """Skip a step when every validation bound to it is requirement-filtered.""" - if capabilities is None: + if capability is None: return steps gated_steps: list[Any] = [] for step in steps: bound_entries = [entry for entry in validation_entries if entry.step == step.name] - if bound_entries and all(not requirements_satisfied(entry.requires, capabilities) for entry in bound_entries): + if bound_entries and all(not requirements_satisfied(entry.requires, capability) for entry in bound_entries): logger.info("Skipping step '%s' because all bound validations are capability-filtered", step.name) gated_steps.append(step.model_copy(update={"skip": True})) else: @@ -382,7 +382,7 @@ def __init__( self._extra_pytest_args: list[str] | None = None self._include_labels: list[str] = [] self._exclude_labels: list[str] = [] - self._capabilities: set[str] | None = None + self._capability: str | None = None self._verbose: bool = False self._junitxml: str | None = None @@ -393,7 +393,7 @@ def run( extra_pytest_args: list[str] | None = None, include_labels: list[str] | None = None, exclude_labels: list[str] | None = None, - capabilities: set[str] | None = None, + capability: str | None = None, verbose: bool = False, junitxml: str | None = None, ) -> OrchestratorResult: @@ -408,7 +408,7 @@ def run( (labels are mirrored as pytest marks) include_labels: Labels that selected validations must all contain. exclude_labels: Labels that selected validations must not contain. - capabilities: Capability context used to filter validation requirements. + capability: Single capability context used to filter validation requirements. verbose: Enable verbose output for validations junitxml: Path to write JUnit XML report for validations @@ -422,7 +422,7 @@ def run( self._extra_pytest_args = extra_pytest_args self._include_labels = include_labels or [] self._exclude_labels = exclude_labels or [] - self._capabilities = capabilities + self._capability = capability self._verbose = verbose self._junitxml = junitxml @@ -504,7 +504,7 @@ def _run_steps_mode( if self.config.tests and self.config.tests.validations: all_validations = self.config.tests.validations validation_entries = parse_validations(all_validations) - steps = _apply_capability_step_gates(steps, validation_entries, self._capabilities) + steps = _apply_capability_step_gates(steps, validation_entries, self._capability) logger.info(f"Configured phases: {config_phases}") @@ -823,7 +823,7 @@ def _resolve_validation_entries( exclude_tests=exclude_tests, released_tests=released_tests, render_context=self.context.get_accumulated_context(), - capabilities=self._capabilities, + capability=self._capability, ) def _resolve_remaining_validation_entries( diff --git a/isvctl/tests/test_suite_resolution.py b/isvctl/tests/test_suite_resolution.py index 08bc606e6..7e4735e1d 100644 --- a/isvctl/tests/test_suite_resolution.py +++ b/isvctl/tests/test_suite_resolution.py @@ -11,7 +11,7 @@ from isvctl.config.schema import RunConfig from isvctl.config.suite_resolution import ( SuiteResolutionError, - parse_capabilities, + parse_capability, resolve_suite, ) @@ -41,14 +41,16 @@ def test_one_suite_flag_resolves_platform_and_plain_suites(tmp_path: Path) -> No assert plain.platform is None -def test_capabilities_use_catalog_vocabulary(tmp_path: Path) -> None: - """Unknown capabilities are rejected while omitted context disables filtering.""" +def test_capability_uses_catalog_vocabulary(tmp_path: Path) -> None: + """An unknown capability is rejected while omitted context disables filtering.""" _write_catalog(tmp_path) - assert parse_capabilities(None, tmp_path) is None - assert parse_capabilities("k8s", tmp_path) == {"kubernetes"} - with pytest.raises(SuiteResolutionError, match="non-declarable capabilities: compute"): - parse_capabilities("compute", tmp_path) + assert parse_capability(None, tmp_path) is None + assert parse_capability("k8s", tmp_path) == "kubernetes" + with pytest.raises(SuiteResolutionError, match="non-declarable capability: compute"): + parse_capability("compute", tmp_path) + with pytest.raises(SuiteResolutionError, match="single platform"): + parse_capability("kubernetes,vm", tmp_path) def test_platform_suites_reject_requires_and_unknown_platforms() -> None: diff --git a/isvtest/src/isvtest/core/resolution.py b/isvtest/src/isvtest/core/resolution.py index 4fc8db196..6c075c7e1 100644 --- a/isvtest/src/isvtest/core/resolution.py +++ b/isvtest/src/isvtest/core/resolution.py @@ -122,10 +122,15 @@ def _wiring_requires(params_template: Any) -> tuple[str, ...]: return tuple(item for item in value if isinstance(item, str)) -def requirements_satisfied(requires: Iterable[str], capabilities: AbstractSet[str]) -> bool: - """Return whether any required platform is present in the capability context.""" +def requirements_satisfied(requires: Iterable[str], capability: str) -> bool: + """Return whether the single capability context satisfies a check's requirements. + + The four capabilities are mutually exclusive execution environments, so a run + carries exactly one. A core check (empty ``requires``) always runs; otherwise + the check runs when the context capability is among its any-match prerequisites. + """ required = set(requires) - return not required or not required.isdisjoint(capabilities) + return not required or capability in required def parse_validations(raw_config: Mapping[str, Any]) -> list[ValidationEntry]: @@ -189,7 +194,7 @@ def resolve_entries( exclude_tests: AbstractSet[str], released_tests: AbstractSet[str] | None, render_context: Mapping[str, Any], - capabilities: AbstractSet[str] | None = None, + capability: str | None = None, ) -> list[ResolvedEntry]: """Resolve validation entries into ready or terminal outcomes. @@ -203,7 +208,7 @@ def resolve_entries( exclude_tests: Validation names excluded by config. released_tests: Released test manifest, or None when unreleased checks are included. render_context: Jinja context for validation parameter rendering. - capabilities: Declared capability context, or None to disable requirement filtering. + capability: Declared capability context (a single platform), or None to disable requirement filtering. Returns: A resolved entry for every input entry, in input order. @@ -234,9 +239,9 @@ def resolve_entries( resolved.append(_skip(entry, SkipReason.EXCLUDED, f"validation '{entry.name}' is excluded by name")) continue - if capabilities is not None and not requirements_satisfied(entry.requires, capabilities): + if capability is not None and not requirements_satisfied(entry.requires, capability): requirement_list = ", ".join(entry.requires) or "(none)" - context_list = ", ".join(sorted(capabilities or ())) or "(none)" + context_list = capability resolved.append( _skip( entry, diff --git a/isvtest/tests/test_resolution.py b/isvtest/tests/test_resolution.py index 40eb2a7a0..fdf0a54ab 100644 --- a/isvtest/tests/test_resolution.py +++ b/isvtest/tests/test_resolution.py @@ -89,7 +89,7 @@ def _resolve( exclude_tests: set[str] | None = None, released_tests: set[str] | None = None, render_context: dict[str, Any] | None = None, - capabilities: set[str] | None = None, + capability: str | None = None, ) -> ResolvedEntry: """Resolve one entry and return the single result.""" results = resolve_entries( @@ -102,7 +102,7 @@ def _resolve( exclude_tests=set() if exclude_tests is None else exclude_tests, released_tests=released_tests, render_context={} if render_context is None else render_context, - capabilities=capabilities, + capability=capability, ) assert len(results) == 1 return results[0] @@ -112,23 +112,23 @@ def test_any_declared_requirement_satisfies_capability_filter() -> None: """Orthogonal platform requirements use OR semantics.""" entry = _entry(requires=("vm", "bare_metal")) - assert _resolve(entry, capabilities={"vm"}).is_ready - assert _resolve(entry, capabilities={"bare_metal"}).is_ready - assert _resolve(_entry(requires=()), capabilities={"kubernetes"}).is_ready + assert _resolve(entry, capability="vm").is_ready + assert _resolve(entry, capability="bare_metal").is_ready + assert _resolve(_entry(requires=()), capability="kubernetes").is_ready def test_capability_filter_has_explicit_skip_reason() -> None: """An unmet prerequisite reports both the requirement and active context.""" - resolved = _resolve(_entry(requires=("vm", "bare_metal")), capabilities={"kubernetes"}) + resolved = _resolve(_entry(requires=("vm", "bare_metal")), capability="kubernetes") assert resolved.state == State.SKIPPED assert resolved.skip_reason == SkipReason.CAPABILITY_REQUIREMENT assert resolved.message == "requires vm, bare_metal (context: kubernetes)" -def test_omitted_capabilities_disables_requirement_filtering() -> None: +def test_omitted_capability_disables_requirement_filtering() -> None: """Local development without a capability context runs every check.""" - assert _resolve(_entry(requires=("kubernetes",)), capabilities=None).is_ready + assert _resolve(_entry(requires=("kubernetes",)), capability=None).is_ready @pytest.mark.parametrize( From 3ec094fdceb2e0f4a997f977590f7e1e64b6a27f Mon Sep 17 00:00:00 2001 From: Alexandre Begnoche Date: Fri, 24 Jul 2026 11:21:41 -0400 Subject: [PATCH 17/34] feat(catalog): enforce disjoint capability/suite namespaces MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A plain suite named after a declarable capability (vm/bare_metal/kubernetes/slurm) would collapse the merged frontend test-target list and cross-wire the backend's flat-selection re-split (selection ∩ catalog.platforms). Reject the collision at the upload chokepoint (catalog_document) and in the wiring lint (validate_suite_wiring.py), reusing DECLARABLE_CAPABILITIES, so the capability-vs-suite distinction stays recoverable by construction. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Alexandre Begnoche --- isvtest/src/isvtest/catalog.py | 27 +++++++++++++-- isvtest/tests/test_catalog.py | 17 +++++++++ scripts/tests/test_validate_suite_wiring.py | 38 +++++++++++++++++++++ scripts/validate_suite_wiring.py | 8 +++++ 4 files changed, 88 insertions(+), 2 deletions(-) diff --git a/isvtest/src/isvtest/catalog.py b/isvtest/src/isvtest/catalog.py index ba1f0e5a7..147f81b06 100644 --- a/isvtest/src/isvtest/catalog.py +++ b/isvtest/src/isvtest/catalog.py @@ -333,6 +333,26 @@ def build_suite_vocabulary() -> list[str]: return sorted(suites) +def _assert_disjoint_vocabulary(platforms: list[str], suites: list[str]) -> None: + """Reject a plain suite named after a declarable capability. + + Capabilities and plain suites share one uppercased namespace downstream: the + frontend merges ``platforms`` and ``suites`` into a single selectable + "test target" list, and the backend re-splits a flat selection by + intersecting it with the declared ``platforms``. Both are unambiguous only + while the two namespaces are disjoint, so enforce it at the upload + chokepoint (checked against the full reserved set, not just declared + platforms, so an undeclared capability word like ``slurm`` is caught too). + """ + collisions = sorted(set(suites) & (set(platforms) | DECLARABLE_CAPABILITIES)) + if collisions: + raise ValueError( + "plain suite names collide with declarable capabilities: " + f"{', '.join(collisions)}; rename the suite file(s) so the capability " + "and suite namespaces stay disjoint" + ) + + def catalog_document(entries: list[dict[str, Any]], version: str) -> dict[str, Any]: """Wrap catalog ``entries`` in the versioned upload/artifact envelope. @@ -341,11 +361,14 @@ def catalog_document(entries: list[dict[str, Any]], version: str) -> dict[str, A ``labels`` are intentionally not summarized at the top level - a consumer can derive the label universe from the entries when needed. """ + platforms = build_capability_vocabulary() + suites = build_suite_vocabulary() + _assert_disjoint_vocabulary(platforms, suites) return { "schemaVersion": CATALOG_SCHEMA_VERSION, "isvTestVersion": version, - "platforms": build_capability_vocabulary(), - "suites": build_suite_vocabulary(), + "platforms": platforms, + "suites": suites, "entries": entries, } diff --git a/isvtest/tests/test_catalog.py b/isvtest/tests/test_catalog.py index 8523beeca..e4da53abb 100644 --- a/isvtest/tests/test_catalog.py +++ b/isvtest/tests/test_catalog.py @@ -17,8 +17,11 @@ from unittest.mock import patch +import pytest + from isvtest.catalog import ( CATALOG_SCHEMA_VERSION, + _assert_disjoint_vocabulary, build_capability_vocabulary, build_catalog, build_suite_vocabulary, @@ -66,6 +69,20 @@ def test_catalog_document_wraps_entries_with_metadata(self) -> None: # The label universe is intentionally not summarized at the top level. assert "labels" not in doc + def test_disjoint_vocabulary_accepts_distinct_namespaces(self) -> None: + """Plain suite names that are not capability words pass the guard.""" + _assert_disjoint_vocabulary(["vm", "kubernetes"], ["storage", "iam", "network"]) + + def test_disjoint_vocabulary_rejects_suite_named_after_capability(self) -> None: + """A plain suite named after any declarable capability is a namespace collision.""" + with pytest.raises(ValueError, match="kubernetes"): + _assert_disjoint_vocabulary(["vm", "kubernetes"], ["storage", "kubernetes"]) + + def test_disjoint_vocabulary_rejects_undeclared_capability_word(self) -> None: + """Collision is checked against the full reserved set, not just declared platforms.""" + with pytest.raises(ValueError, match="slurm"): + _assert_disjoint_vocabulary(["vm"], ["slurm"]) + class TestBuildCatalog: """Tests for build_catalog function.""" diff --git a/scripts/tests/test_validate_suite_wiring.py b/scripts/tests/test_validate_suite_wiring.py index 55d41e6ac..1a63042cf 100644 --- a/scripts/tests/test_validate_suite_wiring.py +++ b/scripts/tests/test_validate_suite_wiring.py @@ -137,3 +137,41 @@ def test_plain_suite_requires_are_explicit_and_valid(tmp_path: Path) -> None: errors = validate_suite_wiring.wiring_errors(tmp_path) assert any("MissingCheck" in error and "missing requires" in error for error in errors) assert any("InvalidCheck" in error and "requires must contain only" in error for error in errors) + + +def test_wiring_errors_rejects_plain_suite_named_after_capability(tmp_path: Path) -> None: + """A plain suite file named like a declarable capability is a namespace collision.""" + (tmp_path / "kubernetes.yaml").write_text( + """\ +tests: + validations: + sample: + checks: + SomeCheck: + test_id: "N/A" + labels: ["demo"] + requires: [] +""" + ) + + errors = validate_suite_wiring.wiring_errors(tmp_path) + assert any("kubernetes" in error and "collides with a declarable capability" in error for error in errors) + + +def test_wiring_errors_allows_platform_suite_named_after_capability(tmp_path: Path) -> None: + """The kubernetes *platform* suite (declares tests.platform) is not a collision.""" + (tmp_path / "k8s.yaml").write_text( + """\ +tests: + platform: kubernetes + validations: + sample: + checks: + SomeCheck: + test_id: "N/A" + labels: ["kubernetes"] +""" + ) + + errors = validate_suite_wiring.wiring_errors(tmp_path) + assert not any("collides with a declarable capability" in error for error in errors) diff --git a/scripts/validate_suite_wiring.py b/scripts/validate_suite_wiring.py index 6171b34ab..6e0ff60fe 100644 --- a/scripts/validate_suite_wiring.py +++ b/scripts/validate_suite_wiring.py @@ -164,6 +164,14 @@ def wiring_errors(suites_dir: Path = SUITES_DIR) -> list[str]: errors.append(f"{path}: tests.module is no longer supported") if platform is not None and platform not in DECLARABLE_CAPABILITIES: errors.append(f"{path}: tests.platform must be one of: {', '.join(sorted(DECLARABLE_CAPABILITIES))}") + suite_is_platform = isinstance(platform, str) and platform in DECLARABLE_CAPABILITIES + if not suite_is_platform: + suite_name = path.stem.replace("-", "_") + if suite_name in DECLARABLE_CAPABILITIES: + errors.append( + f"{path}: plain suite name {suite_name!r} collides with a declarable " + "capability; rename the file so capability and suite namespaces stay disjoint" + ) for category, name, params in checks: key = (path, category, name) line_numbers = find_check_line_numbers(lines, category, name) From 4e229005d74836e5374d9453c1aa1025748ad344 Mon Sep 17 00:00:00 2001 From: Alexandre Begnoche Date: Fri, 24 Jul 2026 11:38:15 -0400 Subject: [PATCH 18/34] feat(config): harden requires validation at validate and lint time test validate now rejects unknown/duplicate requires values via the Pydantic suite-shape validator, matching the run-time and wiring checks. validate_suite_wiring flags a requires naming a capability with no platform suite (no ISV can declare it, so the check is unreachable). Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Alexandre Begnoche --- isvctl/src/isvctl/config/schema.py | 13 +++++++++ isvctl/tests/test_suite_resolution.py | 20 +++++++++++++ scripts/tests/test_validate_suite_wiring.py | 31 +++++++++++++++++++++ scripts/validate_suite_wiring.py | 22 +++++++++++++++ 4 files changed, 86 insertions(+) diff --git a/isvctl/src/isvctl/config/schema.py b/isvctl/src/isvctl/config/schema.py index 698c8ccd6..898075972 100644 --- a/isvctl/src/isvctl/config/schema.py +++ b/isvctl/src/isvctl/config/schema.py @@ -388,6 +388,19 @@ def validate_suite_shape(self) -> "ValidationConfig": raise ValueError("requires is not allowed in platform suites") if any("platforms" in entry.params_template for entry in entries): raise ValueError("per-check platforms is no longer supported; use requires in plain suites") + if not self.platform: + for entry in entries: + raw_requires = entry.params_template.get("requires") + if raw_requires is None: + continue + if not isinstance(raw_requires, list) or any( + not isinstance(value, str) or value not in DECLARABLE_CAPABILITIES for value in raw_requires + ): + raise ValueError( + f"requires must be a list containing only: {', '.join(sorted(DECLARABLE_CAPABILITIES))}" + ) + if len(raw_requires) != len(set(raw_requires)): + raise ValueError("requires must not contain duplicates") return self diff --git a/isvctl/tests/test_suite_resolution.py b/isvctl/tests/test_suite_resolution.py index 7e4735e1d..fa2fc73c4 100644 --- a/isvctl/tests/test_suite_resolution.py +++ b/isvctl/tests/test_suite_resolution.py @@ -61,3 +61,23 @@ def test_platform_suites_reject_requires_and_unknown_platforms() -> None: RunConfig.model_validate({"tests": {"platform": "kubernetes", "validations": validation}}) with pytest.raises(ValidationError, match=r"tests\.platform must be one of"): RunConfig.model_validate({"tests": {"platform": "compute", "validations": {}}}) + + +def test_plain_suite_rejects_unknown_requires_vocabulary() -> None: + """`test validate` catches a bad `requires` value, not just a run does.""" + validation = {"sample": {"checks": {"PlainCheck": {"requires": ["compute"]}}}} + with pytest.raises(ValidationError, match="requires must be a list containing only"): + RunConfig.model_validate({"tests": {"validations": validation}}) + + +def test_plain_suite_rejects_duplicate_requires() -> None: + """Duplicate requirements are rejected at schema-validation time.""" + validation = {"sample": {"checks": {"PlainCheck": {"requires": ["vm", "vm"]}}}} + with pytest.raises(ValidationError, match="requires must not contain duplicates"): + RunConfig.model_validate({"tests": {"validations": validation}}) + + +def test_plain_suite_accepts_valid_requires() -> None: + """A well-formed requires list passes schema validation.""" + validation = {"sample": {"checks": {"PlainCheck": {"requires": ["vm", "bare_metal"]}}}} + RunConfig.model_validate({"tests": {"validations": validation}}) diff --git a/scripts/tests/test_validate_suite_wiring.py b/scripts/tests/test_validate_suite_wiring.py index 1a63042cf..6c7798e96 100644 --- a/scripts/tests/test_validate_suite_wiring.py +++ b/scripts/tests/test_validate_suite_wiring.py @@ -158,6 +158,37 @@ def test_wiring_errors_rejects_plain_suite_named_after_capability(tmp_path: Path assert any("kubernetes" in error and "collides with a declarable capability" in error for error in errors) +def test_wiring_errors_flags_requires_with_no_platform_suite(tmp_path: Path) -> None: + """A `requires` naming a capability that has no platform suite is unreachable.""" + (tmp_path / "vm.yaml").write_text( + """\ +tests: + platform: vm + validations: + example: + checks: + VmCheck: + test_id: "N/A" + labels: ["vm"] +""" + ) + (tmp_path / "demo.yaml").write_text( + """\ +tests: + validations: + example: + checks: + DeadCheck: + test_id: "N/A" + labels: ["demo"] + requires: [slurm] +""" + ) + + errors = validate_suite_wiring.wiring_errors(tmp_path) + assert any("DeadCheck" in error and "slurm" in error and "no platform suite" in error for error in errors) + + def test_wiring_errors_allows_platform_suite_named_after_capability(tmp_path: Path) -> None: """The kubernetes *platform* suite (declares tests.platform) is not a collision.""" (tmp_path / "k8s.yaml").write_text( diff --git a/scripts/validate_suite_wiring.py b/scripts/validate_suite_wiring.py index 6e0ff60fe..5602f99ab 100644 --- a/scripts/validate_suite_wiring.py +++ b/scripts/validate_suite_wiring.py @@ -149,6 +149,21 @@ def wiring_errors(suites_dir: Path = SUITES_DIR) -> list[str]: occurrence: dict[tuple[Path, str, str], int] = defaultdict(int) wiring_locations: dict[str, str] = {} + # A `requires` value is only satisfiable if an ISV can declare that + # capability, which requires a platform suite to exist for it. Collect the + # platform capabilities that actually have a suite so unreachable (dead) + # requirements can be flagged below. + declared_platforms: set[str] = set() + for path in sorted(suites_dir.glob("*.yaml")): + try: + data = yaml.safe_load(path.read_text()) or {} + except (OSError, yaml.YAMLError): + continue + tests = data.get("tests") if isinstance(data, dict) else None + platform = tests.get("platform") if isinstance(tests, dict) else None + if isinstance(platform, str) and platform in DECLARABLE_CAPABILITIES: + declared_platforms.add(platform) + for path in sorted(suites_dir.glob("*.yaml")): try: lines = path.read_text().splitlines() @@ -214,6 +229,13 @@ def wiring_errors(suites_dir: Path = SUITES_DIR) -> list[str]: ) elif len(requires) != len(set(requires)): errors.append(f"{location}: requires must not contain duplicates") + else: + dead = sorted(set(requires) - declared_platforms) + if dead: + errors.append( + f"{location}: requires names {', '.join(dead)} which has no platform " + "suite; no ISV can declare it, so the check is unreachable" + ) return errors From e5a83e75c1b59733023e6b05b1a87f808c0b19e2 Mon Sep 17 00:00:00 2001 From: Alexandre Begnoche Date: Fri, 24 Jul 2026 11:49:57 -0400 Subject: [PATCH 19/34] fix(storage): gate EKS lifecycle by capability Signed-off-by: Alexandre Begnoche --- .../configs/providers/aws/config/storage.yaml | 2 ++ isvctl/schemas/config.schema.json | 8 ++++++++ isvctl/src/isvctl/config/schema.py | 18 +++++++++++++++++- isvctl/src/isvctl/orchestrator/loop.py | 11 ++++++++++- isvctl/tests/test_orchestrator_loop.py | 15 +++++++++++++++ isvctl/tests/test_schema.py | 10 ++++++++++ isvctl/tests/test_suite_resolution.py | 13 +++++++++++++ 7 files changed, 75 insertions(+), 2 deletions(-) diff --git a/isvctl/configs/providers/aws/config/storage.yaml b/isvctl/configs/providers/aws/config/storage.yaml index b2d8e5dcc..7a08a2a92 100644 --- a/isvctl/configs/providers/aws/config/storage.yaml +++ b/isvctl/configs/providers/aws/config/storage.yaml @@ -62,6 +62,7 @@ commands: - name: setup_cluster phase: setup command: "../scripts/eks/setup.sh" + requires: [kubernetes] timeout: 5400 env: TF_AUTO_APPROVE: "true" @@ -234,6 +235,7 @@ commands: - name: teardown_cluster phase: teardown command: "../scripts/eks/teardown.sh" + requires: [kubernetes] timeout: 1800 env: TF_AUTO_APPROVE: "true" diff --git a/isvctl/schemas/config.schema.json b/isvctl/schemas/config.schema.json index 44efdbe3b..e6acb8120 100644 --- a/isvctl/schemas/config.schema.json +++ b/isvctl/schemas/config.schema.json @@ -145,6 +145,14 @@ "title": "Skip", "type": "boolean" }, + "requires": { + "description": "Capability contexts allowed to run this step. Empty delegates capability gating to bound validations.", + "items": { + "type": "string" + }, + "title": "Requires", + "type": "array" + }, "requires_available_validations": { "description": "Validation names that must be available after release filtering for this step to run. Unreleased validations are available only when ISVTEST_INCLUDE_UNRELEASED=1.", "items": { diff --git a/isvctl/src/isvctl/config/schema.py b/isvctl/src/isvctl/config/schema.py index 898075972..24960ae02 100644 --- a/isvctl/src/isvctl/config/schema.py +++ b/isvctl/src/isvctl/config/schema.py @@ -25,7 +25,7 @@ from typing import Any from isvtest.core.resolution import DECLARABLE_CAPABILITIES, parse_validations -from pydantic import BaseModel, ConfigDict, Field, model_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator class LabConfig(BaseModel): @@ -87,6 +87,12 @@ class StepConfig(BaseModel): env: dict[str, str] = Field(default_factory=dict, description="Additional environment variables") working_dir: str | None = Field(default=None, description="Working directory for command execution") skip: bool = Field(default=False, description="Skip this step") + requires: list[str] = Field( + default_factory=list, + description=( + "Capability contexts allowed to run this step. Empty delegates capability gating to bound validations." + ), + ) requires_available_validations: list[str] = Field( default_factory=list, description=( @@ -108,6 +114,16 @@ class StepConfig(BaseModel): description="Additional argument patterns to mask in logs (e.g., ['--my-secret'])", ) + @field_validator("requires") + @classmethod + def validate_requires(cls, requires: list[str]) -> list[str]: + """Keep step requirements in the same vocabulary as validation requirements.""" + if any(requirement not in DECLARABLE_CAPABILITIES for requirement in requires): + raise ValueError(f"requires must contain only: {', '.join(sorted(DECLARABLE_CAPABILITIES))}") + if len(requires) != len(set(requires)): + raise ValueError("requires must not contain duplicates") + return requires + class PlatformCommands(BaseModel): """Lifecycle commands for a specific platform. diff --git a/isvctl/src/isvctl/orchestrator/loop.py b/isvctl/src/isvctl/orchestrator/loop.py index d247d9361..24c4d9e81 100644 --- a/isvctl/src/isvctl/orchestrator/loop.py +++ b/isvctl/src/isvctl/orchestrator/loop.py @@ -340,11 +340,20 @@ def _apply_capability_step_gates( validation_entries: list[ValidationEntry], capability: str | None, ) -> list[Any]: - """Skip a step when every validation bound to it is requirement-filtered.""" + """Skip a step when its explicit or validation-derived requirements do not match.""" if capability is None: return steps gated_steps: list[Any] = [] for step in steps: + explicit_requires = getattr(step, "requires", []) + if explicit_requires and not requirements_satisfied(explicit_requires, capability): + logger.info( + "Skipping step '%s' because it requires capability: %s", + step.name, + ", ".join(explicit_requires), + ) + gated_steps.append(step.model_copy(update={"skip": True})) + continue bound_entries = [entry for entry in validation_entries if entry.step == step.name] if bound_entries and all(not requirements_satisfied(entry.requires, capability) for entry in bound_entries): logger.info("Skipping step '%s' because all bound validations are capability-filtered", step.name) diff --git a/isvctl/tests/test_orchestrator_loop.py b/isvctl/tests/test_orchestrator_loop.py index 3dee004b9..edf691f9e 100644 --- a/isvctl/tests/test_orchestrator_loop.py +++ b/isvctl/tests/test_orchestrator_loop.py @@ -32,6 +32,7 @@ from isvctl.orchestrator.loop import ( Orchestrator, Phase, + _apply_capability_step_gates, _entries_missing_from_junit, _merge_junit_xmls, _write_terminal_junit_xml, @@ -62,6 +63,20 @@ """ +def test_explicit_step_requires_gate_unbound_lifecycle_steps() -> None: + """An unbound teardown step is skipped when its explicit capability does not match.""" + steps = [ + StepConfig(name="setup_cluster", command="setup", phase="setup", requires=["kubernetes"]), + StepConfig(name="teardown_cluster", command="teardown", phase="teardown", requires=["kubernetes"]), + ] + + vm_steps = _apply_capability_step_gates(steps, [], "vm") + kubernetes_steps = _apply_capability_step_gates(steps, [], "kubernetes") + + assert all(step.skip for step in vm_steps) + assert all(not step.skip for step in kubernetes_steps) + + def test_python_script_path_falls_back_to_current_working_directory( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, diff --git a/isvctl/tests/test_schema.py b/isvctl/tests/test_schema.py index e462dd377..7eb616290 100644 --- a/isvctl/tests/test_schema.py +++ b/isvctl/tests/test_schema.py @@ -75,6 +75,7 @@ def test_minimal_step(self) -> None: assert step.timeout == 300 assert step.phase == "setup" assert step.skip is False + assert step.requires == [] assert step.requires_available_validations == [] def test_full_step(self) -> None: @@ -88,6 +89,7 @@ def test_full_step(self) -> None: working_dir="/tmp", phase="setup", skip=False, + requires=["vm", "bare_metal"], requires_available_validations=["NewCheck"], continue_on_failure=True, output_schema="vpc", @@ -98,10 +100,18 @@ def test_full_step(self) -> None: assert step.timeout == 600 assert step.env == {"AWS_REGION": "us-west-2"} assert step.phase == "setup" + assert step.requires == ["vm", "bare_metal"] assert step.requires_available_validations == ["NewCheck"] assert step.continue_on_failure is True assert step.output_schema == "vpc" + def test_step_rejects_unknown_or_duplicate_requires(self) -> None: + """Step requirements use the declarable capability vocabulary.""" + with pytest.raises(ValidationError, match="requires must contain only"): + StepConfig(name="setup", command="echo", requires=["compute"]) + with pytest.raises(ValidationError, match="requires must not contain duplicates"): + StepConfig(name="setup", command="echo", requires=["vm", "vm"]) + class TestCommandOutput: """Tests for CommandOutput model (setup command JSON output).""" diff --git a/isvctl/tests/test_suite_resolution.py b/isvctl/tests/test_suite_resolution.py index fa2fc73c4..ee9f3be64 100644 --- a/isvctl/tests/test_suite_resolution.py +++ b/isvctl/tests/test_suite_resolution.py @@ -8,6 +8,7 @@ import pytest from pydantic import ValidationError +from isvctl.config.merger import merge_yaml_files from isvctl.config.schema import RunConfig from isvctl.config.suite_resolution import ( SuiteResolutionError, @@ -15,6 +16,8 @@ resolve_suite, ) +CONFIGS_ROOT = Path(__file__).resolve().parents[1] / "configs" + def _write_catalog(root: Path) -> None: """Write one platform suite, one plain suite, and provider imports.""" @@ -81,3 +84,13 @@ def test_plain_suite_accepts_valid_requires() -> None: """A well-formed requires list passes schema validation.""" validation = {"sample": {"checks": {"PlainCheck": {"requires": ["vm", "bare_metal"]}}}} RunConfig.model_validate({"tests": {"validations": validation}}) + + +def test_aws_storage_eks_lifecycle_requires_kubernetes() -> None: + """Both sides of the destructive EKS lifecycle carry the same capability gate.""" + config_path = CONFIGS_ROOT / "providers" / "aws" / "config" / "storage.yaml" + config = RunConfig.model_validate(merge_yaml_files([str(config_path)])) + steps = {step.name: step for step in config.get_steps("storage")} + + assert steps["setup_cluster"].requires == ["kubernetes"] + assert steps["teardown_cluster"].requires == ["kubernetes"] From f123565fcbe9cf3392ae49eb0c446fb0a8b1ebcd Mon Sep 17 00:00:00 2001 From: Alexandre Begnoche Date: Fri, 24 Jul 2026 11:59:38 -0400 Subject: [PATCH 20/34] fix(selection): default plain suites to core checks Signed-off-by: Alexandre Begnoche --- .../configs/providers/aws/config/storage.yaml | 2 + .../providers/my-isv/config/storage.yaml | 2 + isvctl/configs/suites/storage.yaml | 2 +- isvctl/src/isvctl/cli/test.py | 9 ++- isvctl/tests/test_suite_resolution.py | 14 +++-- isvctl/tests/test_test_cli_labels.py | 63 +++++++++++++++++++ 6 files changed, 85 insertions(+), 7 deletions(-) diff --git a/isvctl/configs/providers/aws/config/storage.yaml b/isvctl/configs/providers/aws/config/storage.yaml index 7a08a2a92..91d8911be 100644 --- a/isvctl/configs/providers/aws/config/storage.yaml +++ b/isvctl/configs/providers/aws/config/storage.yaml @@ -208,6 +208,7 @@ commands: - name: teardown_volume phase: teardown command: "python3 ../scripts/storage/teardown_volume.py" + requires: [vm, bare_metal] args: - "--region" - "{{region}}" @@ -219,6 +220,7 @@ commands: - name: teardown phase: teardown command: "python3 ../scripts/vm/teardown.py" + requires: [vm, bare_metal] args: - "--instance-id" - "{{steps.launch_instance.instance_id}}" diff --git a/isvctl/configs/providers/my-isv/config/storage.yaml b/isvctl/configs/providers/my-isv/config/storage.yaml index c0efca903..79421fc82 100644 --- a/isvctl/configs/providers/my-isv/config/storage.yaml +++ b/isvctl/configs/providers/my-isv/config/storage.yaml @@ -240,6 +240,7 @@ commands: - name: teardown_volume phase: teardown command: "python ../scripts/storage/teardown_volume.py" + requires: [vm, bare_metal] args: - "--region" - "{{region}}" @@ -251,6 +252,7 @@ commands: - name: teardown phase: teardown command: "python ../scripts/vm/teardown.py" + requires: [vm, bare_metal] args: - "--instance-id" - "{{steps.launch_instance.instance_id}}" diff --git a/isvctl/configs/suites/storage.yaml b/isvctl/configs/suites/storage.yaml index e17897ac6..edc568631 100644 --- a/isvctl/configs/suites/storage.yaml +++ b/isvctl/configs/suites/storage.yaml @@ -243,7 +243,7 @@ tests: StepSuccessCheck: test_id: "N/A" labels: ["storage"] - requires: [] + requires: [vm, bare_metal] # Kubernetes storage checks use a normal provider-owned cluster fixture. k8s_storage: diff --git a/isvctl/src/isvctl/cli/test.py b/isvctl/src/isvctl/cli/test.py index 423b43d74..e598302b4 100644 --- a/isvctl/src/isvctl/cli/test.py +++ b/isvctl/src/isvctl/cli/test.py @@ -54,6 +54,7 @@ logger = logging.getLogger(__name__) CONFIGS_ROOT = Path(__file__).resolve().parents[3] / "configs" +CORE_REQUIREMENT_CONTEXT = "core" class TeeWriter: @@ -172,7 +173,10 @@ def run( str | None, typer.Option( "--suite", - help="Run one platform or plain suite from the selected provider.", + help=( + "Run one platform or plain suite from the selected provider. " + "Plain suites default to core checks unless --capability is set." + ), ), ] = None, capability: Annotated[ @@ -328,6 +332,9 @@ def run( print_error(str(exc)) raise typer.Exit(code=1) print_progress(f"Selected {selected_suite.name!r} suite for provider {provider!r}.") + if selected_suite.platform is None and capability_context is None: + capability_context = CORE_REQUIREMENT_CONTEXT + print_progress("No capability selected; running the plain suite's core checks.") config_files = [selected_suite.config_path] provider = None diff --git a/isvctl/tests/test_suite_resolution.py b/isvctl/tests/test_suite_resolution.py index ee9f3be64..c5db1873e 100644 --- a/isvctl/tests/test_suite_resolution.py +++ b/isvctl/tests/test_suite_resolution.py @@ -86,11 +86,15 @@ def test_plain_suite_accepts_valid_requires() -> None: RunConfig.model_validate({"tests": {"validations": validation}}) -def test_aws_storage_eks_lifecycle_requires_kubernetes() -> None: - """Both sides of the destructive EKS lifecycle carry the same capability gate.""" - config_path = CONFIGS_ROOT / "providers" / "aws" / "config" / "storage.yaml" +@pytest.mark.parametrize("provider", ["aws", "my-isv"]) +def test_storage_cleanup_steps_have_explicit_capability_gates(provider: str) -> None: + """Destructive storage cleanup runs only in the context that owns its resources.""" + config_path = CONFIGS_ROOT / "providers" / provider / "config" / "storage.yaml" config = RunConfig.model_validate(merge_yaml_files([str(config_path)])) steps = {step.name: step for step in config.get_steps("storage")} - assert steps["setup_cluster"].requires == ["kubernetes"] - assert steps["teardown_cluster"].requires == ["kubernetes"] + assert steps["teardown_volume"].requires == ["vm", "bare_metal"] + assert steps["teardown"].requires == ["vm", "bare_metal"] + if provider == "aws": + assert steps["setup_cluster"].requires == ["kubernetes"] + assert steps["teardown_cluster"].requires == ["kubernetes"] diff --git a/isvctl/tests/test_test_cli_labels.py b/isvctl/tests/test_test_cli_labels.py index ebbf71028..ad34e315d 100644 --- a/isvctl/tests/test_test_cli_labels.py +++ b/isvctl/tests/test_test_cli_labels.py @@ -271,6 +271,69 @@ def test_provider_without_suite_or_label_mentions_both_options(monkeypatch: pyte assert _FakeOrchestrator.calls == [] +def test_plain_suite_without_capability_defaults_to_core( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """A declared plain suite runs core checks unless a capability is selected.""" + configs_root = tmp_path / "configs" + suite_path = configs_root / "suites" / "storage.yaml" + suite_path.parent.mkdir(parents=True) + suite_path.write_text( + """\ +tests: + validations: + core: + checks: + CoreCheck: + test_id: "N/A" + labels: ["storage"] + requires: [] + vm: + checks: + VmCheck: + test_id: "N/A" + labels: ["storage"] + requires: [vm] +""", + encoding="utf-8", + ) + (configs_root / "suites" / "vm.yaml").write_text( + "tests:\n platform: vm\n validations: {}\n", + encoding="utf-8", + ) + _write_provider_config(configs_root, "aws", "storage.yaml", "storage.yaml", "storage") + monkeypatch.setattr(test_cli, "CONFIGS_ROOT", configs_root) + + core_result = runner.invoke( + test_cli.app, + ["run", "--provider", "aws", "--suite", "storage", "--dry-run", "--no-upload"], + ) + vm_result = runner.invoke( + test_cli.app, + [ + "run", + "--provider", + "aws", + "--suite", + "storage", + "--capability", + "vm", + "--dry-run", + "--no-upload", + ], + ) + + assert core_result.exit_code == 0, core_result.output + assert "Capability: core" in core_result.stdout + assert "[RUN] CoreCheck" in core_result.stdout + assert "[SKIP] VmCheck" in core_result.stdout + assert vm_result.exit_code == 0, vm_result.output + assert "Capability: vm" in vm_result.stdout + assert "[RUN] CoreCheck" in vm_result.stdout + assert "[RUN] VmCheck" in vm_result.stdout + + def test_unknown_option_before_separator_is_rejected(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: """Stale flags like `--platform` fail before they can be forwarded to pytest.""" config = _write_config(tmp_path) From 8194627a1f966705d4e0fd661903f301d0682ddc Mon Sep 17 00:00:00 2001 From: Alexandre Begnoche Date: Fri, 24 Jul 2026 12:46:50 -0400 Subject: [PATCH 21/34] fix(reporting): upload complete catalog envelope Signed-off-by: Alexandre Begnoche --- isvctl/src/isvctl/cli/test.py | 9 ++--- isvctl/src/isvctl/reporting.py | 16 ++++---- isvctl/tests/test_reporting.py | 48 ++++++++++++++++++++++++ isvctl/tests/test_test_cli_labels.py | 43 +++++++++++++++++++++ isvreporter/src/isvreporter/client.py | 10 ++--- isvreporter/src/isvreporter/main.py | 3 ++ isvreporter/tests/test_catalog_upload.py | 37 +++++++++++++++++- 7 files changed, 147 insertions(+), 19 deletions(-) diff --git a/isvctl/src/isvctl/cli/test.py b/isvctl/src/isvctl/cli/test.py index e598302b4..215ba8430 100644 --- a/isvctl/src/isvctl/cli/test.py +++ b/isvctl/src/isvctl/cli/test.py @@ -506,8 +506,7 @@ def run( # Build test catalog early so it runs inside the TeeWriter context # (avoids logging errors from stale stream references after the log file closes) - catalog_entries: list[dict] | None = None - catalog_version: str | None = None + test_catalog_document: dict[str, Any] | None = None # Always capture output to log file while still displaying (like `tee`) with open(log_file_path, "w") as log_file: @@ -528,9 +527,10 @@ def run( try: catalog_entries = build_catalog() catalog_version = get_catalog_version() + test_catalog_document = catalog_document(catalog_entries, catalog_version) print_progress(f"Built test catalog: {len(catalog_entries)} tests (version: {catalog_version})") catalog_path = output_dir / "test_catalog.json" - catalog_path.write_text(json.dumps(catalog_document(catalog_entries, catalog_version), indent=2)) + catalog_path.write_text(json.dumps(test_catalog_document, indent=2)) print_progress(f" Saved test catalog to: {catalog_path}") except Exception as e: logger.warning("Failed to build test catalog: %s", e) @@ -558,8 +558,7 @@ def run( junit_xml=junit_path if junit_path.exists() else None, log_file=log_file_path if log_file_path.exists() else None, isv_software_version=isv_software_version, - catalog_entries=catalog_entries, - catalog_version=catalog_version, + catalog_document=test_catalog_document, ): print_progress(typer.style("[OK]", fg=typer.colors.GREEN) + " Test results uploaded successfully") else: diff --git a/isvctl/src/isvctl/reporting.py b/isvctl/src/isvctl/reporting.py index 6480b5f3b..7fadd9302 100644 --- a/isvctl/src/isvctl/reporting.py +++ b/isvctl/src/isvctl/reporting.py @@ -23,6 +23,7 @@ import os from datetime import UTC, datetime from pathlib import Path +from typing import Any from isvctl.redaction import redact_text @@ -144,8 +145,7 @@ def update_test_run( junit_xml: Path | None = None, log_content: str | None = None, isv_software_version: str | None = None, - catalog_entries: list[dict] | None = None, - catalog_version: str | None = None, + catalog_document: dict[str, Any] | None = None, ) -> bool: """Update a test run in ISV Lab Service. @@ -158,8 +158,7 @@ def update_test_run( junit_xml: Path to JUnit XML file (optional) log_content: Direct log content string (optional, alternative to log_file) isv_software_version: ISV software stack version (opaque string from ISV) - catalog_entries: Test catalog entries for coverage tracking (optional) - catalog_version: Test suite version for the catalog (optional) + catalog_document: Complete test catalog document for coverage tracking (optional) Returns: True if successful, False otherwise @@ -209,15 +208,18 @@ def update_test_run( jwt_token = get_jwt_token(ssa_issuer, client_id, client_secret) # Upload test catalog for coverage tracking (if provided) - if catalog_entries and catalog_version: + if catalog_document: try: from isvreporter.client import upload_test_catalog as client_upload_catalog client_upload_catalog( endpoint=endpoint, jwt_token=jwt_token, - isv_test_version=catalog_version, - entries=catalog_entries, + isv_test_version=catalog_document["isvTestVersion"], + entries=catalog_document["entries"], + schema_version=catalog_document["schemaVersion"], + platforms=catalog_document["platforms"], + suites=catalog_document["suites"], ) except SystemExit: logger.warning("Failed to upload test catalog to ISV Lab Service") diff --git a/isvctl/tests/test_reporting.py b/isvctl/tests/test_reporting.py index 627254643..4b6cd6810 100644 --- a/isvctl/tests/test_reporting.py +++ b/isvctl/tests/test_reporting.py @@ -22,6 +22,7 @@ check_upload_credentials, get_environment_config, get_isv_test_version, + update_test_run, ) @@ -124,3 +125,50 @@ def test_returns_none_on_import_error(self) -> None: result = get_isv_test_version() # Result depends on whether __version__ is available assert result is None or isinstance(result, str) + + +class TestUpdateTestRun: + """Tests for result and catalog upload orchestration.""" + + @patch("isvreporter.client.update_test_run") + @patch("isvreporter.client.upload_test_catalog") + @patch("isvreporter.auth.get_jwt_token", return_value="jwt-token") + @patch( + "isvctl.reporting.get_environment_config", return_value=("https://api.example.com", "https://ssa.example.com") + ) + @patch("isvctl.reporting.check_upload_credentials", return_value=(True, "client-id", "client-secret")) + def test_forwards_complete_catalog_document( + self, + _mock_credentials: MagicMock, + _mock_environment: MagicMock, + _mock_token: MagicMock, + mock_upload_catalog: MagicMock, + _mock_update_run: MagicMock, + ) -> None: + """Automatic result upload preserves all catalog envelope metadata.""" + document = { + "schemaVersion": 2, + "isvTestVersion": "1.2.3", + "platforms": ["kubernetes", "vm"], + "suites": ["storage", "iam"], + "entries": [{"name": "TestA"}], + } + + result = update_test_run( + lab_id=7, + test_run_id="run-123", + success=True, + start_time="2026-07-24T12:00:00Z", + catalog_document=document, + ) + + assert result is True + mock_upload_catalog.assert_called_once_with( + endpoint="https://api.example.com", + jwt_token="jwt-token", + isv_test_version="1.2.3", + entries=[{"name": "TestA"}], + schema_version=2, + platforms=["kubernetes", "vm"], + suites=["storage", "iam"], + ) diff --git a/isvctl/tests/test_test_cli_labels.py b/isvctl/tests/test_test_cli_labels.py index ad34e315d..c630debb5 100644 --- a/isvctl/tests/test_test_cli_labels.py +++ b/isvctl/tests/test_test_cli_labels.py @@ -127,6 +127,49 @@ def test_test_run_forwards_label_filters(monkeypatch: pytest.MonkeyPatch, tmp_pa assert _FakeOrchestrator.captured["include_labels"] == ["gpu", "slow"] +def test_test_run_uploads_the_complete_catalog_document( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """The automatic result path forwards the same complete catalog it saves.""" + config = _write_config(tmp_path) + output_dir = tmp_path / "_output" + output_dir.mkdir() + document = { + "schemaVersion": 2, + "isvTestVersion": "1.2.3", + "platforms": ["kubernetes", "vm"], + "suites": ["storage"], + "entries": [{"name": "TestA"}], + } + captured: dict[str, Any] = {} + + monkeypatch.setattr(test_cli, "Orchestrator", _FakeOrchestrator) + monkeypatch.setattr(test_cli, "get_output_dir", lambda: output_dir) + monkeypatch.setattr(test_cli, "check_upload_credentials", lambda: (True, "client-id", "client-secret")) + monkeypatch.setattr( + test_cli, + "get_environment_config", + lambda: ("https://api.example.com", "https://ssa.example.com"), + ) + monkeypatch.setattr(test_cli, "create_test_run", lambda **_kwargs: "run-123") + monkeypatch.setattr(test_cli, "build_catalog", lambda: document["entries"]) + monkeypatch.setattr(test_cli, "get_catalog_version", lambda: document["isvTestVersion"]) + monkeypatch.setattr(test_cli, "catalog_document", lambda _entries, _version: document) + + def capture_update(**kwargs: Any) -> bool: + captured.update(kwargs) + return True + + monkeypatch.setattr(test_cli, "update_test_run", capture_update) + + result = runner.invoke(test_cli.app, ["run", "-f", str(config), "--lab-id", "7"]) + + assert result.exit_code == 0, result.output + assert captured["catalog_document"] == document + assert json.loads((output_dir / "test_catalog.json").read_text()) == document + + def test_short_l_flag_binds_to_label_not_lab_id(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: """`-l` is the short flag for `--label`, not the legacy `--lab-id`. diff --git a/isvreporter/src/isvreporter/client.py b/isvreporter/src/isvreporter/client.py index 869ef33e2..82cc0ec6d 100644 --- a/isvreporter/src/isvreporter/client.py +++ b/isvreporter/src/isvreporter/client.py @@ -282,9 +282,9 @@ def upload_test_catalog( isv_test_version: str, entries: list[dict[str, Any]], *, - schema_version: int = 2, - platforms: list[str] | None = None, - suites: list[str] | None = None, + schema_version: int, + platforms: list[str], + suites: list[str], ) -> bool: """Upload test catalog for a suite version (idempotent per version). @@ -322,8 +322,8 @@ def upload_test_catalog( payload = { "schemaVersion": schema_version, "isvTestVersion": isv_test_version, - "platforms": platforms or [], - "suites": suites or [], + "platforms": platforms, + "suites": suites, "entries": [ { "name": e["name"], diff --git a/isvreporter/src/isvreporter/main.py b/isvreporter/src/isvreporter/main.py index d89f7dbc1..05f6ac4a1 100644 --- a/isvreporter/src/isvreporter/main.py +++ b/isvreporter/src/isvreporter/main.py @@ -300,6 +300,9 @@ def update( jwt_token=jwt_token, isv_test_version=catalog_version, entries=catalog_entries, + schema_version=catalog_data.get("schemaVersion", 1), + platforms=catalog_data.get("platforms", []), + suites=catalog_data.get("suites", []), ) except FileNotFoundError: typer.echo(f"Warning: Test catalog file not found: {test_catalog}", err=True) diff --git a/isvreporter/tests/test_catalog_upload.py b/isvreporter/tests/test_catalog_upload.py index 80fb9d5ff..ab3f771fd 100644 --- a/isvreporter/tests/test_catalog_upload.py +++ b/isvreporter/tests/test_catalog_upload.py @@ -20,12 +20,24 @@ from unittest.mock import MagicMock, patch from urllib.error import HTTPError, URLError +import pytest + from isvreporter.client import upload_test_catalog class TestUploadTestCatalog: """Tests for upload_test_catalog function.""" + def test_requires_the_complete_catalog_envelope(self) -> None: + """Callers cannot silently upload a v2 catalog without its axes.""" + with pytest.raises(TypeError): + upload_test_catalog( + endpoint="https://api.example.com", + jwt_token="test-token", + isv_test_version="1.2.3", + entries=[{"name": "TestA"}], + ) + @patch("isvreporter.client.urlopen") def test_successful_upload(self, mock_urlopen: MagicMock) -> None: """Test successful catalog upload returns True.""" @@ -69,6 +81,9 @@ def test_successful_upload(self, mock_urlopen: MagicMock) -> None: jwt_token="test-token", isv_test_version="1.2.3", entries=entries, + schema_version=2, + platforms=["kubernetes", "vm"], + suites=["storage"], ) assert result is True @@ -82,8 +97,8 @@ def test_successful_upload(self, mock_urlopen: MagicMock) -> None: payload = json.loads(request.data.decode()) assert payload["isvTestVersion"] == "1.2.3" assert payload["schemaVersion"] == 2 - assert payload["platforms"] == [] - assert payload["suites"] == [] + assert payload["platforms"] == ["kubernetes", "vm"] + assert payload["suites"] == ["storage"] assert len(payload["entries"]) == 2 assert payload["entries"][0]["name"] == "TestA" assert payload["entries"][0]["labels"] == ["k8s"] @@ -106,6 +121,9 @@ def test_skips_upload_when_version_exists(self, mock_urlopen: MagicMock) -> None jwt_token="test-token", isv_test_version="1.2.3", entries=[{"name": "TestA"}], + schema_version=2, + platforms=["kubernetes"], + suites=["storage"], ) assert result is True @@ -127,6 +145,9 @@ def test_conflict_returns_true(self, mock_urlopen: MagicMock) -> None: jwt_token="test-token", isv_test_version="1.2.3", entries=[{"name": "TestA"}], + schema_version=2, + platforms=["kubernetes"], + suites=["storage"], ) assert result is True @@ -147,6 +168,9 @@ def test_server_error_returns_false(self, mock_urlopen: MagicMock) -> None: jwt_token="test-token", isv_test_version="1.2.3", entries=[{"name": "TestA"}], + schema_version=2, + platforms=["kubernetes"], + suites=["storage"], ) assert result is False @@ -161,6 +185,9 @@ def test_connection_error_returns_false(self, mock_urlopen: MagicMock) -> None: jwt_token="test-token", isv_test_version="1.2.3", entries=[{"name": "TestA"}], + schema_version=2, + platforms=["kubernetes"], + suites=["storage"], ) assert result is False @@ -181,6 +208,9 @@ def test_empty_optional_fields_use_defaults(self, mock_urlopen: MagicMock) -> No jwt_token="test-token", isv_test_version="1.0.0", entries=entries, + schema_version=2, + platforms=["kubernetes"], + suites=["storage"], ) call_args = mock_urlopen.call_args @@ -241,6 +271,9 @@ def test_markers_field_is_not_forwarded(self, mock_urlopen: MagicMock) -> None: jwt_token="test-token", isv_test_version="1.0.0", entries=[{"name": "TestA", "labels": ["gpu"], "markers": ["gpu"]}], + schema_version=2, + platforms=["kubernetes"], + suites=["storage"], ) request = mock_urlopen.call_args[0][0] From 2894d5cacac6f0d7c0ff455779bd5939d19deceb Mon Sep 17 00:00:00 2001 From: Alexandre Begnoche Date: Fri, 24 Jul 2026 13:17:30 -0400 Subject: [PATCH 22/34] fix(selection): compose suite and label filters Signed-off-by: Alexandre Begnoche --- isvctl/src/isvctl/cli/test.py | 24 +++++++++--- isvctl/tests/test_test_cli_labels.py | 58 ++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 5 deletions(-) diff --git a/isvctl/src/isvctl/cli/test.py b/isvctl/src/isvctl/cli/test.py index 215ba8430..d4a689557 100644 --- a/isvctl/src/isvctl/cli/test.py +++ b/isvctl/src/isvctl/cli/test.py @@ -117,11 +117,18 @@ def _junitxml_for_discovered_config(junitxml: Path, match: ProviderConfigMatch, return junitxml.with_name(f"{junitxml.stem}-{match.config_path.stem}{junitxml.suffix}") -def _human_readable_dry_run(config: RunConfig, capability: str | None) -> str: +def _human_readable_dry_run( + config: RunConfig, + capability: str | None, + include_labels: list[str] | None = None, + exclude_labels: list[str] | None = None, +) -> str: """Render the validation requirement plan without executing lifecycle steps.""" platform = config.tests.platform if config.tests and config.tests.platform else None suite_type = f"platform ({platform})" if platform else "plain" context = "not filtered" if capability is None else capability + selected_labels = set(include_labels or []) + rejected_labels = set(exclude_labels or []) validations = config.tests.validations if config.tests else {} entries = parse_validations(validations) @@ -131,10 +138,19 @@ def _human_readable_dry_run(config: RunConfig, capability: str | None) -> str: f" Capability: {context}", f" Checks: {len(entries)}", ] + if selected_labels: + lines.append(f" Labels: {', '.join(sorted(selected_labels))} (all required)") + if rejected_labels: + lines.append(f" Excluded labels: {', '.join(sorted(rejected_labels))}") + for entry in entries: if capability is not None and not requirements_satisfied(entry.requires, capability): requirement = ", ".join(entry.requires) lines.append(f" [SKIP] {entry.name}: requires {requirement} (context: {capability})") + elif missing_labels := sorted(selected_labels.difference(entry.labels)): + lines.append(f" [SKIP] {entry.name}: does not match all selected labels: {', '.join(missing_labels)}") + elif matched_labels := sorted(rejected_labels.intersection(entry.labels)): + lines.append(f" [SKIP] {entry.name}: excluded by label: {', '.join(matched_labels)}") elif entry.requires: lines.append(f" [RUN] {entry.name}: requires {', '.join(entry.requires)}") else: @@ -300,6 +316,7 @@ def run( Examples: isvctl test run --provider aws --suite k8s + isvctl test run --provider aws --suite storage --label min_req isvctl test run --provider aws --label network isvctl test run -f lab.yaml -f commands.yaml -f suites/k8s.yaml isvctl test run -f config.yaml --set context.node_count=8 @@ -323,9 +340,6 @@ def run( if config_files: print_error("--suite cannot be combined with --config/-f.") raise typer.Exit(code=1) - if labels: - print_error("--suite cannot be combined with --label/-l; use labels after -- with pytest selection.") - raise typer.Exit(code=1) try: selected_suite = resolve_suite(provider, suite, configs_root=CONFIGS_ROOT) except SuiteResolutionError as exc: @@ -440,7 +454,7 @@ def run( raise typer.Exit(code=1) if dry_run: - typer.echo(_human_readable_dry_run(config, capability_context)) + typer.echo(_human_readable_dry_run(config, capability_context, labels, exclude_labels)) if extra_pytest_args: print_progress(f"\n--- Extra pytest args ---\n{extra_pytest_args}") return diff --git a/isvctl/tests/test_test_cli_labels.py b/isvctl/tests/test_test_cli_labels.py index c630debb5..28274fd90 100644 --- a/isvctl/tests/test_test_cli_labels.py +++ b/isvctl/tests/test_test_cli_labels.py @@ -377,6 +377,64 @@ def test_plain_suite_without_capability_defaults_to_core( assert "[RUN] VmCheck" in vm_result.stdout +def test_suite_and_label_filters_compose( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """A suite selects one lifecycle while labels narrow its checks.""" + configs_root = tmp_path / "configs" + suite_path = configs_root / "suites" / "storage.yaml" + suite_path.parent.mkdir(parents=True) + suite_path.write_text( + """\ +tests: + validations: + storage: + checks: + FastCheck: + test_id: "N/A" + labels: ["storage"] + SlowCheck: + test_id: "N/A" + labels: ["storage", "slow"] + DestructiveSlowCheck: + test_id: "N/A" + labels: ["storage", "slow", "destructive"] +""", + encoding="utf-8", + ) + _write_provider_config(configs_root, "aws", "storage.yaml", "storage.yaml", "storage") + _FakeOrchestrator.captured = {} + _FakeOrchestrator.calls = [] + monkeypatch.setattr(test_cli, "CONFIGS_ROOT", configs_root) + monkeypatch.setattr(test_cli, "Orchestrator", _FakeOrchestrator) + + command = [ + "run", + "--provider", + "aws", + "--suite", + "storage", + "--label", + "slow", + "--exclude-label", + "destructive", + "--no-upload", + ] + run_result = runner.invoke(test_cli.app, command) + dry_run_result = runner.invoke(test_cli.app, [*command, "--dry-run"]) + + assert run_result.exit_code == 0, run_result.output + assert _FakeOrchestrator.captured["include_labels"] == ["slow"] + assert _FakeOrchestrator.captured["exclude_labels"] == ["destructive"] + assert dry_run_result.exit_code == 0, dry_run_result.output + assert "Labels: slow (all required)" in dry_run_result.stdout + assert "Excluded labels: destructive" in dry_run_result.stdout + assert "[SKIP] FastCheck: does not match all selected labels: slow" in dry_run_result.stdout + assert "[RUN] SlowCheck" in dry_run_result.stdout + assert "[SKIP] DestructiveSlowCheck: excluded by label: destructive" in dry_run_result.stdout + + def test_unknown_option_before_separator_is_rejected(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: """Stale flags like `--platform` fail before they can be forwarded to pytest.""" config = _write_config(tmp_path) From 80fb1dadca660613b48566fac2d6179f8fa53a58 Mon Sep 17 00:00:00 2001 From: Alexandre Begnoche Date: Fri, 24 Jul 2026 14:33:02 -0400 Subject: [PATCH 23/34] fix(selection): gate teardown steps by capability Capability gating skips a step when it carries an explicit `requires:` or when every validation bound to it is requirement-filtered. Teardown steps have no bound validations, so inference cannot reach them: under the core-by-default `--suite` path they kept running against fixtures that were never created. Only storage carried explicit gates; three other suites had the same hole. - image-registry: teardown deleted both the core upload_image resources and the vm-only launch_instance ones, so it referenced a skipped step and was abandoned whole with "missing step reference" - leaking the image, disks and bucket. Split by owner into teardown_instance (requires: [vm]) and teardown_image (core); the scripts already guard each resource independently, so no script change was needed. Rebind the suite's teardown check to teardown_image, the half that always runs. - observability: gate launch_host/teardown_host on [vm, bare_metal] so a core run no longer launches the reference metal host only to skip every check that needed it. - security: gate topology_block_teardown/capacity_teardown on [vm, bare_metal], mirroring the gate inferred on the test-phase steps that allocate those resources. StorageCapacityTelemetryCheck moves from core to [vm, bare_metal]. It probes the volumes attached to a running instance via the same script and --instance-id as its StoragePerformanceTelemetryCheck twin, which already declared that gate; with no instance it reports "no volumes attached", so it could never pass as a core check. Needs a catalog re-upload. Add test_capability_step_gating.py: instead of three point tests it walks every plain-suite provider config across every context and asserts no surviving step reads a gated-off step's output without a `default(...)`, so new suites inherit the guarantee. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Alexandre Begnoche --- .../providers/aws/config/image-registry.yaml | 31 +++-- .../providers/aws/config/observability.yaml | 7 ++ .../providers/aws/config/security.yaml | 5 + .../my-isv/config/image-registry.yaml | 31 +++-- isvctl/configs/suites/README.md | 3 +- isvctl/configs/suites/image-registry.yaml | 5 +- isvctl/configs/suites/observability.yaml | 6 +- isvctl/tests/test_capability_step_gating.py | 108 ++++++++++++++++++ 8 files changed, 179 insertions(+), 17 deletions(-) create mode 100644 isvctl/tests/test_capability_step_gating.py diff --git a/isvctl/configs/providers/aws/config/image-registry.yaml b/isvctl/configs/providers/aws/config/image-registry.yaml index b6b5dcf9a..1092d4e35 100644 --- a/isvctl/configs/providers/aws/config/image-registry.yaml +++ b/isvctl/configs/providers/aws/config/image-registry.yaml @@ -102,18 +102,18 @@ commands: timeout: 120 # Step 5: Cleanup all resources - - name: teardown + # Teardown is split by owner so each half can be gated independently. + # This half cleans up what launch_instance created, so it carries the + # same `requires: [vm]` gate and is skipped whole in other contexts. + # Declared first: terminate the instance before deregistering the AMI it + # was launched from. + - name: teardown_instance phase: teardown command: "python3 ../scripts/image-registry/teardown.py" + requires: [vm] args: - "--instance-id" - "{{steps.launch_instance.instance_id}}" - - "--ami-id" - - "{{steps.upload_image.image_id}}" - - "--snapshot-ids" - - "{{steps.upload_image.disk_ids | join(',')}}" - - "--bucket-name" - - "{{steps.upload_image.storage_bucket}}" - "--key-name" - "{{steps.launch_instance.key_name}}" - "--security-group-id" @@ -125,6 +125,23 @@ commands: - "{{teardown_flag}}" timeout: 1800 + # This half cleans up what the core upload_image step created, so it is + # ungated and runs in every capability context. + - name: teardown_image + phase: teardown + command: "python3 ../scripts/image-registry/teardown.py" + args: + - "--ami-id" + - "{{steps.upload_image.image_id}}" + - "--snapshot-ids" + - "{{steps.upload_image.disk_ids | join(',')}}" + - "--bucket-name" + - "{{steps.upload_image.storage_bucket}}" + - "--region" + - "{{region}}" + - "{{teardown_flag}}" + timeout: 1800 + tests: cluster_name: "aws-image-registry-validation" description: "AWS VM Import ISO/VMDK validation tests" diff --git a/isvctl/configs/providers/aws/config/observability.yaml b/isvctl/configs/providers/aws/config/observability.yaml index 8573f3bb7..2ab9ae9f8 100644 --- a/isvctl/configs/providers/aws/config/observability.yaml +++ b/isvctl/configs/providers/aws/config/observability.yaml @@ -61,9 +61,13 @@ commands: - "isv-observability" timeout: 300 + # The host fixture only serves the vm/bare_metal telemetry checks; no + # bound validation can infer that (nothing validates the launch itself), + # so the gate is explicit. Core runs skip the metal instance entirely. - name: launch_host phase: setup command: "python3 ../scripts/bare_metal/launch_instance.py" + requires: [vm, bare_metal] args: - "--name" - "isv-observability-host" @@ -340,9 +344,12 @@ commands: - "switch_kernel_logs" timeout: 60 + # Same gate as launch_host: both sides of the fixture move together, so + # a core run never tears down a host it never launched. - name: teardown_host phase: teardown command: "python3 ../scripts/bare_metal/teardown.py" + requires: [vm, bare_metal] args: - "--instance-id" - "{{steps.launch_host.instance_id}}" diff --git a/isvctl/configs/providers/aws/config/security.yaml b/isvctl/configs/providers/aws/config/security.yaml index 901fd5057..c773d42c7 100644 --- a/isvctl/configs/providers/aws/config/security.yaml +++ b/isvctl/configs/providers/aws/config/security.yaml @@ -334,9 +334,13 @@ commands: # its finally block; this teardown step reclaims resources left behind # when AWS_CAPACITY_SKIP_DESTROY=true (a no-op while that flag is set, so # a later standalone `--phase teardown` without it does the cleanup). + # Mirrors the gate inferred on topology_block_atomic_allocation (its only + # check is vm/bare_metal): a core run allocates no block, so it must not + # try to release one. - name: topology_block_teardown phase: teardown command: "python3 ../scripts/capacity/topology_block_atomic_allocation.py" + requires: [vm, bare_metal] requires_available_validations: - CapacityTopologyBlockAtomicAllocationCheck args: @@ -365,6 +369,7 @@ commands: - name: capacity_teardown phase: teardown command: "python3 ../scripts/capacity/reservation_grouping.py" + requires: [vm, bare_metal] requires_available_validations: - CapacityReservationGroupingCheck args: diff --git a/isvctl/configs/providers/my-isv/config/image-registry.yaml b/isvctl/configs/providers/my-isv/config/image-registry.yaml index 98594a3b9..371000696 100644 --- a/isvctl/configs/providers/my-isv/config/image-registry.yaml +++ b/isvctl/configs/providers/my-isv/config/image-registry.yaml @@ -111,18 +111,18 @@ commands: - "{{region}}" timeout: 60 - - name: teardown + # Teardown is split by owner so each half can be gated independently. + # This half cleans up what launch_instance created, so it carries the + # same `requires: [vm]` gate and is skipped whole in other contexts. + # Declared first: terminate the instance before deregistering the image + # it was launched from. + - name: teardown_instance phase: teardown command: "python ../scripts/image-registry/teardown.py" + requires: [vm] args: - "--instance-id" - "{{steps.launch_instance.instance_id}}" - - "--image-id" - - "{{steps.upload_image.image_id}}" - - "--disk-ids" - - "{{steps.upload_image.disk_ids | join(',')}}" - - "--bucket-name" - - "{{steps.upload_image.storage_bucket}}" - "--key-name" - "{{steps.launch_instance.key_name}}" - "--security-group-id" @@ -134,6 +134,23 @@ commands: - "{{teardown_flag}}" timeout: 60 + # This half cleans up what the core upload_image step created, so it is + # ungated and runs in every capability context. + - name: teardown_image + phase: teardown + command: "python ../scripts/image-registry/teardown.py" + args: + - "--image-id" + - "{{steps.upload_image.image_id}}" + - "--disk-ids" + - "{{steps.upload_image.disk_ids | join(',')}}" + - "--bucket-name" + - "{{steps.upload_image.storage_bucket}}" + - "--region" + - "{{region}}" + - "{{teardown_flag}}" + timeout: 60 + tests: cluster_name: "my-isv-image-registry-validation" description: "my-isv image registry lifecycle validation (generic stubs, dummy overrides)" diff --git a/isvctl/configs/suites/README.md b/isvctl/configs/suites/README.md index 09c19cafe..1ee15bf11 100644 --- a/isvctl/configs/suites/README.md +++ b/isvctl/configs/suites/README.md @@ -193,7 +193,8 @@ Validations use `sinfo`/`srun` directly: partitions, GPU allocation, job schedul | `crud_install_config` | test | `providers/my-isv/scripts/image-registry/crud_install_config.py` | `config_id`, `config_name`, `operations` | | `install_image_bm` | test | `providers/my-isv/scripts/image-registry/install_image_bm.py` | `instance_id`, `image_id`, `instance_state` | | `install_config_bm` | test | `providers/my-isv/scripts/image-registry/install_config_bm.py` | `instance_id`, `config_id`, `instance_state`, `state` | -| `teardown` | teardown | `providers/my-isv/scripts/image-registry/teardown.py` | `resources_deleted`, `message` | +| `teardown_instance` | teardown | `providers/my-isv/scripts/image-registry/teardown.py` | `resources_deleted`, `message` (instance, key pair, security group, instance profile — only under `vm`) | +| `teardown_image` | teardown | `providers/my-isv/scripts/image-registry/teardown.py` | `resources_deleted`, `message` (image, disks, bucket — always) | ### Security (`security.yaml`) diff --git a/isvctl/configs/suites/image-registry.yaml b/isvctl/configs/suites/image-registry.yaml index e2c6d263c..07a184dc9 100644 --- a/isvctl/configs/suites/image-registry.yaml +++ b/isvctl/configs/suites/image-registry.yaml @@ -166,8 +166,11 @@ tests: requires: [bare_metal] expected_state: "running" + # Bound to the image half of teardown: it is ungated, so this stays a core + # check. The instance half (teardown_instance) only runs under vm and owns + # nothing an ISV must prove, so it carries no check of its own. teardown_checks: - step: teardown + step: teardown_image checks: StepSuccessCheck: test_id: "N/A" diff --git a/isvctl/configs/suites/observability.yaml b/isvctl/configs/suites/observability.yaml index e67d0f615..650339130 100644 --- a/isvctl/configs/suites/observability.yaml +++ b/isvctl/configs/suites/observability.yaml @@ -108,7 +108,11 @@ tests: StorageCapacityTelemetryCheck: test_id: "TELEM05-01" labels: ["min_req", "observability"] - requires: [] + # Probes the volumes attached to a running instance, exactly like its + # StoragePerformanceTelemetryCheck twin (same script, different + # --aspect). Without one it reports "no volumes attached", so it can + # never pass as a core check. + requires: [vm, bare_metal] step: storage_capacity_telemetry storage_performance_telemetry: diff --git a/isvctl/tests/test_capability_step_gating.py b/isvctl/tests/test_capability_step_gating.py new file mode 100644 index 000000000..2b76614fb --- /dev/null +++ b/isvctl/tests/test_capability_step_gating.py @@ -0,0 +1,108 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Capability gating must leave every provider config internally consistent. + +Running a plain suite under one capability context skips two kinds of step: +those carrying an explicit ``requires:``, and those whose bound validations are +all requirement-filtered. Either way the step produces no output, so any step +that survives the gate must not depend on a skipped step's output without an +explicit ``default(...)`` - otherwise the orchestrator raises +``MissingStepRefError`` and silently abandons the cleanup that step owned. + +This walks every provider config rather than the three that regressed, so a new +suite inherits the guarantee instead of re-discovering it. +""" + +import re +from pathlib import Path +from typing import Any + +import pytest +from isvtest.core.resolution import ( + DECLARABLE_CAPABILITIES, + ValidationEntry, + parse_validations, + requirements_satisfied, +) + +from isvctl.cli.test import CORE_REQUIREMENT_CONTEXT +from isvctl.config.merger import merge_yaml_files +from isvctl.config.schema import RunConfig + +CONFIGS_ROOT = Path(__file__).resolve().parents[1] / "configs" +PROVIDERS = ("aws", "my-isv") +# Every context a plain suite can be run under, including the core-only default +# that `--suite NAME` (no `--capability`) selects. +CONTEXTS = (CORE_REQUIREMENT_CONTEXT, *sorted(DECLARABLE_CAPABILITIES)) +STEP_REFERENCE = re.compile(r"steps\.([A-Za-z0-9_]+)") + + +def _plain_suite_configs() -> list[tuple[str, Path]]: + """Return (provider, config path) for every plain-suite provider config.""" + configs: list[tuple[str, Path]] = [] + for provider in PROVIDERS: + for path in sorted((CONFIGS_ROOT / "providers" / provider / "config").glob("*.yaml")): + config = RunConfig.model_validate(merge_yaml_files([str(path)])) + # Platform suites carry no `requires:` on their checks, so nothing + # is ever gated inside them. + if config.tests and config.tests.platform: + continue + configs.append((provider, path)) + return configs + + +def _gated_step_names(steps: list[Any], entries: list[ValidationEntry], context: str) -> set[str]: + """Return the steps `_apply_capability_step_gates` would skip in a context.""" + gated: set[str] = set() + for step in steps: + if step.requires and not requirements_satisfied(step.requires, context): + gated.add(step.name) + continue + bound = [entry for entry in entries if entry.step == step.name] + if bound and all(not requirements_satisfied(entry.requires, context) for entry in bound): + gated.add(step.name) + return gated + + +def _unguarded_references(value: Any) -> set[str]: + """Return step names referenced without a `default(...)` fallback.""" + referenced: set[str] = set() + if isinstance(value, str): + if "default(" in value: + return referenced + return set(STEP_REFERENCE.findall(value)) + if isinstance(value, dict): + for item in value.values(): + referenced |= _unguarded_references(item) + elif isinstance(value, list): + for item in value: + referenced |= _unguarded_references(item) + return referenced + + +@pytest.mark.parametrize(("provider", "config_path"), _plain_suite_configs(), ids=lambda v: getattr(v, "stem", v)) +@pytest.mark.parametrize("context", CONTEXTS) +def test_surviving_steps_never_depend_on_gated_steps(provider: str, config_path: Path, context: str) -> None: + """No step that survives capability gating may read a gated step's output.""" + config = RunConfig.model_validate(merge_yaml_files([str(config_path)])) + entries = parse_validations(config.tests.validations if config.tests else {}) + + violations: list[str] = [] + for platform_key in config.commands: + steps = config.get_steps(platform_key) + gated = _gated_step_names(steps, entries, context) + if not gated: + continue + for step in steps: + if step.name in gated: + continue + dangling = _unguarded_references(step.model_dump()) & gated + for missing in sorted(dangling): + violations.append(f"step '{step.name}' reads skipped step '{missing}'") + + assert not violations, ( + f"{provider}/{config_path.name} under context '{context}': " + + "; ".join(violations) + + ". Gate the step with `requires:` or give the reference a `default(...)`." + ) From 3735674b56e3ac6f875670fc20888ccc6d74575e Mon Sep 17 00:00:00 2001 From: Alexandre Begnoche Date: Fri, 24 Jul 2026 14:42:48 -0400 Subject: [PATCH 24/34] fix(my-isv): align the scaffold with the reference lifecycle my-isv is the folder ISVs copy, so a guarantee that holds only in aws/ teaches the wrong lifecycle. Three divergences, all in the capability-gated paths added on this branch. - storage: setup_cluster carried no `requires: [kubernetes]` and had no teardown_cluster at all, so the scaffold demonstrated acquiring a cluster and never releasing it - the opposite of the setup_cluster/teardown_cluster contract aws/ has carried since 607c121. Add both, with STORAGE_SKIP_TEARDOWN mirroring AWS_SKIP_TEARDOWN, and a teardown_cluster.py stub in the house style. A no-op teardown is a valid answer when the stub reuses a long-lived cluster; the step still has to exist so a standalone storage run cannot leak a cluster it provisioned. - kubernetes/slurm: no config existed, so `--suite kubernetes --provider my-isv` errored - and that is exactly what the frontend emits for its default provider. The scripts were already there; only the wiring was missing. Steps are overridden rather than inherited because the canonical suites point at ../providers/my-isv/scripts/*, which resolves wrong from a provider config directory. Neither joins `make demo-test`: both drive real kubectl/sinfo, so a dummy-success stub has nothing to return. Also fixes a pre-existing scaffold break found while verifying the above: `--suite storage --capability kubernetes` failed with "'cluster_name' is a required property". The step name auto-detects the platform-suite `cluster` schema, which demands cluster_name/node_count - inventory no storage check reads. Set `output_schema: generic` on both providers; scoping it to my-isv would have recreated the divergence this commit removes, and per the JSON contract discipline storage must not require cluster inventory of an ISV. test_storage_cleanup_steps_have_explicit_capability_gates encoded the divergence as `if provider == "aws":`; both providers now assert the same gates. Add coverage that every declarable capability resolves to a my-isv platform suite, so a UI-emitted `--suite ` cannot silently become an error again. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Alexandre Begnoche --- .../configs/providers/aws/config/storage.yaml | 5 ++ .../configs/providers/my-isv/config/k8s.yaml | 58 +++++++++++++++++++ .../providers/my-isv/config/slurm.yaml | 55 ++++++++++++++++++ .../providers/my-isv/config/storage.yaml | 25 ++++++++ .../scripts/storage/teardown_cluster.py | 56 ++++++++++++++++++ isvctl/configs/suites/README.md | 4 +- isvctl/tests/test_suite_resolution.py | 37 ++++++++++-- 7 files changed, 235 insertions(+), 5 deletions(-) create mode 100644 isvctl/configs/providers/my-isv/config/k8s.yaml create mode 100644 isvctl/configs/providers/my-isv/config/slurm.yaml create mode 100644 isvctl/configs/providers/my-isv/scripts/storage/teardown_cluster.py diff --git a/isvctl/configs/providers/aws/config/storage.yaml b/isvctl/configs/providers/aws/config/storage.yaml index 91d8911be..7cfab39df 100644 --- a/isvctl/configs/providers/aws/config/storage.yaml +++ b/isvctl/configs/providers/aws/config/storage.yaml @@ -63,6 +63,11 @@ commands: phase: setup command: "../scripts/eks/setup.sh" requires: [kubernetes] + # Same contract as my-isv's stub: kubeconfig + StorageClass names. The + # EKS script happens to emit full cluster inventory too, but storage + # must not require that of an ISV, so both providers validate the + # fixture against its own contract, not the platform `cluster` schema. + output_schema: generic timeout: 5400 env: TF_AUTO_APPROVE: "true" diff --git a/isvctl/configs/providers/my-isv/config/k8s.yaml b/isvctl/configs/providers/my-isv/config/k8s.yaml new file mode 100644 index 000000000..d1edf638d --- /dev/null +++ b/isvctl/configs/providers/my-isv/config/k8s.yaml @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# my-isv Kubernetes Platform Suite - Living Example +# +# This is the obligation attached to declaring the `kubernetes` capability. +# It imports the provider-agnostic Kubernetes validation contract and wires +# the lifecycle to the generic scripts in scripts/k8s/*. +# +# Usage: +# uv run isvctl test run --provider my-isv --suite kubernetes +# +# Unlike the other my-isv examples this one talks to a REAL cluster: the +# validations run kubectl directly, so there is nothing for a dummy-success +# stub to return. That is why it is not part of `make demo-test`. +# +# ACTION: point KUBECTL at your cluster (or make `kubectl` resolve to it), +# then replace scripts/k8s/setup.sh with your own provisioning if you want +# isvctl to create the cluster rather than reuse an existing one. +# +# Environment Variables: +# - KUBECTL: kubectl-compatible CLI prefix (defaults to kubectl, then microk8s kubectl) +# - NGC_API_KEY: NGC API key for NIM workloads + +import: ../../../suites/k8s.yaml + +version: "1.0" + +# Steps are overridden (not inherited) so the script paths resolve from this +# provider's config directory. Lists are replaced wholesale on merge. +commands: + kubernetes: + phases: ["setup", "test", "teardown"] + steps: + - name: setup + phase: setup + command: "../scripts/k8s/setup.sh" + timeout: 120 + + - name: teardown + phase: teardown + command: "../scripts/k8s/teardown.sh" + timeout: 30 + +tests: + description: "my-isv Kubernetes platform suite (real cluster required)" diff --git a/isvctl/configs/providers/my-isv/config/slurm.yaml b/isvctl/configs/providers/my-isv/config/slurm.yaml new file mode 100644 index 000000000..d65cc5e62 --- /dev/null +++ b/isvctl/configs/providers/my-isv/config/slurm.yaml @@ -0,0 +1,55 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# my-isv Slurm Platform Suite - Living Example +# +# This is the obligation attached to declaring the `slurm` capability. It +# imports the provider-agnostic Slurm validation contract and wires the +# lifecycle to the generic scripts in scripts/slurm/*. +# +# Usage: +# uv run isvctl test run --provider my-isv --suite slurm +# +# Unlike the other my-isv examples this one talks to a REAL cluster: the +# validations run sinfo/srun directly, so there is nothing for a dummy-success +# stub to return. That is why it is not part of `make demo-test`. +# +# ACTION: make the Slurm client commands (sinfo, scontrol, srun) resolve to +# your cluster, then replace scripts/slurm/setup.sh with your own +# provisioning if you want isvctl to create the cluster rather than reuse an +# existing one. + +import: ../../../suites/slurm.yaml + +version: "1.0" + +# Steps are overridden (not inherited) so the script paths resolve from this +# provider's config directory. Lists are replaced wholesale on merge. +commands: + slurm: + phases: ["setup", "test", "teardown"] + steps: + - name: setup + phase: setup + command: "../scripts/slurm/setup.sh" + timeout: 120 + + - name: teardown + phase: teardown + command: "../scripts/slurm/teardown.sh" + timeout: 30 + +tests: + description: "my-isv Slurm platform suite (real cluster required)" diff --git a/isvctl/configs/providers/my-isv/config/storage.yaml b/isvctl/configs/providers/my-isv/config/storage.yaml index 79421fc82..c5cdbb9a3 100644 --- a/isvctl/configs/providers/my-isv/config/storage.yaml +++ b/isvctl/configs/providers/my-isv/config/storage.yaml @@ -49,9 +49,19 @@ commands: storage: phases: ["setup", "test", "teardown"] steps: + # Only the kubernetes CSI checks need a cluster, so both sides of the + # fixture carry the same gate: a `--capability vm` run never provisions + # one. Acquire is idempotent - reuse an existing cluster if you have one. - name: setup_cluster phase: setup command: "python ../scripts/storage/setup_cluster.py" + requires: [kubernetes] + # The step name auto-detects the platform-suite `cluster` schema, which + # demands cluster_name/node_count. This fixture has its own, smaller + # contract - kubeconfig + StorageClass names, the only fields the + # k8s_storage checks read - so opt out rather than make every ISV + # report inventory no check consumes. + output_schema: generic timeout: 60 - name: launch_instance @@ -262,6 +272,20 @@ commands: - "--delete-security-group" timeout: 60 + # The other half of the setup_cluster fixture. Teardown may be a no-op if + # you reuse a long-lived cluster, but the step must exist so a standalone + # storage run does not leak a cluster it provisioned. STORAGE_SKIP_TEARDOWN + # keeps it alive for cheap reruns (mirrors AWS_SKIP_TEARDOWN in aws/). + - name: teardown_cluster + phase: teardown + command: "python ../scripts/storage/teardown_cluster.py" + requires: [kubernetes] + args: + - "--kubeconfig" + - "{{steps.setup_cluster.kubeconfig_path}}" + - "{{cluster_teardown_flag}}" + timeout: 60 + tests: cluster_name: "my-isv-storage-validation" description: "my-isv storage validation (block volume + high-speed stubs, dummy overrides)" @@ -271,3 +295,4 @@ tests: instance_type: "my-isv.standard.1x" volume_size_gib: "10" teardown_flag: "{{(env.BLOCK_STORAGE_SKIP_TEARDOWN == 'true') | ternary('--skip-destroy', '')}}" + cluster_teardown_flag: "{{(env.STORAGE_SKIP_TEARDOWN == 'true') | ternary('--skip-destroy', '')}}" diff --git a/isvctl/configs/providers/my-isv/scripts/storage/teardown_cluster.py b/isvctl/configs/providers/my-isv/scripts/storage/teardown_cluster.py new file mode 100644 index 000000000..019cdf76f --- /dev/null +++ b/isvctl/configs/providers/my-isv/scripts/storage/teardown_cluster.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Release the Kubernetes cluster that setup_cluster acquired.""" + +import argparse +import json +import os +import sys +from typing import Any + +DEMO_MODE = os.environ.get("ISVCTL_DEMO_MODE") == "1" + + +def main() -> int: + """Emit the provider-neutral cluster teardown contract.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--kubeconfig", default="", help="Kubeconfig emitted by setup_cluster") + parser.add_argument("--skip-destroy", action="store_true", help="Keep the cluster for cheap reruns") + args = parser.parse_args() + + result: dict[str, Any] = { + "success": False, + "platform": "kubernetes", + "resources_deleted": [], + "message": "", + } + + if args.skip_destroy: + result.update({"success": True, "skipped": True, "message": "Teardown skipped (--skip-destroy)"}) + print(json.dumps(result, indent=2)) + return 0 + + # TODO: Release whatever setup_cluster acquired. This is deliberately the + # mirror of that step: if setup_cluster created a cluster, destroy it here; + # if it reused a long-lived one, a no-op success is the correct answer. The + # step must still exist either way, so a standalone storage run never leaks + # a cluster it provisioned. + if DEMO_MODE: + result.update( + { + "success": True, + "resources_deleted": ["demo-cluster"], + "message": "Cluster released", + } + ) + else: + result["error"] = "Not implemented - release the cluster setup_cluster acquired" + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/suites/README.md b/isvctl/configs/suites/README.md index 1ee15bf11..6a15292fb 100644 --- a/isvctl/configs/suites/README.md +++ b/isvctl/configs/suites/README.md @@ -141,13 +141,15 @@ volume. The three test-phase steps all reuse that fixture. | Step | Phase | Script | Key JSON Fields | |------|-------|--------|-----------------| -| `launch_instance` | setup | `providers/my-isv/scripts/vm/launch_instance.py` | `instance_id`, `state`, `public_ip`, `key_file` (reuses VM script) | +| `setup_cluster` | setup | `providers/my-isv/scripts/storage/setup_cluster.py` | `kubeconfig_path`, `csi.{block,shared_fs,nfs}_storage_class` (only under `kubernetes`) | +| `launch_instance` | setup | `providers/my-isv/scripts/vm/launch_instance.py` | `instance_id`, `state`, `public_ip`, `key_file` (reuses VM script; only under `vm`/`bare_metal`) | | `create_volume` | setup | `providers/my-isv/scripts/storage/create_volume.py` | `volume_id`, `mount_point`, `sentinel_content`, `operations.{create,attach,format,mount,write_sentinel}` | | `snapshot_lifecycle` | test | `providers/my-isv/scripts/storage/snapshot_lifecycle.py` | `volume_id`, `snapshot_id`, `operations.{create_snapshot,restore_volume,verify_data}` (verify_data includes `content_matches`) | | `volume_resize` | test | `providers/my-isv/scripts/storage/volume_resize.py` | `volume_id`, `operations.{modify_volume,grow_partition,resize_filesystem,verify_size}` | | `volume_persistence` | test | `providers/my-isv/scripts/storage/volume_persistence.py` | `volume_id`, `operations.{stop,start,verify_attached,verify_data}` (verify_data includes `content_matches`) | | `teardown_volume` | teardown | `providers/my-isv/scripts/storage/teardown_volume.py` | `resources_deleted`, `message` | | `teardown` | teardown | `providers/my-isv/scripts/vm/teardown.py` | `resources_deleted`, `message` (reuses VM script) | +| `teardown_cluster` | teardown | `providers/my-isv/scripts/storage/teardown_cluster.py` | `resources_deleted`, `message` (only under `kubernetes`; may be a no-op if you reuse a cluster) | ### Kubernetes (`k8s.yaml`) diff --git a/isvctl/tests/test_suite_resolution.py b/isvctl/tests/test_suite_resolution.py index c5db1873e..534c85687 100644 --- a/isvctl/tests/test_suite_resolution.py +++ b/isvctl/tests/test_suite_resolution.py @@ -6,6 +6,7 @@ from pathlib import Path import pytest +from isvtest.core.resolution import DECLARABLE_CAPABILITIES from pydantic import ValidationError from isvctl.config.merger import merge_yaml_files @@ -88,13 +89,41 @@ def test_plain_suite_accepts_valid_requires() -> None: @pytest.mark.parametrize("provider", ["aws", "my-isv"]) def test_storage_cleanup_steps_have_explicit_capability_gates(provider: str) -> None: - """Destructive storage cleanup runs only in the context that owns its resources.""" + """Destructive storage cleanup runs only in the context that owns its resources. + + Both providers carry the same gates: my-isv is the scaffold ISVs copy, so a + reference-only guarantee would teach the wrong lifecycle. + """ config_path = CONFIGS_ROOT / "providers" / provider / "config" / "storage.yaml" config = RunConfig.model_validate(merge_yaml_files([str(config_path)])) steps = {step.name: step for step in config.get_steps("storage")} assert steps["teardown_volume"].requires == ["vm", "bare_metal"] assert steps["teardown"].requires == ["vm", "bare_metal"] - if provider == "aws": - assert steps["setup_cluster"].requires == ["kubernetes"] - assert steps["teardown_cluster"].requires == ["kubernetes"] + assert steps["setup_cluster"].requires == ["kubernetes"] + assert steps["teardown_cluster"].requires == ["kubernetes"] + + +@pytest.mark.parametrize("capability", sorted(DECLARABLE_CAPABILITIES)) +def test_my_isv_scaffold_covers_every_declarable_capability(capability: str) -> None: + """Every capability an ISV can declare has a my-isv platform suite to copy. + + The UI emits `--suite ` against the default provider, so a + missing scaffold config turns a documented command into an error. + """ + resolved = resolve_suite("my-isv", capability, configs_root=CONFIGS_ROOT) + assert resolved.platform == capability + + +@pytest.mark.parametrize("provider", ["aws", "my-isv"]) +def test_storage_cluster_fixture_uses_its_own_output_contract(provider: str) -> None: + """The storage cluster fixture is not held to the platform `cluster` schema. + + `setup_cluster` auto-detects that schema by name, and it demands + cluster_name/node_count - inventory no storage check reads. + """ + config_path = CONFIGS_ROOT / "providers" / provider / "config" / "storage.yaml" + config = RunConfig.model_validate(merge_yaml_files([str(config_path)])) + steps = {step.name: step for step in config.get_steps("storage")} + + assert steps["setup_cluster"].output_schema == "generic" From 1c0d84fd3e4f69b64cbc4c7b53a5f375f5f9d125 Mon Sep 17 00:00:00 2001 From: Alexandre Begnoche Date: Fri, 24 Jul 2026 14:55:16 -0400 Subject: [PATCH 25/34] fix(selection): resolve the capability context once, for every entry path The same config ran a different set of checks depending on how it was named: `--suite network` gave 26 checks (core default), `-f .../network.yaml` gave 35 (unfiltered), and `--provider X --label network` gave 35 as well. The label path is the one requirements.md #4 calls out by name - label selection must not run checks gated on a capability the ISV does not support - and it did. Resolve the context after config validation instead of inside the `--suite` branch, so one rule covers `--suite`, `-f` and `--label` discovery alike: a plain suite with no `--capability` runs its core checks. Platform suites declare no `requires:`, so they keep the unfiltered context. Label discovery inherits this through its recursive call, which closes #4 without special casing it. This drops the "no filtering" pseudo-context as a default. It modelled no real ISV situation - nobody runs on vm and kubernetes at once - so a plain suite now always carries exactly one context. Warn when `--capability` names something no check in the suite requires (`--suite iam --capability kubernetes`): the run proceeds with core checks, but a flag that silently does nothing reads as a typo. UI-composed commands are unaffected. `-f` used to hand the demos their gated checks for free, so the Makefile now names a capability per suite to keep exercising those stubs; without it `make demo-test` would quietly shrink by ~29 checks and stop covering the launch_instance / install_image_bm / install_config_bm scripts while still printing all-green. image-registry splits its gated checks between vm and bare_metal, so it runs once per context. Document the rule in configs/suites/README.md, which described neither `requires:` nor the selection grammar. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Alexandre Begnoche --- Makefile | 22 ++++- isvctl/configs/suites/README.md | 32 +++++++ isvctl/src/isvctl/cli/test.py | 47 ++++++++-- isvctl/tests/test_test_cli_labels.py | 128 +++++++++++++++++++++++++++ 4 files changed, 217 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 282ee4a62..165982e42 100644 --- a/Makefile +++ b/Makefile @@ -84,15 +84,23 @@ test: ## Run tests for all packages @echo "" @echo "✅ All tests passed!" +# run_demo,[,] - a plain suite with no capability runs only +# its core checks, so suites holding gated checks name one to exercise those +# stubs. Unlisted suites are core-only or platform suites. +DEMO_CAP_network := vm +DEMO_CAP_observability := vm +DEMO_CAP_security := vm +DEMO_CAP_storage := vm + define run_demo @echo "" @echo "==========================================" - @echo "Demo test: $(1)" + @echo "Demo test: $(1)$(if $(2), --capability $(2),)" @echo "==========================================" - @echo "Running cmd: ISVCTL_DEMO_MODE=1 uv run isvctl test run -f isvctl/configs/providers/my-isv/config/$(1).yaml$(if $(filter storage,$(1)), --capability vm,)" + @echo "Running cmd: ISVCTL_DEMO_MODE=1 uv run isvctl test run -f isvctl/configs/providers/my-isv/config/$(1).yaml$(if $(2), --capability $(2),)" @ISVCTL_DEMO_MODE=1 uv run isvctl test run \ -f isvctl/configs/providers/my-isv/config/$(1).yaml \ - $(if $(filter storage,$(1)),--capability vm,) + $(if $(2),--capability $(2),) endef demo-test: demo-all ## Alias for demo-all (backward compat) @@ -107,7 +115,13 @@ demo-all: ## Run all my-isv living examples (or demo- for one, e.g. demo- @echo "Suites: $(MY_ISV_SUITES)" $(DEMO_TARGETS): demo-%: - $(call run_demo,$*) + $(call run_demo,$*,$(DEMO_CAP_$*)) + +# image-registry splits its gated checks between vm and bare_metal, so one run +# cannot reach both. This explicit rule overrides the pattern rule above. +demo-image-registry: + $(call run_demo,image-registry,vm) + $(call run_demo,image-registry,bare_metal) coverage: ## Run tests with coverage and generate combined report @echo "Running tests with coverage..." diff --git a/isvctl/configs/suites/README.md b/isvctl/configs/suites/README.md index 6a15292fb..8feee6303 100644 --- a/isvctl/configs/suites/README.md +++ b/isvctl/configs/suites/README.md @@ -9,6 +9,38 @@ commands (steps + scripts) that produce JSON for the validations to check. - **New to the framework?** See the [External Validation Guide](../../../docs/guides/external-validation-guide.md). - **Try it without cloud credentials:** `make demo-test`. +## What runs, and when + +A **platform suite** (`platform: ` — `vm`, `bare_metal`, `k8s`, +`slurm`) is the obligation attached to declaring that capability. Its checks +declare no `requires:`; they all run. + +A **plain suite** (everything else — `storage`, `network`, ...) mixes checks +that need no infrastructure with checks that do. Each declares what it +presupposes: + +```yaml +requires: [] # core - runs in every context +requires: [kubernetes] # runs only under --capability kubernetes +requires: [vm, bare_metal] # any-match: either context satisfies it +``` + +One rule decides what runs, and it does not depend on how you named the +config — `--suite`, `-f`, and `--label` discovery all behave identically: + +> **A plain suite with no `--capability` runs its core checks.** Name a +> capability to add the checks gated on it. + +```bash +isvctl test run --provider aws --suite storage # core only +isvctl test run --provider aws --suite storage --capability kubernetes # core + k8s checks +``` + +There is no "run everything" context: no ISV runs on `vm` and `kubernetes` +at once, so a run always carries exactly one context. Steps follow the same +rule — give a step `requires:` when it builds or tears down a fixture only +some contexts need, so a core run neither provisions nor leaks it. + Suites: [`iam`](iam.yaml), [`network`](network.yaml), diff --git a/isvctl/src/isvctl/cli/test.py b/isvctl/src/isvctl/cli/test.py index d4a689557..975b4f80e 100644 --- a/isvctl/src/isvctl/cli/test.py +++ b/isvctl/src/isvctl/cli/test.py @@ -117,6 +117,27 @@ def _junitxml_for_discovered_config(junitxml: Path, match: ProviderConfigMatch, return junitxml.with_name(f"{junitxml.stem}-{match.config_path.stem}{junitxml.suffix}") +def _resolve_capability_context(config: RunConfig, capability: str | None, suite_label: str) -> str | None: + """Return the requirement context a run should execute under. + + One rule for every entry path: a plain suite with no ``--capability`` runs + its core checks. "Unfiltered" corresponds to no real ISV situation - nobody + runs on vm and kubernetes at once - so a plain suite always has a context. + Platform suites declare no ``requires:``, so they are left alone. + """ + if config.tests and config.tests.platform: + return capability + + if capability is None: + print_progress(f"No capability selected; running {suite_label!r} core checks.") + return CORE_REQUIREMENT_CONTEXT + + entries = parse_validations(config.tests.validations if config.tests else {}) + if not any(capability in entry.requires for entry in entries): + print_warning(f"No check in {suite_label!r} requires {capability}; running core checks only.") + return capability + + def _human_readable_dry_run( config: RunConfig, capability: str | None, @@ -189,17 +210,17 @@ def run( str | None, typer.Option( "--suite", - help=( - "Run one platform or plain suite from the selected provider. " - "Plain suites default to core checks unless --capability is set." - ), + help="Run one platform or plain suite from the selected provider.", ), ] = None, capability: Annotated[ str | None, typer.Option( "--capability", - help="Single capability context (one of the platform suites) used to filter check requirements.", + help=( + "Capability context used to filter check requirements (one of the platform suites). " + "Omit it and a plain suite runs its core checks -- the same rule for --suite, -f and --label." + ), ), ] = None, set_values: Annotated[ @@ -333,6 +354,8 @@ def run( print_error(str(exc)) raise typer.Exit(code=1) + suite_label: str | None = None + if suite: if not provider: print_error("--suite requires --provider.") @@ -346,9 +369,7 @@ def run( print_error(str(exc)) raise typer.Exit(code=1) print_progress(f"Selected {selected_suite.name!r} suite for provider {provider!r}.") - if selected_suite.platform is None and capability_context is None: - capability_context = CORE_REQUIREMENT_CONTEXT - print_progress("No capability selected; running the plain suite's core checks.") + suite_label = selected_suite.name config_files = [selected_suite.config_path] provider = None @@ -453,6 +474,16 @@ def run( print_error(f"Configuration validation failed: {e}") raise typer.Exit(code=1) + # Resolve the requirement context here rather than per entry path, so the + # same config behaves identically whether it was reached via --suite, -f, + # or --label discovery. Platform suites carry no `requires:`, so they keep + # the unfiltered context (filtering there would be a no-op anyway). + capability_context = _resolve_capability_context( + config, + capability_context, + suite_label or config_files[0].stem, + ) + if dry_run: typer.echo(_human_readable_dry_run(config, capability_context, labels, exclude_labels)) if extra_pytest_args: diff --git a/isvctl/tests/test_test_cli_labels.py b/isvctl/tests/test_test_cli_labels.py index 28274fd90..38a26808a 100644 --- a/isvctl/tests/test_test_cli_labels.py +++ b/isvctl/tests/test_test_cli_labels.py @@ -377,6 +377,134 @@ def test_plain_suite_without_capability_defaults_to_core( assert "[RUN] VmCheck" in vm_result.stdout +def test_every_entry_path_resolves_the_same_capability_context( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """One config runs the same checks whether reached via --suite or -f. + + The context models what an ISV declared, which cannot depend on how the + config was named on the command line. + """ + configs_root = tmp_path / "configs" + suite_path = configs_root / "suites" / "storage.yaml" + suite_path.parent.mkdir(parents=True) + suite_path.write_text( + """\ +tests: + validations: + core: + checks: + CoreCheck: + test_id: "N/A" + labels: ["storage"] + requires: [] + vm: + checks: + VmCheck: + test_id: "N/A" + labels: ["storage"] + requires: [vm] +""", + encoding="utf-8", + ) + (configs_root / "suites" / "vm.yaml").write_text( + "tests:\n platform: vm\n validations: {}\n", + encoding="utf-8", + ) + provider_config = _write_provider_config(configs_root, "aws", "storage.yaml", "storage.yaml", "storage") + monkeypatch.setattr(test_cli, "CONFIGS_ROOT", configs_root) + + via_suite = runner.invoke( + test_cli.app, + ["run", "--provider", "aws", "--suite", "storage", "--dry-run", "--no-upload"], + ) + via_file = runner.invoke( + test_cli.app, + ["run", "-f", str(provider_config), "--dry-run", "--no-upload"], + ) + + assert via_suite.exit_code == 0, via_suite.output + assert via_file.exit_code == 0, via_file.output + for output in (via_suite.stdout, via_file.stdout): + assert "Capability: core" in output + assert "[RUN] CoreCheck" in output + assert "[SKIP] VmCheck" in output + + +def test_platform_suite_keeps_the_unfiltered_context( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """Platform suites declare no requires, so the core default must not apply.""" + configs_root = tmp_path / "configs" + suites = configs_root / "suites" + suites.mkdir(parents=True) + (suites / "vm.yaml").write_text( + """\ +tests: + platform: vm + validations: + vm: + checks: + PlatformCheck: + test_id: "N/A" + labels: ["vm"] +""", + encoding="utf-8", + ) + _write_provider_config(configs_root, "aws", "vm.yaml", "vm.yaml", "vm") + monkeypatch.setattr(test_cli, "CONFIGS_ROOT", configs_root) + + result = runner.invoke( + test_cli.app, + ["run", "--provider", "aws", "--suite", "vm", "--dry-run", "--no-upload"], + ) + + assert result.exit_code == 0, result.output + assert "Capability: not filtered" in result.stdout + assert "[RUN] PlatformCheck" in result.stdout + + +def test_capability_with_no_matching_check_warns( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """A capability no check in the suite requires is a likely mistake.""" + configs_root = tmp_path / "configs" + suite_path = configs_root / "suites" / "iam.yaml" + suite_path.parent.mkdir(parents=True) + suite_path.write_text( + """\ +tests: + validations: + iam: + checks: + CoreCheck: + test_id: "N/A" + labels: ["iam"] + requires: [] +""", + encoding="utf-8", + ) + (configs_root / "suites" / "kubernetes.yaml").write_text( + "tests:\n platform: kubernetes\n validations: {}\n", + encoding="utf-8", + ) + _write_provider_config(configs_root, "aws", "iam.yaml", "iam.yaml", "iam") + monkeypatch.setattr(test_cli, "CONFIGS_ROOT", configs_root) + + result = runner.invoke( + test_cli.app, + ["run", "--provider", "aws", "--suite", "iam", "--capability", "kubernetes", "--dry-run", "--no-upload"], + ) + + assert result.exit_code == 0, result.output + # print_warning goes to stderr; the dry-run body to stdout. + assert "No check in 'iam' requires kubernetes" in result.stderr + assert "[RUN] CoreCheck" in result.stdout + + def test_suite_and_label_filters_compose( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, From 1b4b00f19ec38896c2b1353d1bfba4eb164d014a Mon Sep 17 00:00:00 2001 From: Alexandre Begnoche Date: Fri, 24 Jul 2026 14:59:02 -0400 Subject: [PATCH 26/34] docs: document the capability model `requires:`, `--suite` and `--capability` appeared in no user-facing doc. The model existed only in the code and in decision.md, an internal design record, so an ISV had no way to learn what governs which checks they owe. - configs/suites/README.md already gained the rule when the default changed; this adds the full model to the two guides an ISV actually reads. - configuration.md: a "Capabilities and requires" section (the four mutually exclusive capabilities, platform vs plain suites, any-match semantics and why AND is deliberately inexpressible, the one selection rule, opt-in scope, the shared namespace), plus `requires` in the step field table with the gate-both-halves-of-a-fixture guidance. - external-validation-guide.md: the same model stated for an ISV audience, and a Running Validations section that shows the --suite/--capability grammar, --dry-run, label composition and single-check rerun by pytest passthrough. - my-isv scaffold README: real invocations rather than only the -f form, plus what governs which checks run. Also corrects drift found in the scaffold's Domains table: 7 of 11 script counts were wrong, and k8s/slurm listed no provider YAML - true until those configs landed in 3735674. Both now note they need a real cluster and so sit outside `make demo-test`. Drops a stale MY_ISV_DOMAINS reference for the MY_ISV_SUITES / DEMO_CAP_ variables that replaced it. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Alexandre Begnoche --- docs/guides/configuration.md | 80 +++++++++++++++++++ docs/guides/external-validation-guide.md | 71 +++++++++++++++- .../providers/my-isv/scripts/README.md | 48 ++++++++--- 3 files changed, 189 insertions(+), 10 deletions(-) diff --git a/docs/guides/configuration.md b/docs/guides/configuration.md index 5c0f9bc2a..ad1a05f70 100644 --- a/docs/guides/configuration.md +++ b/docs/guides/configuration.md @@ -195,6 +195,28 @@ Each step defines a command to execute: | `skip` | No | Skip this step | | `continue_on_failure` | No | Continue even if this step fails | | `output_schema` | No | Schema name for output validation | +| `requires` | No | Capability contexts this step runs in (see [Capabilities](#capabilities-and-requires)) | + +#### Gating a step with `requires` + +A step is skipped automatically when **every** validation bound to it is +filtered out by the run's capability. That inference covers most steps, but it +cannot reach a step no validation binds to — typically a teardown step: + +```yaml +# Runs only under --capability kubernetes. Without the gate, a core run would +# try to tear down a cluster it never created. +- name: teardown_cluster + phase: teardown + command: "./scripts/teardown_cluster.sh" + requires: [kubernetes] +``` + +Rule of thumb: **if a step builds or destroys a fixture that only some contexts +need, give it an explicit `requires:`** — and give both halves of the fixture the +same one, so setup and teardown always move together. A step that survives the +gate must not reference a gated-off step's output; use `default(...)` if it +legitimately might be absent. ### Validation Configuration @@ -267,6 +289,64 @@ The part before the dash must match an existing validation class name (e.g., `K8 - The suffix is free-form: `K8sNimHelmWorkload-small`, `SlurmPartition-cpu`, `SlurmGpuAllocation-1gpu` are all valid. - Each variant is a distinct test entry in coverage tracking. +## Capabilities and `requires` + +An ISV declares which **capabilities** it supports. There are exactly four, and +they are mutually exclusive execution environments — you run on one at a time, +never a combination: + +`vm` · `bare_metal` · `kubernetes` · `slurm` + +Each has a **platform suite** (`suites/vm.yaml`, `suites/k8s.yaml`, ...) carrying +the checks you owe by declaring it. Its `tests.platform:` key names the +capability, and its checks declare no `requires:` — they all run. + +Everything else is a **plain suite** (`storage`, `network`, `iam`, ...), named by +its filename. A plain suite mixes checks that need no particular infrastructure +with checks that presuppose some. Each check says which: + +```yaml +requires: [] # core - runs in every context +requires: [kubernetes] # runs only under --capability kubernetes +requires: [vm, bare_metal] # any-match: either context satisfies it +``` + +`requires` is **any-match**, not a set to satisfy simultaneously: a check runs +when its list is empty, or when the run's capability appears in it. There is +deliberately no way to express "needs vm AND kubernetes" — no check needs it, +and the mutual exclusivity above means such a check could never run. + +### What runs, and when + +One rule, and it does not depend on how you named the config — `--suite`, `-f`, +and `--label` discovery behave identically: + +> **A plain suite with no `--capability` runs its core checks.** Name a +> capability to add the checks gated on it. + +```bash +isvctl test run --provider acme --suite storage # core only +isvctl test run --provider acme --suite storage --capability kubernetes # core + k8s checks +isvctl test run --provider acme --suite kubernetes # the platform suite +``` + +There is no "run everything" context: a plain suite always carries exactly one. +Passing a capability no check in the suite requires is allowed but warns, since +a flag that silently does nothing is usually a typo. + +Two consequences worth internalising: + +- **Nothing is mandatory.** A check is in scope only if you declared the suite + that contains it. 100% is always relative to what you declared, so declaring + a subset legitimately yields zero checks from the suites you left out. +- **A capability and a plain suite compose.** The 15 CSI checks in `storage` + need `storage` *and* `kubernetes`. Declaring `kubernetes` alone runs the + Kubernetes platform suite but no storage CSI checks. + +Capability names and plain-suite names share one namespace, so a plain suite may +not be named after a capability. `catalog_document` and +`scripts/validate_suite_wiring.py` both reject the collision. + ## Import and Override Provider configs can import a canonical test suite and override command definitions while inheriting validations (unless explicitly overridden): diff --git a/docs/guides/external-validation-guide.md b/docs/guides/external-validation-guide.md index 6d5dd3045..399b3b900 100644 --- a/docs/guides/external-validation-guide.md +++ b/docs/guides/external-validation-guide.md @@ -186,9 +186,64 @@ For validation timing and phase control, see the [Configuration Guide](configura --- +## Capabilities and check requirements + +You declare which **capabilities** you support. There are four, and they are +mutually exclusive execution environments — you run on one at a time: + +`vm` · `bare_metal` · `kubernetes` · `slurm` + +Each has a **platform suite** you owe by declaring it (`vm`, `bare_metal`, +`kubernetes`, `slurm`). Everything else is a **plain suite** (`storage`, +`network`, `iam`, ...) that mixes checks needing no particular infrastructure +with checks that presuppose some. Each check declares which: + +```yaml +requires: [] # core - runs in every context +requires: [kubernetes] # runs only under --capability kubernetes +requires: [vm, bare_metal] # any-match: either context satisfies it +``` + +**Nothing is mandatory.** A check is in scope only if you declared the suite +containing it, so 100% is always relative to what you declared. Declaring a +subset legitimately yields zero checks from the suites you left out — that is +the design, not a gap. + +One rule decides what runs, and it does not depend on how you named the config +(`--suite`, `-f` and `--label` behave identically): + +> **A plain suite with no `--capability` runs its core checks.** Name a +> capability to add the checks gated on it. + +If a step builds or destroys a fixture only some contexts need, gate it the same +way so a core run neither provisions nor leaks it: + +```yaml +- name: teardown_cluster + phase: teardown + command: "./scripts/teardown_cluster.sh" + requires: [kubernetes] +``` + +See the [Configuration Guide](configuration.md#capabilities-and-requires) for the +full model. + +--- + ## Running Validations ```bash +# One suite for your provider - the form the UI emits +isvctl test run --provider acme --suite storage # core checks +isvctl test run --provider acme --suite storage --capability kubernetes # + k8s checks +isvctl test run --provider acme --suite kubernetes # a platform suite + +# See what would run, without executing anything +isvctl test run --provider acme --suite storage --capability vm --dry-run + +# Point at a config file directly (same capability rule applies) +isvctl test run -f config.yaml + # Run all phases isvctl test run -f config.yaml @@ -209,10 +264,20 @@ isvctl test run -f config.yaml -- -k "ConnectivityCheck" isvctl test run -f config.yaml --label gpu isvctl test run -f config.yaml -- -m "not slow" +# Labels compose with suite selection +isvctl test run --provider acme --suite storage --capability vm --label min_req + +# Re-run one failed check in its lifecycle context (pytest passthrough) +isvctl test run --provider acme --suite storage --capability kubernetes -- -k K8sCsiPvcExpandCheck + # Debug: full output on failure isvctl test run -f config.yaml -v -- -s --tb=long ``` +Re-running a single failed check is pytest passthrough after `--`; there are no +dedicated rerun flags. Setup steps re-run, which is the deliberate trade for not +maintaining a dependency graph. + > **Teardown behavior:** By default, teardown runs even when setup or test validations fail, ensuring cloud resources are cleaned up. Individual teardown step failures don't block remaining teardown steps (best-effort execution). --- @@ -266,9 +331,13 @@ Preview the whole pipeline with no cloud: ```bash make demo-test # sets ISVCTL_DEMO_MODE=1 and runs all my-isv configs (~10s) -# Domains are listed in the Makefile MY_ISV_DOMAINS variable. +# Suites are listed in the Makefile MY_ISV_SUITES variable; DEMO_CAP_ +# names the capability a suite runs under so its gated checks are exercised. ``` +The k8s and Slurm examples are excluded: they drive a real cluster, so a +dummy-success stub has nothing to return for them. + --- ## Related Documentation diff --git a/isvctl/configs/providers/my-isv/scripts/README.md b/isvctl/configs/providers/my-isv/scripts/README.md index 3d7b89dba..d15e34e57 100644 --- a/isvctl/configs/providers/my-isv/scripts/README.md +++ b/isvctl/configs/providers/my-isv/scripts/README.md @@ -28,16 +28,20 @@ template, then fill in the TODOs. | Domain | Scripts | Contract | Provider YAML | AWS reference | |--------|---------|----------|---------------|---------------| | `iam/` | 3 | [`suites/iam.yaml`](../../../suites/iam.yaml) | [`config/iam.yaml`](../config/iam.yaml) | [`providers/aws/scripts/iam/`](../../aws/scripts/iam/) | -| `control-plane/` | 10 | [`suites/control-plane.yaml`](../../../suites/control-plane.yaml) | [`config/control-plane.yaml`](../config/control-plane.yaml) | [`providers/aws/scripts/control-plane/`](../../aws/scripts/control-plane/) | -| `vm/` | 10 | [`suites/vm.yaml`](../../../suites/vm.yaml) | [`config/vm.yaml`](../config/vm.yaml) | [`providers/aws/scripts/vm/`](../../aws/scripts/vm/) | -| `bare_metal/` | 12 | [`suites/bare_metal.yaml`](../../../suites/bare_metal.yaml) | [`config/bare_metal.yaml`](../config/bare_metal.yaml) | [`providers/aws/scripts/bare_metal/`](../../aws/scripts/bare_metal/) | -| `storage/` | 17 | [`suites/storage.yaml`](../../../suites/storage.yaml) | [`config/storage.yaml`](../config/storage.yaml) | [`providers/aws/scripts/storage/`](../../aws/scripts/storage/) | -| `network/` | 18 | [`suites/network.yaml`](../../../suites/network.yaml) | [`config/network.yaml`](../config/network.yaml) | [`providers/aws/scripts/network/`](../../aws/scripts/network/) | -| `observability/` | 1 | [`suites/observability.yaml`](../../../suites/observability.yaml) | [`config/observability.yaml`](../config/observability.yaml) | [`providers/aws/scripts/observability/`](../../aws/scripts/observability/) | +| `control-plane/` | 11 | [`suites/control-plane.yaml`](../../../suites/control-plane.yaml) | [`config/control-plane.yaml`](../config/control-plane.yaml) | [`providers/aws/scripts/control-plane/`](../../aws/scripts/control-plane/) | +| `vm/` | 12 | [`suites/vm.yaml`](../../../suites/vm.yaml) | [`config/vm.yaml`](../config/vm.yaml) | [`providers/aws/scripts/vm/`](../../aws/scripts/vm/) | +| `bare_metal/` | 14 | [`suites/bare_metal.yaml`](../../../suites/bare_metal.yaml) | [`config/bare_metal.yaml`](../config/bare_metal.yaml) | [`providers/aws/scripts/bare_metal/`](../../aws/scripts/bare_metal/) | +| `storage/` | 20 | [`suites/storage.yaml`](../../../suites/storage.yaml) | [`config/storage.yaml`](../config/storage.yaml) | [`providers/aws/scripts/storage/`](../../aws/scripts/storage/) | +| `network/` | 24 | [`suites/network.yaml`](../../../suites/network.yaml) | [`config/network.yaml`](../config/network.yaml) | [`providers/aws/scripts/network/`](../../aws/scripts/network/) | +| `observability/` | 5 | [`suites/observability.yaml`](../../../suites/observability.yaml) | [`config/observability.yaml`](../config/observability.yaml) | [`providers/aws/scripts/observability/`](../../aws/scripts/observability/) | | `image-registry/` | 7 | [`suites/image-registry.yaml`](../../../suites/image-registry.yaml) | [`config/image-registry.yaml`](../config/image-registry.yaml) | [`providers/aws/scripts/image-registry/`](../../aws/scripts/image-registry/) | -| `security/` | 19 | [`suites/security.yaml`](../../../suites/security.yaml) | [`config/security.yaml`](../config/security.yaml) | [`providers/aws/scripts/security/`](../../aws/scripts/security/), [`providers/aws/scripts/capacity/`](../../aws/scripts/capacity/) | -| `k8s/` | 9 shell | [`suites/k8s.yaml`](../../../suites/k8s.yaml) | - | [`providers/aws/scripts/eks/`](../../aws/scripts/eks/) | -| `slurm/` | 2 shell | [`suites/slurm.yaml`](../../../suites/slurm.yaml) | - | - | +| `security/` | 17 | [`suites/security.yaml`](../../../suites/security.yaml) | [`config/security.yaml`](../config/security.yaml) | [`providers/aws/scripts/security/`](../../aws/scripts/security/), [`providers/aws/scripts/capacity/`](../../aws/scripts/capacity/) | +| `k8s/` | 9 shell | [`suites/k8s.yaml`](../../../suites/k8s.yaml) | [`config/k8s.yaml`](../config/k8s.yaml) | [`providers/aws/scripts/eks/`](../../aws/scripts/eks/) | +| `slurm/` | 2 shell | [`suites/slurm.yaml`](../../../suites/slurm.yaml) | [`config/slurm.yaml`](../config/slurm.yaml) | - | + +The `k8s/` and `slurm/` examples drive a **real** cluster (validations shell out +to `kubectl` / `sinfo`), so they are not part of `make demo-test` — a +dummy-success stub has nothing to return for them. See [`suites/README.md`](../../../suites/README.md) for the per-step / per-field breakdown. @@ -63,9 +67,35 @@ to generate outside `isvctl/configs/providers/`. **4. Run for real (no demo flag):** ```bash +# A platform suite - the obligation attached to declaring that capability +uv run isvctl test run --provider acme --suite vm + +# A plain suite: core checks by default, capability-gated checks when you name one +uv run isvctl test run --provider acme --suite storage +uv run isvctl test run --provider acme --suite storage --capability vm + +# Or point at the config file directly - the same capability rule applies uv run isvctl test run -f isvctl/configs/providers/acme/config/vm.yaml ``` +Add `--dry-run` to any of these to list what would run and what would be +skipped, without executing a thing. + +### Which checks run + +Checks in a plain suite declare what they presuppose. `requires: []` (core) runs +in every context; `requires: [kubernetes]` runs only under +`--capability kubernetes`; `requires: [vm, bare_metal]` is any-match — either +context satisfies it. A plain suite with no `--capability` runs its core checks. + +The same applies to your steps: if a step builds or destroys a fixture only some +contexts need, gate it so a core run neither provisions nor leaks it. Both halves +of a fixture take the same gate — see `config/storage.yaml`, where +`setup_cluster` and `teardown_cluster` are both `requires: [kubernetes]`. + +Nothing is mandatory. A check is in scope only if you declared the suite holding +it, so 100% is always relative to what you declared. + ## Private provider repositories You do not need to contribute your provider scripts back to this repository, From 514b4e61546c2f2e09facf6ca57fd28a8c866a5e Mon Sep 17 00:00:00 2001 From: Alexandre Begnoche Date: Fri, 24 Jul 2026 16:10:49 -0400 Subject: [PATCH 27/34] feat(reporting): report the (suite, capability) pair on test runs A run's signal is the pair, not the suite alone: `network` and `network --capability vm` execute different checks, so recording only one axis loses the distinction at the point where it matters most. Neither axis was transmitted at all -- the service inferred a single test_target_type from the first command, which collapsed both. Two client-side spellings have to be translated before they leave the process. CORE_REQUIREMENT_CONTEXT is this module's word for "no capability", which the service records as NULL rather than a sentinel. And a platform suite carries no explicit context because its own platform *is* the capability it runs under, so report that instead of dropping the axis for every platform-suite run. resolve_suite_name recovers the suite from every entry path, including `-f lab.yaml -f commands.yaml -f suites/k8s.yaml` where the first config is not the suite. Co-Authored-By: Claude Opus 5 (1M context) --- isvctl/src/isvctl/cli/deploy.py | 5 ++ isvctl/src/isvctl/cli/test.py | 28 ++++++- isvctl/src/isvctl/config/suite_resolution.py | 32 ++++++++ isvctl/src/isvctl/reporting.py | 7 ++ isvctl/tests/test_reporting.py | 37 ++++++++++ isvctl/tests/test_suite_resolution.py | 28 +++++++ isvreporter/src/isvreporter/client.py | 14 ++++ isvreporter/src/isvreporter/main.py | 16 ++++ isvreporter/tests/test_client.py | 77 +++++++++++++++++++- 9 files changed, 240 insertions(+), 4 deletions(-) diff --git a/isvctl/src/isvctl/cli/deploy.py b/isvctl/src/isvctl/cli/deploy.py index 4d1ee8f6c..7b0743e63 100644 --- a/isvctl/src/isvctl/cli/deploy.py +++ b/isvctl/src/isvctl/cli/deploy.py @@ -32,6 +32,8 @@ from isvctl.cli import setup_logging from isvctl.cli.common import get_output_dir, print_error, print_progress, print_step, print_warning +from isvctl.cli.test import CONFIGS_ROOT +from isvctl.config.suite_resolution import resolve_suite_name from isvctl.orchestrator.loop import Phase from isvctl.remote import SCPTransfer, SSHClient, TarArchive from isvctl.remote.archive import DEFAULT_EXCLUDES as DEFAULT_ARCHIVE_EXCLUDES @@ -396,6 +398,9 @@ def run( executed_by="isvctl deploy", ci_reference="local-deployment", isv_software_version=isv_software_version, + # deploy has no --capability of its own; the remote `test run` + # it invokes is core-only unless the config is a platform suite. + suite=resolve_suite_name(list(config_files), CONFIGS_ROOT), ) if not test_run_id: print_warning("Failed to create test run, continuing without upload") diff --git a/isvctl/src/isvctl/cli/test.py b/isvctl/src/isvctl/cli/test.py index 975b4f80e..b711e30c0 100644 --- a/isvctl/src/isvctl/cli/test.py +++ b/isvctl/src/isvctl/cli/test.py @@ -48,7 +48,12 @@ ) from isvctl.config.merger import merge_yaml_files from isvctl.config.schema import RunConfig -from isvctl.config.suite_resolution import SuiteResolutionError, parse_capability, resolve_suite +from isvctl.config.suite_resolution import ( + SuiteResolutionError, + parse_capability, + resolve_suite, + resolve_suite_name, +) from isvctl.orchestrator.loop import Orchestrator, Phase from isvctl.reporting import check_upload_credentials, create_test_run, get_environment_config, update_test_run @@ -138,6 +143,22 @@ def _resolve_capability_context(config: RunConfig, capability: str | None, suite return capability +def _reported_capability(config: RunConfig, capability_context: str | None) -> str | None: + """Return the capability to record on the uploaded test run. + + Two client-side spellings have to be translated before they leave the process. + ``CORE_REQUIREMENT_CONTEXT`` is this module's word for "no capability", which + the service records as NULL. A platform suite is left with no explicit context + because its own platform *is* the capability it runs under, so report that + rather than losing the axis for every platform-suite run. + """ + if capability_context == CORE_REQUIREMENT_CONTEXT: + return None + if capability_context is None and config.tests and config.tests.platform: + return config.tests.platform + return capability_context + + def _human_readable_dry_run( config: RunConfig, capability: str | None, @@ -478,10 +499,11 @@ def run( # same config behaves identically whether it was reached via --suite, -f, # or --label discovery. Platform suites carry no `requires:`, so they keep # the unfiltered context (filtering there would be a no-op anyway). + suite_name = suite_label or resolve_suite_name(config_files, CONFIGS_ROOT) capability_context = _resolve_capability_context( config, capability_context, - suite_label or config_files[0].stem, + suite_name or config_files[0].stem, ) if dry_run: @@ -539,6 +561,8 @@ def run( tags=tags or ["validation-test", "isvctl"], start_time=start_time, isv_software_version=isv_software_version, + suite=suite_name, + capability=_reported_capability(config, capability_context), ) if not test_run_id: print_warning("Failed to create test run, continuing without upload") diff --git a/isvctl/src/isvctl/config/suite_resolution.py b/isvctl/src/isvctl/config/suite_resolution.py index 14b838bf3..01b40a4df 100644 --- a/isvctl/src/isvctl/config/suite_resolution.py +++ b/isvctl/src/isvctl/config/suite_resolution.py @@ -45,6 +45,13 @@ def platform_vocabulary(configs_root: Path) -> frozenset[str]: return frozenset(platforms) +def suite_vocabulary(configs_root: Path) -> frozenset[str]: + """Return plain suite names declared by canonical suite YAML.""" + declarable = platform_vocabulary(configs_root) + names = {_normalize_name(path.stem) for path in (configs_root / "suites").glob("*.yaml")} + return frozenset(names - declarable) + + def parse_capability(value: str | None, configs_root: Path) -> str | None: """Parse the single capability context (one platform suite name). @@ -101,6 +108,31 @@ def _suite_name(config_path: Path, declarable: frozenset[str]) -> tuple[str, str return _normalize_name(name), None +def resolve_suite_name(config_paths: list[Path], configs_root: Path) -> str | None: + """Return the suite name a set of ``-f`` configs resolves to. + + A run's identity is (suite, capability), so the suite has to be recoverable + from every entry path -- including ``-f lab.yaml -f commands.yaml + -f suites/k8s.yaml``, where the first config is not the suite. Classify each + config in order and take the first that names a known platform or plain + suite; fall back to the first config's stem when nothing matches, which is + the best available label for an ad-hoc config. + """ + if not config_paths: + return None + + declarable = platform_vocabulary(configs_root) + known = declarable | suite_vocabulary(configs_root) + for path in config_paths: + try: + name, _ = _suite_name(path, declarable) + except SuiteResolutionError: + continue + if name in known: + return name + return _normalize_name(config_paths[0].stem) + + def resolve_suite(provider: str, suite: str, *, configs_root: Path) -> ResolvedSuite: """Resolve exactly one provider config for a platform or plain suite.""" config_dir = configs_root / "providers" / provider / "config" diff --git a/isvctl/src/isvctl/reporting.py b/isvctl/src/isvctl/reporting.py index 7fadd9302..bb64161d3 100644 --- a/isvctl/src/isvctl/reporting.py +++ b/isvctl/src/isvctl/reporting.py @@ -80,6 +80,8 @@ def create_test_run( executed_by: str = "isvctl", ci_reference: str = "local-run", isv_software_version: str | None = None, + suite: str | None = None, + capability: str | None = None, ) -> str | None: """Create a test run in ISV Lab Service. @@ -91,6 +93,9 @@ def create_test_run( executed_by: Tool that executed the test ci_reference: CI/CD reference identifier isv_software_version: ISV software stack version (opaque string from ISV) + suite: Suite that was run (e.g. "network", "vm") + capability: Capability context resolved from ``--capability``; None for + a core-only run Returns: Test run ID if successful, None otherwise @@ -125,6 +130,8 @@ def create_test_run( start_time=start_time, isv_software_version=isv_software_version, isv_test_version=isv_test_version, + suite=suite, + capability=capability, ) return result.get("data", {}).get("testRunId") except SystemExit: diff --git a/isvctl/tests/test_reporting.py b/isvctl/tests/test_reporting.py index 4b6cd6810..22b832252 100644 --- a/isvctl/tests/test_reporting.py +++ b/isvctl/tests/test_reporting.py @@ -18,6 +18,8 @@ import os from unittest.mock import MagicMock, patch +from isvctl.cli.test import CORE_REQUIREMENT_CONTEXT, _reported_capability +from isvctl.config.schema import RunConfig from isvctl.reporting import ( check_upload_credentials, get_environment_config, @@ -172,3 +174,38 @@ def test_forwards_complete_catalog_document( platforms=["kubernetes", "vm"], suites=["storage", "iam"], ) + + +class TestReportedCapability: + """The capability recorded on a test run, translated out of client spellings. + + A run's signal is the (suite, capability) pair: `network` and + `network --capability vm` execute different checks, so the capability has to + survive the trip to the service in a form the service understands. + """ + + @staticmethod + def _config(platform: str | None) -> RunConfig: + raw: dict = {"commands": {}, "tests": {"validations": {}}} + if platform: + raw["tests"]["platform"] = platform + return RunConfig.model_validate(raw) + + def test_core_context_is_reported_as_no_capability(self) -> None: + """`core` is this client's word for "no capability", not a fifth one.""" + config = self._config(None) + assert _reported_capability(config, CORE_REQUIREMENT_CONTEXT) is None + + def test_platform_suite_reports_its_own_platform(self) -> None: + """A platform suite gets no explicit context because its platform is one. + + Passing the resolved context straight through would record NULL for + every platform-suite run and lose the axis entirely. + """ + config = self._config("vm") + assert _reported_capability(config, None) == "vm" + + def test_explicit_capability_wins(self) -> None: + config = self._config(None) + assert _reported_capability(config, "vm") == "vm" + assert _reported_capability(self._config("vm"), "kubernetes") == "kubernetes" diff --git a/isvctl/tests/test_suite_resolution.py b/isvctl/tests/test_suite_resolution.py index 534c85687..4acee1990 100644 --- a/isvctl/tests/test_suite_resolution.py +++ b/isvctl/tests/test_suite_resolution.py @@ -15,6 +15,7 @@ SuiteResolutionError, parse_capability, resolve_suite, + resolve_suite_name, ) CONFIGS_ROOT = Path(__file__).resolve().parents[1] / "configs" @@ -127,3 +128,30 @@ def test_storage_cluster_fixture_uses_its_own_output_contract(provider: str) -> steps = {step.name: step for step in config.get_steps("storage")} assert steps["setup_cluster"].output_schema == "generic" + + +def test_suite_name_survives_every_entry_path(tmp_path: Path) -> None: + """A run's suite must be recoverable from the configs, not just from --suite. + + `-f lab.yaml -f commands.yaml -f suites/k8s.yaml` is a documented entry + path, and there the first config is not the suite - taking its stem would + record the run against "lab". + """ + _write_catalog(tmp_path) + configs = tmp_path / "providers" / "acme" / "config" + (configs / "lab.yaml").write_text("context: {}\n") + + assert resolve_suite_name([configs / "eks.yaml"], tmp_path) == "kubernetes" + assert resolve_suite_name([configs / "storage.yaml"], tmp_path) == "storage" + assert resolve_suite_name([configs / "lab.yaml", configs / "eks.yaml"], tmp_path) == "kubernetes" + assert resolve_suite_name([tmp_path / "suites" / "k8s.yaml"], tmp_path) == "kubernetes" + + +def test_ad_hoc_config_falls_back_to_its_own_stem(tmp_path: Path) -> None: + """An unrecognized config still labels its run rather than recording nothing.""" + _write_catalog(tmp_path) + ad_hoc = tmp_path / "one-off.yaml" + ad_hoc.write_text("commands: {}\n") + + assert resolve_suite_name([ad_hoc], tmp_path) == "one_off" + assert resolve_suite_name([], tmp_path) is None diff --git a/isvreporter/src/isvreporter/client.py b/isvreporter/src/isvreporter/client.py index 82cc0ec6d..6765103f8 100644 --- a/isvreporter/src/isvreporter/client.py +++ b/isvreporter/src/isvreporter/client.py @@ -39,6 +39,8 @@ def create_test_run( start_time: str, isv_software_version: str | None = None, isv_test_version: str | None = None, + suite: str | None = None, + capability: str | None = None, ) -> dict[str, Any]: """ Create a new test run record. @@ -54,6 +56,10 @@ def create_test_run( start_time: Test run start time (ISO 8601 format) isv_software_version: ISV software stack version (opaque string from ISV) isv_test_version: ISV test tool version (e.g., "1.12.3") + suite: Suite that was run (e.g. "network", "vm") + capability: Capability context the suite ran under (e.g. "vm"); None + means the run was core-only, which is a different signal than the + same suite run under a capability Returns: API response dictionary containing test run ID @@ -75,6 +81,12 @@ def create_test_run( payload["isvSoftwareVersion"] = isv_software_version if isv_test_version: payload["isvTestVersion"] = isv_test_version + if suite: + payload["suite"] = suite + # Sent only when set: a core-only run carries no capability, and null is + # the signal for that rather than a sentinel value. + if capability: + payload["capability"] = capability headers = { "Content-Type": "application/json", @@ -88,6 +100,8 @@ def create_test_run( test_run_id = result["data"]["testRunId"] print("Test run created successfully") print(f" Test Run ID: {test_run_id}") + if suite: + print(f" Suite: {suite} ({capability or 'core only'})") print(f" URL: {endpoint}/v1/labs/{lab_id}/test-runs/{test_run_id}") # Save test run ID to file for later use in after_script diff --git a/isvreporter/src/isvreporter/main.py b/isvreporter/src/isvreporter/main.py index 05f6ac4a1..968c8264a 100644 --- a/isvreporter/src/isvreporter/main.py +++ b/isvreporter/src/isvreporter/main.py @@ -135,6 +135,20 @@ def create( help="ISV test tool version (e.g., '1.12.3')", ), ] = None, + suite: Annotated[ + str | None, + typer.Option( + "--suite", + help="Suite that was run (e.g. 'network', 'vm')", + ), + ] = None, + capability: Annotated[ + str | None, + typer.Option( + "--capability", + help="Capability context the suite ran under (e.g. 'vm'). Omit for a core-only run.", + ), + ] = None, ) -> None: """Create a new test run in ISV Lab Service. @@ -167,6 +181,8 @@ def create( start_time=start_time, isv_software_version=isv_software_version, isv_test_version=isv_test_version, + suite=suite, + capability=capability, ) diff --git a/isvreporter/tests/test_client.py b/isvreporter/tests/test_client.py index b24529b5e..01882db4c 100644 --- a/isvreporter/tests/test_client.py +++ b/isvreporter/tests/test_client.py @@ -15,10 +15,11 @@ """Tests for the ISV Lab Service API client.""" +import json from pathlib import Path -from unittest.mock import patch +from unittest.mock import MagicMock, patch -from isvreporter.client import calculate_duration, load_test_run_id +from isvreporter.client import calculate_duration, create_test_run, load_test_run_id class TestCalculateDuration: @@ -88,3 +89,75 @@ def test_load_empty_test_run_id(self, tmp_path: Path) -> None: with patch("isvreporter.client.TEST_RUN_ID_FILE", test_run_file): result = load_test_run_id() assert result == "" + + +class TestCreateTestRunPayload: + """The create payload has to carry both axes of what the run exercised.""" + + @staticmethod + def _posted_payload(mock_urlopen: MagicMock) -> dict: + request = mock_urlopen.call_args[0][0] + return json.loads(request.data.decode()) + + @staticmethod + def _response() -> MagicMock: + response = MagicMock() + response.read.return_value = json.dumps({"data": {"testRunId": 42}}).encode() + response.__enter__ = lambda self: self + response.__exit__ = lambda *args: False + return response + + @patch("isvreporter.client.urlopen") + def test_sends_suite_and_capability(self, mock_urlopen: MagicMock, tmp_path: Path) -> None: + mock_urlopen.return_value = self._response() + + with ( + patch("isvreporter.client.OUTPUT_DIR", tmp_path), + patch("isvreporter.client.TEST_RUN_ID_FILE", tmp_path / "testrun_id.txt"), + ): + create_test_run( + endpoint="https://api.example.com", + lab_id=1, + jwt_token="jwt", + test_target_type="VM", + tags=[], + executed_by="isvctl", + ci_reference="local-run", + start_time="2026-07-24T12:00:00Z", + suite="network", + capability="vm", + ) + + payload = self._posted_payload(mock_urlopen) + assert payload["suite"] == "network" + assert payload["capability"] == "vm" + + @patch("isvreporter.client.urlopen") + def test_core_only_run_omits_capability(self, mock_urlopen: MagicMock, tmp_path: Path) -> None: + """A core-only run sends no capability - the absence is the signal. + + Sending a sentinel instead would make it indistinguishable from a real + capability in every downstream filter and column. + """ + mock_urlopen.return_value = self._response() + + with ( + patch("isvreporter.client.OUTPUT_DIR", tmp_path), + patch("isvreporter.client.TEST_RUN_ID_FILE", tmp_path / "testrun_id.txt"), + ): + create_test_run( + endpoint="https://api.example.com", + lab_id=1, + jwt_token="jwt", + test_target_type="NETWORK", + tags=[], + executed_by="isvctl", + ci_reference="local-run", + start_time="2026-07-24T12:00:00Z", + suite="network", + capability=None, + ) + + payload = self._posted_payload(mock_urlopen) + assert payload["suite"] == "network" + assert "capability" not in payload From 780782ea06d9bde4379312413b0d8c085c454743 Mon Sep 17 00:00:00 2001 From: Alexandre Begnoche Date: Fri, 24 Jul 2026 16:42:23 -0400 Subject: [PATCH 28/34] refactor(suites): share the capability vocabulary and stop re-parsing suites The pre-commit validator carried a hand-copied DECLARABLE_CAPABILITIES literal, so the hook that exists to enforce the vocabulary was the one place that could silently drift from the schema it enforces. It now imports the shared constant, and the REQUIREMENT_VOCABULARY alias (a pure alias with no external importers) is gone. Each suite YAML was also read three to four times and parsed two to three times per run: _build_suite_map parsed a file and then called iter_config_checks, which re-read and re-parsed the same file, and wiring_errors globbed the directory twice while reading each file twice more in its main loop. Both now expose iter_checks_from_data so a caller that has already parsed the document reuses it, and wiring_errors reads and parses each file exactly once. The suite vocabularies are cached: resolve_suite_name alone scanned the suites directory twice per invocation. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Alexandre Begnoche --- isvctl/src/isvctl/config/suite_resolution.py | 9 ++++- isvtest/src/isvtest/catalog.py | 10 +++++- isvtest/src/isvtest/core/resolution.py | 5 ++- scripts/validate_suite_wiring.py | 35 ++++++++++++-------- 4 files changed, 40 insertions(+), 19 deletions(-) diff --git a/isvctl/src/isvctl/config/suite_resolution.py b/isvctl/src/isvctl/config/suite_resolution.py index 01b40a4df..89b4841a3 100644 --- a/isvctl/src/isvctl/config/suite_resolution.py +++ b/isvctl/src/isvctl/config/suite_resolution.py @@ -4,6 +4,7 @@ """Resolve one platform or plain suite to a provider configuration.""" from dataclasses import dataclass +from functools import cache from pathlib import Path from typing import Any @@ -31,8 +32,13 @@ def _normalize_name(value: str) -> str: return "kubernetes" if normalized == "k8s" else normalized +@cache def platform_vocabulary(configs_root: Path) -> frozenset[str]: - """Return declarable capabilities from canonical platform suite YAML.""" + """Return declarable capabilities from canonical platform suite YAML. + + Cached: the suite directory is fixed for the life of a CLI invocation, and + several entry points ask for the vocabulary two or three times per run. + """ platforms: set[str] = set() for path in (configs_root / "suites").glob("*.yaml"): try: @@ -45,6 +51,7 @@ def platform_vocabulary(configs_root: Path) -> frozenset[str]: return frozenset(platforms) +@cache def suite_vocabulary(configs_root: Path) -> frozenset[str]: """Return plain suite names declared by canonical suite YAML.""" declarable = platform_vocabulary(configs_root) diff --git a/isvtest/src/isvtest/catalog.py b/isvtest/src/isvtest/catalog.py index 147f81b06..671d1bb04 100644 --- a/isvtest/src/isvtest/catalog.py +++ b/isvtest/src/isvtest/catalog.py @@ -68,7 +68,15 @@ def iter_config_checks(config_path: Path) -> Iterator[tuple[str, dict[str, Any]] data = yaml.safe_load(config_path.read_text()) except Exception: return + yield from iter_checks_from_data(data) + +def iter_checks_from_data(data: Any) -> Iterator[tuple[str, dict[str, Any]]]: + """Yield ``(check_name, params)`` from an already-parsed config document. + + Lets callers that have parsed the YAML for other reasons avoid a second + read and parse of the same file. + """ validations = (data or {}).get("tests", {}).get("validations", {}) if not isinstance(validations, dict): return @@ -210,7 +218,7 @@ def _build_suite_map() -> dict[str, dict[str, Any]]: tests = data.get("tests") or {} platform = tests.get("platform") if isinstance(tests, dict) else None suite = str(platform) if isinstance(platform, str) and platform else config_path.stem.replace("-", "_") - for check_name, params in iter_config_checks(config_path): + for check_name, params in iter_checks_from_data(data): if check_name in suite_map and enforce_unique: raise ValueError(f"Suite wiring name {check_name!r} is not globally unique") requires = params.get("requires", []) diff --git a/isvtest/src/isvtest/core/resolution.py b/isvtest/src/isvtest/core/resolution.py index 6c075c7e1..c247f88d3 100644 --- a/isvtest/src/isvtest/core/resolution.py +++ b/isvtest/src/isvtest/core/resolution.py @@ -34,7 +34,6 @@ ADAPTER_HANDLED_CATEGORIES = {"reframe"} DEFAULT_VALIDATION_PHASE = "test" DECLARABLE_CAPABILITIES = frozenset({"vm", "bare_metal", "kubernetes", "slurm"}) -REQUIREMENT_VOCABULARY = DECLARABLE_CAPABILITIES class State(StrEnum): @@ -457,11 +456,11 @@ def _validate_entry_shape(entry: ValidationEntry) -> str | None: raw_requires = entry.params_template.get("requires") if raw_requires is not None: if not isinstance(raw_requires, list) or any( - not isinstance(value, str) or value not in REQUIREMENT_VOCABULARY for value in raw_requires + not isinstance(value, str) or value not in DECLARABLE_CAPABILITIES for value in raw_requires ): return ( f"validation '{entry.name}' requires must be a list containing only: " - f"{', '.join(sorted(REQUIREMENT_VOCABULARY))}" + f"{', '.join(sorted(DECLARABLE_CAPABILITIES))}" ) if len(raw_requires) != len(set(raw_requires)): return f"validation '{entry.name}' requires must not contain duplicates" diff --git a/scripts/validate_suite_wiring.py b/scripts/validate_suite_wiring.py index 5602f99ab..9d1b3811e 100644 --- a/scripts/validate_suite_wiring.py +++ b/scripts/validate_suite_wiring.py @@ -42,12 +42,11 @@ from typing import Any import yaml +from isvtest.core.resolution import DECLARABLE_CAPABILITIES REPO_ROOT = Path(__file__).resolve().parent.parent SUITES_DIR = REPO_ROOT / "isvctl" / "configs" / "suites" _NEXT_CATEGORY_LINE = re.compile(r"^ \S") -DECLARABLE_CAPABILITIES = {"vm", "bare_metal", "kubernetes", "slurm"} -REQUIREMENT_VOCABULARY = DECLARABLE_CAPABILITIES # Opt-in until unique wiring names land in a dedicated PR. ENFORCE_UNIQUE_WIRING = os.environ.get("ISVCTL_ENFORCE_UNIQUE_WIRING") == "1" @@ -108,7 +107,11 @@ def iter_suite_checks(config_path: Path) -> Iterator[tuple[str, str, dict[str, A data = yaml.safe_load(config_path.read_text()) except (OSError, yaml.YAMLError) as exc: raise ValueError(f"failed to read/parse {config_path}: {exc}") from exc + yield from iter_checks_from_data(data) + +def iter_checks_from_data(data: Any) -> Iterator[tuple[str, str, dict[str, Any]]]: + """Yield ``(category, check_name, params)`` from an already-parsed suite doc.""" validations = (data or {}).get("tests", {}).get("validations", {}) if not isinstance(validations, dict): return @@ -149,27 +152,31 @@ def wiring_errors(suites_dir: Path = SUITES_DIR) -> list[str]: occurrence: dict[tuple[Path, str, str], int] = defaultdict(int) wiring_locations: dict[str, str] = {} + # Read and parse each suite once; both the dead-requirement pre-pass and the + # per-check loop below work off these parsed documents. + parsed: list[tuple[Path, list[str], dict[str, Any]]] = [] + for path in sorted(suites_dir.glob("*.yaml")): + try: + text = path.read_text() + parsed.append((path, text.splitlines(), yaml.safe_load(text) or {})) + except (OSError, yaml.YAMLError) as exc: + errors.append(f"failed to read/parse {path}: {exc}") + # A `requires` value is only satisfiable if an ISV can declare that # capability, which requires a platform suite to exist for it. Collect the # platform capabilities that actually have a suite so unreachable (dead) # requirements can be flagged below. declared_platforms: set[str] = set() - for path in sorted(suites_dir.glob("*.yaml")): - try: - data = yaml.safe_load(path.read_text()) or {} - except (OSError, yaml.YAMLError): - continue + for _, _, data in parsed: tests = data.get("tests") if isinstance(data, dict) else None platform = tests.get("platform") if isinstance(tests, dict) else None if isinstance(platform, str) and platform in DECLARABLE_CAPABILITIES: declared_platforms.add(platform) - for path in sorted(suites_dir.glob("*.yaml")): + for path, lines, data in parsed: try: - lines = path.read_text().splitlines() - data = yaml.safe_load(path.read_text()) or {} - checks = list(iter_suite_checks(path)) - except (ValueError, yaml.YAMLError) as exc: + checks = list(iter_checks_from_data(data)) + except (ValueError, AttributeError) as exc: errors.append(f"failed to read/parse {path}: {exc}") continue tests = data.get("tests") or {} @@ -221,11 +228,11 @@ def wiring_errors(suites_dir: Path = SUITES_DIR) -> list[str]: if not isinstance(requires, list): errors.append(f"{location}: missing requires (use [] for core checks)") elif any( - not isinstance(requirement, str) or requirement not in REQUIREMENT_VOCABULARY + not isinstance(requirement, str) or requirement not in DECLARABLE_CAPABILITIES for requirement in requires ): errors.append( - f"{location}: requires must contain only: {', '.join(sorted(REQUIREMENT_VOCABULARY))}" + f"{location}: requires must contain only: {', '.join(sorted(DECLARABLE_CAPABILITIES))}" ) elif len(requires) != len(set(requires)): errors.append(f"{location}: requires must not contain duplicates") From bb2d30edb9f77f1c075c0051a92d22fe13be641c Mon Sep 17 00:00:00 2001 From: Alexandre Begnoche Date: Fri, 24 Jul 2026 17:16:05 -0400 Subject: [PATCH 29/34] chore: move tests Signed-off-by: Alexandre Begnoche --- isvctl/configs/suites/storage.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/isvctl/configs/suites/storage.yaml b/isvctl/configs/suites/storage.yaml index edc568631..e5ee24537 100644 --- a/isvctl/configs/suites/storage.yaml +++ b/isvctl/configs/suites/storage.yaml @@ -236,15 +236,6 @@ tests: requires: [vm, bare_metal] step: multipath - # ─── Teardown ──────────────────────────────────────────────────── - teardown_checks: - step: teardown_volume - checks: - StepSuccessCheck: - test_id: "N/A" - labels: ["storage"] - requires: [vm, bare_metal] - # Kubernetes storage checks use a normal provider-owned cluster fixture. k8s_storage: step: setup_cluster @@ -464,5 +455,14 @@ tests: namespace_prefix: "isvtest-fs-posix" timeout: 3600 + # ─── Teardown tests ────────────────────────────────────────────── + teardown_checks: + step: teardown_volume + checks: + StepSuccessCheck: + test_id: "N/A" + labels: ["storage"] + requires: [vm, bare_metal] + exclude: labels: [] From 7a012db4c1a70bbcacee4ce7efd97e2f857fd35f Mon Sep 17 00:00:00 2001 From: Alexandre Begnoche Date: Mon, 27 Jul 2026 16:22:07 -0400 Subject: [PATCH 30/34] refactor: state the requires rule and suite-name canonicalization once Two rules had each been written out four times. `requires` validation (a list, drawn from DECLARABLE_CAPABILITIES, no duplicates) lived in StepConfig.validate_requires, in ValidationConfig.validate_suite_shape, in _validate_entry_shape, and in the wiring script -- with four separately maintained message strings, so "requires must contain only" and "requires must be a list containing only" were the same verdict spelled two ways. requires_error() states it next to the vocabulary it validates; each caller adds only its own prefix. The pre-commit hook and the runtime validator can no longer disagree about what a valid suite file is. Suite-name canonicalization was worse, because suite name is the join key between a test run and its catalog entries: `_normalize_name` did lower + `-`/`_` + the k8s alias, `catalog.py` did the replace alone in two places, and the wiring script hardcoded `if stem == "k8s"`. Adding a second aliased filename meant finding all four. canonical_suite_name() in isvtest.core.resolution is now the single authority -- isvctl already depends on isvtest, so nothing new is coupled. Alongside those, three smaller consolidations: iter_checks_from_data existed twice, differing only in whether the category is yielded and in a branch the script's copy had dropped (dict-form categories without a `checks` key). The three-tuple version in isvtest.catalog is now the only walker of the wiring shape; no suite YAML uses the missing branch today, so the script only gains coverage. catalog_document parsed the suites directory three times -- once via _build_suite_map, then again in each vocabulary builder. _iter_suite_docs makes it one pass, and suite_vocabularies returns both lists from it. CONFIGS_ROOT moves from cli/test.py to config/suite_resolution.py, so `deploy` no longer imports the whole `test` command module -- typer app, orchestrator and all -- to obtain one Path. _suite_name gains @cache, since classifying a config merges it with the suite it imports and a single run asks both resolve_suite and resolve_suite_name for it. test_capability_step_gating copied the run/skip cascade out of _apply_capability_step_gates into its own _gated_step_names, so it would have kept passing against the old rule if the real gate changed. It now calls the gate. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Alexandre Begnoche --- isvctl/src/isvctl/cli/deploy.py | 3 +- isvctl/src/isvctl/cli/test.py | 2 +- isvctl/src/isvctl/config/schema.py | 20 ++-- isvctl/src/isvctl/config/suite_resolution.py | 20 ++-- isvctl/tests/test_capability_step_gating.py | 19 ++-- isvctl/tests/test_schema.py | 2 +- isvtest/src/isvtest/catalog.py | 102 +++++++++++-------- isvtest/src/isvtest/core/resolution.py | 40 ++++++-- scripts/tests/test_validate_suite_wiring.py | 2 +- scripts/validate_suite_wiring.py | 45 ++------ 10 files changed, 127 insertions(+), 128 deletions(-) diff --git a/isvctl/src/isvctl/cli/deploy.py b/isvctl/src/isvctl/cli/deploy.py index 7b0743e63..53552a2a5 100644 --- a/isvctl/src/isvctl/cli/deploy.py +++ b/isvctl/src/isvctl/cli/deploy.py @@ -32,8 +32,7 @@ from isvctl.cli import setup_logging from isvctl.cli.common import get_output_dir, print_error, print_progress, print_step, print_warning -from isvctl.cli.test import CONFIGS_ROOT -from isvctl.config.suite_resolution import resolve_suite_name +from isvctl.config.suite_resolution import CONFIGS_ROOT, resolve_suite_name from isvctl.orchestrator.loop import Phase from isvctl.remote import SCPTransfer, SSHClient, TarArchive from isvctl.remote.archive import DEFAULT_EXCLUDES as DEFAULT_ARCHIVE_EXCLUDES diff --git a/isvctl/src/isvctl/cli/test.py b/isvctl/src/isvctl/cli/test.py index b711e30c0..ccd67d57b 100644 --- a/isvctl/src/isvctl/cli/test.py +++ b/isvctl/src/isvctl/cli/test.py @@ -49,6 +49,7 @@ from isvctl.config.merger import merge_yaml_files from isvctl.config.schema import RunConfig from isvctl.config.suite_resolution import ( + CONFIGS_ROOT, SuiteResolutionError, parse_capability, resolve_suite, @@ -58,7 +59,6 @@ from isvctl.reporting import check_upload_credentials, create_test_run, get_environment_config, update_test_run logger = logging.getLogger(__name__) -CONFIGS_ROOT = Path(__file__).resolve().parents[3] / "configs" CORE_REQUIREMENT_CONTEXT = "core" diff --git a/isvctl/src/isvctl/config/schema.py b/isvctl/src/isvctl/config/schema.py index 24960ae02..2039ffafd 100644 --- a/isvctl/src/isvctl/config/schema.py +++ b/isvctl/src/isvctl/config/schema.py @@ -24,7 +24,7 @@ from typing import Any -from isvtest.core.resolution import DECLARABLE_CAPABILITIES, parse_validations +from isvtest.core.resolution import DECLARABLE_CAPABILITIES, parse_validations, requires_error from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator @@ -118,10 +118,9 @@ class StepConfig(BaseModel): @classmethod def validate_requires(cls, requires: list[str]) -> list[str]: """Keep step requirements in the same vocabulary as validation requirements.""" - if any(requirement not in DECLARABLE_CAPABILITIES for requirement in requires): - raise ValueError(f"requires must contain only: {', '.join(sorted(DECLARABLE_CAPABILITIES))}") - if len(requires) != len(set(requires)): - raise ValueError("requires must not contain duplicates") + message = requires_error(requires) + if message: + raise ValueError(message) return requires @@ -409,14 +408,9 @@ def validate_suite_shape(self) -> "ValidationConfig": raw_requires = entry.params_template.get("requires") if raw_requires is None: continue - if not isinstance(raw_requires, list) or any( - not isinstance(value, str) or value not in DECLARABLE_CAPABILITIES for value in raw_requires - ): - raise ValueError( - f"requires must be a list containing only: {', '.join(sorted(DECLARABLE_CAPABILITIES))}" - ) - if len(raw_requires) != len(set(raw_requires)): - raise ValueError("requires must not contain duplicates") + message = requires_error(raw_requires) + if message: + raise ValueError(message) return self diff --git a/isvctl/src/isvctl/config/suite_resolution.py b/isvctl/src/isvctl/config/suite_resolution.py index 89b4841a3..c45208f00 100644 --- a/isvctl/src/isvctl/config/suite_resolution.py +++ b/isvctl/src/isvctl/config/suite_resolution.py @@ -9,9 +9,15 @@ from typing import Any import yaml +from isvtest.core.resolution import canonical_suite_name as _normalize_name from isvctl.config.merger import merge_yaml_files +# The canonical suite/provider config tree shipped with isvctl. Lives here +# rather than on a CLI command module so every entry point (test, deploy, ...) +# sources it from the config layer that consumes it. +CONFIGS_ROOT = Path(__file__).resolve().parents[3] / "configs" + class SuiteResolutionError(Exception): """Raised when a suite selection cannot be resolved unambiguously.""" @@ -26,12 +32,6 @@ class ResolvedSuite: platform: str | None -def _normalize_name(value: str) -> str: - """Normalize CLI and filename spellings to catalog suite names.""" - normalized = value.strip().lower().replace("-", "_") - return "kubernetes" if normalized == "k8s" else normalized - - @cache def platform_vocabulary(configs_root: Path) -> frozenset[str]: """Return declarable capabilities from canonical platform suite YAML. @@ -97,8 +97,14 @@ def _raw_imports(config_path: Path) -> list[str]: return [] +@cache def _suite_name(config_path: Path, declarable: frozenset[str]) -> tuple[str, str | None]: - """Return the logical suite name and optional platform key for a config.""" + """Return the logical suite name and optional platform key for a config. + + Cached: classifying a config merges it with the suite it imports, and a + single ``isvctl test run`` asks for the same config's identity from both + ``resolve_suite`` and ``resolve_suite_name``. + """ merged = merge_yaml_files([config_path]) tests = merged.get("tests") or {} platform = tests.get("platform") if isinstance(tests, dict) else None diff --git a/isvctl/tests/test_capability_step_gating.py b/isvctl/tests/test_capability_step_gating.py index 2b76614fb..ce4376666 100644 --- a/isvctl/tests/test_capability_step_gating.py +++ b/isvctl/tests/test_capability_step_gating.py @@ -23,12 +23,12 @@ DECLARABLE_CAPABILITIES, ValidationEntry, parse_validations, - requirements_satisfied, ) from isvctl.cli.test import CORE_REQUIREMENT_CONTEXT from isvctl.config.merger import merge_yaml_files from isvctl.config.schema import RunConfig +from isvctl.orchestrator.loop import _apply_capability_step_gates CONFIGS_ROOT = Path(__file__).resolve().parents[1] / "configs" PROVIDERS = ("aws", "my-isv") @@ -53,16 +53,13 @@ def _plain_suite_configs() -> list[tuple[str, Path]]: def _gated_step_names(steps: list[Any], entries: list[ValidationEntry], context: str) -> set[str]: - """Return the steps `_apply_capability_step_gates` would skip in a context.""" - gated: set[str] = set() - for step in steps: - if step.requires and not requirements_satisfied(step.requires, context): - gated.add(step.name) - continue - bound = [entry for entry in entries if entry.step == step.name] - if bound and all(not requirements_satisfied(entry.requires, context) for entry in bound): - gated.add(step.name) - return gated + """Return the steps `_apply_capability_step_gates` skips in a context. + + Asks the real gate rather than restating its rules, so a change to what + gets skipped is exercised here instead of silently passing against a copy. + """ + gated = _apply_capability_step_gates(steps, entries, context) + return {after.name for before, after in zip(steps, gated, strict=True) if after.skip and not before.skip} def _unguarded_references(value: Any) -> set[str]: diff --git a/isvctl/tests/test_schema.py b/isvctl/tests/test_schema.py index 7eb616290..cac241ec1 100644 --- a/isvctl/tests/test_schema.py +++ b/isvctl/tests/test_schema.py @@ -107,7 +107,7 @@ def test_full_step(self) -> None: def test_step_rejects_unknown_or_duplicate_requires(self) -> None: """Step requirements use the declarable capability vocabulary.""" - with pytest.raises(ValidationError, match="requires must contain only"): + with pytest.raises(ValidationError, match="requires must be a list containing only"): StepConfig(name="setup", command="echo", requires=["compute"]) with pytest.raises(ValidationError, match="requires must not contain duplicates"): StepConfig(name="setup", command="echo", requires=["vm", "vm"]) diff --git a/isvtest/src/isvtest/catalog.py b/isvtest/src/isvtest/catalog.py index 671d1bb04..d553981dd 100644 --- a/isvtest/src/isvtest/catalog.py +++ b/isvtest/src/isvtest/catalog.py @@ -33,7 +33,7 @@ from isvreporter.version import get_version from isvtest.core.discovery import discover_all_tests -from isvtest.core.resolution import DECLARABLE_CAPABILITIES, resolve_class_key +from isvtest.core.resolution import DECLARABLE_CAPABILITIES, canonical_suite_name, resolve_class_key from isvtest.release_manifest import INCLUDE_UNRELEASED_ENV, load_released_test_filter logger = logging.getLogger(__name__) @@ -68,37 +68,40 @@ def iter_config_checks(config_path: Path) -> Iterator[tuple[str, dict[str, Any]] data = yaml.safe_load(config_path.read_text()) except Exception: return - yield from iter_checks_from_data(data) + for _, name, params in iter_checks_from_data(data): + yield name, params -def iter_checks_from_data(data: Any) -> Iterator[tuple[str, dict[str, Any]]]: - """Yield ``(check_name, params)`` from an already-parsed config document. +def iter_checks_from_data(data: Any) -> Iterator[tuple[str, str, dict[str, Any]]]: + """Yield ``(category, check_name, params)`` from an already-parsed document. Lets callers that have parsed the YAML for other reasons avoid a second - read and parse of the same file. + read and parse of the same file. This is the only walker of the wiring + shape - the catalog, the suite map, and the wiring validator all read it + here, so a new nesting form is taught to the codebase once. """ validations = (data or {}).get("tests", {}).get("validations", {}) if not isinstance(validations, dict): return - def _from_mapping(mapping: Any) -> Iterator[tuple[str, dict[str, Any]]]: + def _from_mapping(category: str, mapping: Any) -> Iterator[tuple[str, str, dict[str, Any]]]: if isinstance(mapping, dict): for name, params in mapping.items(): - yield name, params if isinstance(params, dict) else {} + yield category, name, params if isinstance(params, dict) else {} - for cat_config in validations.values(): + for category, cat_config in validations.items(): if isinstance(cat_config, dict) and "checks" in cat_config: checks_val = cat_config["checks"] if isinstance(checks_val, dict): - yield from _from_mapping(checks_val) + yield from _from_mapping(category, checks_val) elif isinstance(checks_val, list): for check in checks_val: - yield from _from_mapping(check) + yield from _from_mapping(category, check) elif isinstance(cat_config, dict): - yield from _from_mapping(cat_config) + yield from _from_mapping(category, cat_config) elif isinstance(cat_config, list): for check in cat_config: - yield from _from_mapping(check) + yield from _from_mapping(category, check) def _extract_checks_from_config(config_path: Path) -> list[str]: @@ -200,25 +203,35 @@ def build_label_file_map() -> dict[str, set[str]]: return label_files +def _iter_suite_docs() -> Iterator[tuple[Path, dict[str, Any]]]: + """Yield ``(path, document)`` for each canonical suite YAML, parsed once.""" + configs_dir = _find_configs_dir() + if not configs_dir: + logger.warning("Could not locate isvctl/configs/ directory") + return + for config_path in sorted((configs_dir / "suites").glob("*.yaml")): + yield config_path, yaml.safe_load(config_path.read_text(encoding="utf-8")) or {} + + +def _declared_platform(data: dict[str, Any]) -> str | None: + """Return the platform key a suite document declares, or None for a plain suite.""" + tests = data.get("tests") or {} + platform = tests.get("platform") if isinstance(tests, dict) else None + return platform if isinstance(platform, str) and platform else None + + def _build_suite_map() -> dict[str, dict[str, Any]]: """Map suite wiring names to suite placement and requirements. Duplicate wiring names currently last-wins. Global uniqueness enforcement is deferred to a follow-up PR (``ISVCTL_ENFORCE_UNIQUE_WIRING=1``). """ - configs_dir = _find_configs_dir() - if not configs_dir: - logger.warning("Could not locate isvctl/configs/ directory") - return {} - enforce_unique = os.environ.get("ISVCTL_ENFORCE_UNIQUE_WIRING") == "1" suite_map: dict[str, dict[str, Any]] = {} - for config_path in sorted((configs_dir / "suites").glob("*.yaml")): - data = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {} - tests = data.get("tests") or {} - platform = tests.get("platform") if isinstance(tests, dict) else None - suite = str(platform) if isinstance(platform, str) and platform else config_path.stem.replace("-", "_") - for check_name, params in iter_checks_from_data(data): + for config_path, data in _iter_suite_docs(): + platform = _declared_platform(data) + suite = platform if platform else canonical_suite_name(config_path.stem) + for _, check_name, params in iter_checks_from_data(data): if check_name in suite_map and enforce_unique: raise ValueError(f"Suite wiring name {check_name!r} is not globally unique") requires = params.get("requires", []) @@ -314,31 +327,33 @@ def build_catalog(*, released_only: bool = True) -> list[dict[str, Any]]: return catalog +def suite_vocabularies() -> tuple[list[str], list[str]]: + """Return ``(capabilities, plain_suites)`` from one pass over the suite YAML. + + A suite file is either a platform suite (it declares a declarable + capability) or a plain suite named after its file - one classification, so + the two vocabularies are read off a single parse of the directory rather + than a scan each. + """ + capabilities: set[str] = set() + suites: set[str] = set() + for config_path, data in _iter_suite_docs(): + platform = _declared_platform(data) + if platform in DECLARABLE_CAPABILITIES: + capabilities.add(str(platform)) + else: + suites.add(canonical_suite_name(config_path.stem)) + return sorted(capabilities), sorted(suites) + + def build_capability_vocabulary() -> list[str]: """Return declarable capabilities derived from platform suite YAML.""" - suite_map = _build_suite_map() - return sorted( - platform - for platform in {entry["platform"] for entry in suite_map.values() if entry["platform"]} - if platform in DECLARABLE_CAPABILITIES - ) + return suite_vocabularies()[0] def build_suite_vocabulary() -> list[str]: """Return plain suite names declared by canonical suite YAML.""" - configs_dir = _find_configs_dir() - if not configs_dir: - return [] - - suites: set[str] = set() - for config_path in sorted((configs_dir / "suites").glob("*.yaml")): - data = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {} - tests = data.get("tests") or {} - platform = tests.get("platform") if isinstance(tests, dict) else None - if isinstance(platform, str) and platform in DECLARABLE_CAPABILITIES: - continue - suites.add(config_path.stem.replace("-", "_")) - return sorted(suites) + return suite_vocabularies()[1] def _assert_disjoint_vocabulary(platforms: list[str], suites: list[str]) -> None: @@ -369,8 +384,7 @@ def catalog_document(entries: list[dict[str, Any]], version: str) -> dict[str, A ``labels`` are intentionally not summarized at the top level - a consumer can derive the label universe from the entries when needed. """ - platforms = build_capability_vocabulary() - suites = build_suite_vocabulary() + platforms, suites = suite_vocabularies() _assert_disjoint_vocabulary(platforms, suites) return { "schemaVersion": CATALOG_SCHEMA_VERSION, diff --git a/isvtest/src/isvtest/core/resolution.py b/isvtest/src/isvtest/core/resolution.py index c247f88d3..afbcd01b1 100644 --- a/isvtest/src/isvtest/core/resolution.py +++ b/isvtest/src/isvtest/core/resolution.py @@ -36,6 +36,34 @@ DECLARABLE_CAPABILITIES = frozenset({"vm", "bare_metal", "kubernetes", "slurm"}) +def requires_error(values: Any) -> str | None: + """Return why ``values`` is not a valid ``requires`` list, or None when it is. + + One statement of the rule for every place that enforces it - the pydantic + step and suite validators, the runtime entry-shape check, and the suite + wiring script - so adding a capability or relaxing the rule is a single + edit and the four call sites cannot report different verdicts. + """ + if not isinstance(values, list) or any( + not isinstance(value, str) or value not in DECLARABLE_CAPABILITIES for value in values + ): + return f"requires must be a list containing only: {', '.join(sorted(DECLARABLE_CAPABILITIES))}" + if len(values) != len(set(values)): + return "requires must not contain duplicates" + return None + + +def canonical_suite_name(value: str) -> str: + """Normalize a CLI spelling, filename stem, or platform key to a suite name. + + Suite name is the join key between a test run and its catalog entries, so + the producer, the CLI resolver, and the wiring validator all have to spell + it the same way - including the ``k8s`` filename alias. + """ + normalized = value.strip().lower().replace("-", "_") + return "kubernetes" if normalized == "k8s" else normalized + + class State(StrEnum): """Terminal state of a validation in the report.""" @@ -455,15 +483,9 @@ def _validate_entry_shape(entry: ValidationEntry) -> str | None: return str(invalid_message) raw_requires = entry.params_template.get("requires") if raw_requires is not None: - if not isinstance(raw_requires, list) or any( - not isinstance(value, str) or value not in DECLARABLE_CAPABILITIES for value in raw_requires - ): - return ( - f"validation '{entry.name}' requires must be a list containing only: " - f"{', '.join(sorted(DECLARABLE_CAPABILITIES))}" - ) - if len(raw_requires) != len(set(raw_requires)): - return f"validation '{entry.name}' requires must not contain duplicates" + message = requires_error(raw_requires) + if message: + return f"validation '{entry.name}' {message}" return None diff --git a/scripts/tests/test_validate_suite_wiring.py b/scripts/tests/test_validate_suite_wiring.py index 6c7798e96..b860de059 100644 --- a/scripts/tests/test_validate_suite_wiring.py +++ b/scripts/tests/test_validate_suite_wiring.py @@ -136,7 +136,7 @@ def test_plain_suite_requires_are_explicit_and_valid(tmp_path: Path) -> None: errors = validate_suite_wiring.wiring_errors(tmp_path) assert any("MissingCheck" in error and "missing requires" in error for error in errors) - assert any("InvalidCheck" in error and "requires must contain only" in error for error in errors) + assert any("InvalidCheck" in error and "requires must be a list containing only" in error for error in errors) def test_wiring_errors_rejects_plain_suite_named_after_capability(tmp_path: Path) -> None: diff --git a/scripts/validate_suite_wiring.py b/scripts/validate_suite_wiring.py index 9d1b3811e..025939e3f 100644 --- a/scripts/validate_suite_wiring.py +++ b/scripts/validate_suite_wiring.py @@ -42,7 +42,8 @@ from typing import Any import yaml -from isvtest.core.resolution import DECLARABLE_CAPABILITIES +from isvtest.catalog import iter_checks_from_data +from isvtest.core.resolution import DECLARABLE_CAPABILITIES, canonical_suite_name, requires_error REPO_ROOT = Path(__file__).resolve().parent.parent SUITES_DIR = REPO_ROOT / "isvctl" / "configs" / "suites" @@ -96,9 +97,7 @@ def _normalize_test_id(value: Any) -> str | None: def required_suite_label(config_path: Path) -> str | None: """Return the label every check in a known canonical suite must carry.""" - if config_path.stem == "k8s": - return "kubernetes" - return config_path.stem.replace("-", "_") + return canonical_suite_name(config_path.stem) def iter_suite_checks(config_path: Path) -> Iterator[tuple[str, str, dict[str, Any]]]: @@ -110,31 +109,6 @@ def iter_suite_checks(config_path: Path) -> Iterator[tuple[str, str, dict[str, A yield from iter_checks_from_data(data) -def iter_checks_from_data(data: Any) -> Iterator[tuple[str, str, dict[str, Any]]]: - """Yield ``(category, check_name, params)`` from an already-parsed suite doc.""" - validations = (data or {}).get("tests", {}).get("validations", {}) - if not isinstance(validations, dict): - return - - def _from_mapping(category: str, mapping: Any) -> Iterator[tuple[str, str, dict[str, Any]]]: - """Yield wired checks from a dict- or list-form ``checks`` mapping.""" - if isinstance(mapping, dict): - for name, params in mapping.items(): - yield category, name, params if isinstance(params, dict) else {} - - for category, cat_config in validations.items(): - if isinstance(cat_config, dict) and "checks" in cat_config: - checks_val = cat_config["checks"] - if isinstance(checks_val, dict): - yield from _from_mapping(category, checks_val) - elif isinstance(checks_val, list): - for check in checks_val: - yield from _from_mapping(category, check) - elif isinstance(cat_config, list): - for check in cat_config: - yield from _from_mapping(category, check) - - def _format_location(config_path: Path, category: str, check_name: str, line_number: int | None) -> str: """Return a stable location string for error messages.""" try: @@ -188,7 +162,7 @@ def wiring_errors(suites_dir: Path = SUITES_DIR) -> list[str]: errors.append(f"{path}: tests.platform must be one of: {', '.join(sorted(DECLARABLE_CAPABILITIES))}") suite_is_platform = isinstance(platform, str) and platform in DECLARABLE_CAPABILITIES if not suite_is_platform: - suite_name = path.stem.replace("-", "_") + suite_name = canonical_suite_name(path.stem) if suite_name in DECLARABLE_CAPABILITIES: errors.append( f"{path}: plain suite name {suite_name!r} collides with a declarable " @@ -227,15 +201,8 @@ def wiring_errors(suites_dir: Path = SUITES_DIR) -> list[str]: requires = params.get("requires") if not isinstance(requires, list): errors.append(f"{location}: missing requires (use [] for core checks)") - elif any( - not isinstance(requirement, str) or requirement not in DECLARABLE_CAPABILITIES - for requirement in requires - ): - errors.append( - f"{location}: requires must contain only: {', '.join(sorted(DECLARABLE_CAPABILITIES))}" - ) - elif len(requires) != len(set(requires)): - errors.append(f"{location}: requires must not contain duplicates") + elif message := requires_error(requires): + errors.append(f"{location}: {message}") else: dead = sorted(set(requires) - declared_platforms) if dead: From 8919c808cbe66fd2c94d464b542921bde02ffe79 Mon Sep 17 00:00:00 2001 From: Alexandre Begnoche Date: Mon, 27 Jul 2026 17:10:13 -0400 Subject: [PATCH 31/34] fix(tests): strip rich styling before matching the rejected option name Typer forces a terminal under GITHUB_ACTIONS, so the reported option name arrives spliced with escape codes and the substring match fails in CI while passing locally. Signed-off-by: Alexandre Begnoche --- isvctl/tests/test_test_cli_labels.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/isvctl/tests/test_test_cli_labels.py b/isvctl/tests/test_test_cli_labels.py index 38a26808a..e11e241ba 100644 --- a/isvctl/tests/test_test_cli_labels.py +++ b/isvctl/tests/test_test_cli_labels.py @@ -16,6 +16,7 @@ """Tests for isvctl test CLI label filtering.""" import json +import re from pathlib import Path from typing import Any, ClassVar @@ -27,6 +28,8 @@ runner = CliRunner() +_ANSI_ESCAPE = re.compile(r"\x1b\[[0-9;]*m") + def _write_config(tmp_path: Path) -> Path: """Write a minimal isvctl test config and return its path.""" @@ -574,9 +577,13 @@ def test_unknown_option_before_separator_is_rejected(monkeypatch: pytest.MonkeyP ["run", "-f", str(config), "--platform", "k8s", "--no-upload", "--dry-run"], ) - assert result.exit_code != 0, result.output - assert "No such option" in result.output or "no such option" in result.output.lower() - assert "--platform" in result.output + # Typer forces rich styling under GITHUB_ACTIONS, which splices escape + # codes into the middle of the reported option name. + output = _ANSI_ESCAPE.sub("", result.output) + + assert result.exit_code != 0, output + assert "No such option" in output or "no such option" in output.lower() + assert "--platform" in output assert _FakeOrchestrator.calls == [] From 8f2030a1239d7fafc1adfa90c69ca8d49007ba03 Mon Sep 17 00:00:00 2001 From: Alexandre Begnoche Date: Mon, 27 Jul 2026 17:10:14 -0400 Subject: [PATCH 32/34] fix: name only the four declarable capabilities in schema and docs tests.platform now rejects anything outside vm, bare_metal, kubernetes and slurm, but its field description (and the generated JSON schema) still listed the old vocabulary, and four config examples declared values that raise at load. Plain suites omit the key entirely. Signed-off-by: Alexandre Begnoche --- docs/guides/configuration.md | 1 - docs/guides/external-validation-guide.md | 1 - docs/guides/local-development.md | 1 - .../aws/scripts/control-plane/docs/aws-control-plane.md | 1 - isvctl/configs/suites/README.md | 4 ++-- isvctl/schemas/config.schema.json | 2 +- isvctl/src/isvctl/config/schema.py | 4 ++-- 7 files changed, 5 insertions(+), 9 deletions(-) diff --git a/docs/guides/configuration.md b/docs/guides/configuration.md index ad1a05f70..2912e150f 100644 --- a/docs/guides/configuration.md +++ b/docs/guides/configuration.md @@ -124,7 +124,6 @@ commands: timeout: 300 tests: - platform: network cluster_name: "aws-network-test" settings: diff --git a/docs/guides/external-validation-guide.md b/docs/guides/external-validation-guide.md index 399b3b900..ff2726d28 100644 --- a/docs/guides/external-validation-guide.md +++ b/docs/guides/external-validation-guide.md @@ -126,7 +126,6 @@ commands: timeout: 300 tests: - platform: myplatform cluster_name: "my-validation" settings: diff --git a/docs/guides/local-development.md b/docs/guides/local-development.md index ef6d80ef6..c81d9ef17 100644 --- a/docs/guides/local-development.md +++ b/docs/guides/local-development.md @@ -149,7 +149,6 @@ commands: timeout: 60 tests: - platform: network cluster_name: "local-test" settings: region: "us-west-2" diff --git a/isvctl/configs/providers/aws/scripts/control-plane/docs/aws-control-plane.md b/isvctl/configs/providers/aws/scripts/control-plane/docs/aws-control-plane.md index e1323eab7..ca8378039 100644 --- a/isvctl/configs/providers/aws/scripts/control-plane/docs/aws-control-plane.md +++ b/isvctl/configs/providers/aws/scripts/control-plane/docs/aws-control-plane.md @@ -247,7 +247,6 @@ commands: # ... more steps ... tests: - platform: control_plane settings: region: "us-west-2" services: "ec2,s3,iam,sts" diff --git a/isvctl/configs/suites/README.md b/isvctl/configs/suites/README.md index 8feee6303..e801dd0c3 100644 --- a/isvctl/configs/suites/README.md +++ b/isvctl/configs/suites/README.md @@ -11,8 +11,8 @@ commands (steps + scripts) that produce JSON for the validations to check. ## What runs, and when -A **platform suite** (`platform: ` — `vm`, `bare_metal`, `k8s`, -`slurm`) is the obligation attached to declaring that capability. Its checks +A **platform suite** (`tests.platform: ` — `vm`, `bare_metal`, +`kubernetes`, `slurm`) is the obligation attached to declaring that capability. Its checks declare no `requires:`; they all run. A **plain suite** (everything else — `storage`, `network`, ...) mixes checks diff --git a/isvctl/schemas/config.schema.json b/isvctl/schemas/config.schema.json index e6acb8120..b48861131 100644 --- a/isvctl/schemas/config.schema.json +++ b/isvctl/schemas/config.schema.json @@ -242,7 +242,7 @@ } ], "default": null, - "description": "Platform type: KUBERNETES, SLURM, BARE_METAL, CONTROL_PLANE, IAM, NETWORK, SECURITY, VM, IMAGE_REGISTRY, OBSERVABILITY, STORAGE", + "description": "Capability a platform suite declares: vm, bare_metal, kubernetes, slurm. Plain suites omit it and gate each check with requires.", "title": "Platform" }, "settings": { diff --git a/isvctl/src/isvctl/config/schema.py b/isvctl/src/isvctl/config/schema.py index 2039ffafd..bb8eb747d 100644 --- a/isvctl/src/isvctl/config/schema.py +++ b/isvctl/src/isvctl/config/schema.py @@ -377,8 +377,8 @@ class ValidationConfig(BaseModel): platform: str | None = Field( default=None, description=( - "Platform type: KUBERNETES, SLURM, BARE_METAL, CONTROL_PLANE, IAM, NETWORK, " - "SECURITY, VM, IMAGE_REGISTRY, OBSERVABILITY, STORAGE" + "Capability a platform suite declares: vm, bare_metal, kubernetes, slurm. " + "Plain suites omit it and gate each check with requires." ), ) settings: dict[str, Any] = Field(default_factory=dict, description="Test settings") From 738080088ee53e60a6272fc36edbfeea73f480d0 Mon Sep 17 00:00:00 2001 From: Alexandre Begnoche Date: Mon, 27 Jul 2026 17:10:15 -0400 Subject: [PATCH 33/34] test: gate every discovered provider and document the new helpers The gating invariant listed aws and my-isv by name, so nico's plain suites escaped it; discovering the config directories keeps a new provider covered. Signed-off-by: Alexandre Begnoche --- isvctl/tests/test_capability_step_gating.py | 8 +++++--- isvctl/tests/test_reporting.py | 5 ++++- isvreporter/tests/test_client.py | 6 +++++- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/isvctl/tests/test_capability_step_gating.py b/isvctl/tests/test_capability_step_gating.py index ce4376666..3adde0416 100644 --- a/isvctl/tests/test_capability_step_gating.py +++ b/isvctl/tests/test_capability_step_gating.py @@ -31,7 +31,7 @@ from isvctl.orchestrator.loop import _apply_capability_step_gates CONFIGS_ROOT = Path(__file__).resolve().parents[1] / "configs" -PROVIDERS = ("aws", "my-isv") +PROVIDERS_ROOT = CONFIGS_ROOT / "providers" # Every context a plain suite can be run under, including the core-only default # that `--suite NAME` (no `--capability`) selects. CONTEXTS = (CORE_REQUIREMENT_CONTEXT, *sorted(DECLARABLE_CAPABILITIES)) @@ -41,8 +41,10 @@ def _plain_suite_configs() -> list[tuple[str, Path]]: """Return (provider, config path) for every plain-suite provider config.""" configs: list[tuple[str, Path]] = [] - for provider in PROVIDERS: - for path in sorted((CONFIGS_ROOT / "providers" / provider / "config").glob("*.yaml")): + # Discovered rather than listed so a new provider inherits the guarantee. + for config_dir in sorted(PROVIDERS_ROOT.glob("*/config")): + provider = config_dir.parent.name + for path in sorted(config_dir.glob("*.yaml")): config = RunConfig.model_validate(merge_yaml_files([str(path)])) # Platform suites carry no `requires:` on their checks, so nothing # is ever gated inside them. diff --git a/isvctl/tests/test_reporting.py b/isvctl/tests/test_reporting.py index 22b832252..752202aab 100644 --- a/isvctl/tests/test_reporting.py +++ b/isvctl/tests/test_reporting.py @@ -16,6 +16,7 @@ """Tests for reporting module.""" import os +from typing import Any from unittest.mock import MagicMock, patch from isvctl.cli.test import CORE_REQUIREMENT_CONTEXT, _reported_capability @@ -186,7 +187,8 @@ class TestReportedCapability: @staticmethod def _config(platform: str | None) -> RunConfig: - raw: dict = {"commands": {}, "tests": {"validations": {}}} + """Return a minimal config, declaring `platform` only for a platform suite.""" + raw: dict[str, Any] = {"commands": {}, "tests": {"validations": {}}} if platform: raw["tests"]["platform"] = platform return RunConfig.model_validate(raw) @@ -206,6 +208,7 @@ def test_platform_suite_reports_its_own_platform(self) -> None: assert _reported_capability(config, None) == "vm" def test_explicit_capability_wins(self) -> None: + """An explicit `--capability` is reported as-is, whatever the suite declares.""" config = self._config(None) assert _reported_capability(config, "vm") == "vm" assert _reported_capability(self._config("vm"), "kubernetes") == "kubernetes" diff --git a/isvreporter/tests/test_client.py b/isvreporter/tests/test_client.py index 01882db4c..d2b4c55fb 100644 --- a/isvreporter/tests/test_client.py +++ b/isvreporter/tests/test_client.py @@ -17,6 +17,7 @@ import json from pathlib import Path +from typing import Any from unittest.mock import MagicMock, patch from isvreporter.client import calculate_duration, create_test_run, load_test_run_id @@ -95,12 +96,14 @@ class TestCreateTestRunPayload: """The create payload has to carry both axes of what the run exercised.""" @staticmethod - def _posted_payload(mock_urlopen: MagicMock) -> dict: + def _posted_payload(mock_urlopen: MagicMock) -> dict[str, Any]: + """Return the JSON body of the request the client posted.""" request = mock_urlopen.call_args[0][0] return json.loads(request.data.decode()) @staticmethod def _response() -> MagicMock: + """Return a context-manager response yielding a created test run id.""" response = MagicMock() response.read.return_value = json.dumps({"data": {"testRunId": 42}}).encode() response.__enter__ = lambda self: self @@ -109,6 +112,7 @@ def _response() -> MagicMock: @patch("isvreporter.client.urlopen") def test_sends_suite_and_capability(self, mock_urlopen: MagicMock, tmp_path: Path) -> None: + """Both axes of the run reach the service in the create payload.""" mock_urlopen.return_value = self._response() with ( From 8a9d214b003ed2f5881f6b74b6e44a3a356bfc9f Mon Sep 17 00:00:00 2001 From: Alexandre Begnoche Date: Mon, 27 Jul 2026 17:10:15 -0400 Subject: [PATCH 34/34] build: stop warning about the overridden demo-image-registry recipe The explicit two-capability rule overrode the static pattern rule, so every make invocation printed an overriding-recipe warning. Signed-off-by: Alexandre Begnoche --- Makefile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 165982e42..5904512a3 100644 --- a/Makefile +++ b/Makefile @@ -114,11 +114,13 @@ demo-all: ## Run all my-isv living examples (or demo- for one, e.g. demo- @echo "✅ All my-isv living examples passed in demo mode!" @echo "Suites: $(MY_ISV_SUITES)" -$(DEMO_TARGETS): demo-%: +# image-registry is excluded here because it has its own rule below; leaving it +# in would make every make invocation warn about an overridden recipe. +$(filter-out demo-image-registry,$(DEMO_TARGETS)): demo-%: $(call run_demo,$*,$(DEMO_CAP_$*)) # image-registry splits its gated checks between vm and bare_metal, so one run -# cannot reach both. This explicit rule overrides the pattern rule above. +# cannot reach both. demo-image-registry: $(call run_demo,image-registry,vm) $(call run_demo,image-registry,bare_metal)