diff --git a/checkup.schema.json b/checkup.schema.json index fd3b183..7511059 100644 --- a/checkup.schema.json +++ b/checkup.schema.json @@ -12,6 +12,14 @@ "type": "string" } }, + "select": { + "type": "string", + "description": "Default selector for materialized output." + }, + "exclude": { + "type": "string", + "description": "Default selector for materialization to exclude." + }, "providers": { "type": "array", "description": "Data providers for context enrichment", @@ -62,6 +70,26 @@ ], "additionalProperties": false }, + { + "type": "object", + "properties": { + "name": { + "const": "cruft" + }, + "project_path": { + "type": "string", + "default": "." + }, + "fetch_template": { + "type": "boolean", + "default": false + } + }, + "required": [ + "name" + ], + "additionalProperties": false + }, { "type": "object", "properties": { @@ -142,7 +170,14 @@ "type": "object", "properties": { "type": { - "const": "conveyor_is_dirty_deployment" + "const": "conveyor_clean_deployment" + }, + "tags": { + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" } }, "required": [ @@ -155,6 +190,13 @@ "properties": { "type": { "const": "conveyor_last_deployment_time" + }, + "tags": { + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" } }, "required": [ @@ -167,6 +209,217 @@ "properties": { "type": { "const": "conveyor_last_run_status" + }, + "tags": { + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "const": "cruft_commits_behind" + }, + "name": { + "default": "cruft_commits_behind", + "title": "Name", + "type": "string" + }, + "description": { + "default": "Number of template commits the project is behind", + "title": "Description", + "type": "string" + }, + "unit": { + "default": "commits", + "title": "Unit", + "type": "string" + }, + "tags": { + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "const": "cruft_conflicts" + }, + "name": { + "default": "cruft_conflicts", + "title": "Name", + "type": "string" + }, + "description": { + "default": "Number of unresolved cruft template conflicts", + "title": "Description", + "type": "string" + }, + "unit": { + "default": "files", + "title": "Unit", + "type": "string" + }, + "tags": { + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "const": "cruft_days_behind_template" + }, + "name": { + "default": "cruft_days_behind_template", + "title": "Name", + "type": "string" + }, + "description": { + "default": "Days between the pinned commit and the template head", + "title": "Description", + "type": "string" + }, + "unit": { + "default": "days", + "title": "Unit", + "type": "string" + }, + "tags": { + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "const": "cruft_days_since_update" + }, + "name": { + "default": "cruft_days_since_update", + "title": "Name", + "type": "string" + }, + "description": { + "default": "Days since the last cruft template update", + "title": "Description", + "type": "string" + }, + "unit": { + "default": "days", + "title": "Unit", + "type": "string" + }, + "tags": { + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "const": "cruft_linked" + }, + "name": { + "default": "cruft_linked", + "title": "Name", + "type": "string" + }, + "description": { + "default": "Whether a .cruft.json template link is present", + "title": "Description", + "type": "string" + }, + "unit": { + "default": "boolean", + "title": "Unit", + "type": "string" + }, + "tags": { + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "const": "cruft_up_to_date" + }, + "name": { + "default": "cruft_up_to_date", + "title": "Name", + "type": "string" + }, + "description": { + "default": "Whether the project matches the latest template commit", + "title": "Description", + "type": "string" + }, + "unit": { + "default": "boolean", + "title": "Unit", + "type": "string" + }, + "tags": { + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" } }, "required": [ @@ -194,6 +447,13 @@ "default": "percent", "title": "Unit", "type": "string" + }, + "tags": { + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" } }, "required": [ @@ -221,6 +481,13 @@ "default": "tests", "title": "Unit", "type": "string" + }, + "tags": { + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" } }, "required": [ @@ -248,6 +515,13 @@ "default": "columns", "title": "Unit", "type": "string" + }, + "tags": { + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" } }, "required": [ @@ -275,6 +549,13 @@ "default": "columns", "title": "Unit", "type": "string" + }, + "tags": { + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" } }, "required": [ @@ -302,6 +583,13 @@ "default": "columns", "title": "Unit", "type": "string" + }, + "tags": { + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" } }, "required": [ @@ -329,6 +617,13 @@ "default": "tests", "title": "Unit", "type": "string" + }, + "tags": { + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" } }, "required": [ @@ -357,6 +652,13 @@ "title": "Unit", "type": "string" }, + "tags": { + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" + }, "flagged_packages": { "items": { "type": "string" @@ -390,6 +692,13 @@ "default": "models", "title": "Unit", "type": "string" + }, + "tags": { + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" } }, "required": [ @@ -429,6 +738,13 @@ "default": "models", "title": "Unit", "type": "string" + }, + "tags": { + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" } }, "required": [ @@ -456,6 +772,13 @@ "default": "models", "title": "Unit", "type": "string" + }, + "tags": { + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" } }, "required": [ @@ -483,6 +806,13 @@ "default": "columns", "title": "Unit", "type": "string" + }, + "tags": { + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" } }, "required": [ @@ -510,6 +840,13 @@ "default": "models", "title": "Unit", "type": "string" + }, + "tags": { + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" } }, "required": [ @@ -537,6 +874,13 @@ "default": "models", "title": "Unit", "type": "string" + }, + "tags": { + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" } }, "required": [ @@ -564,6 +908,13 @@ "default": "models", "title": "Unit", "type": "string" + }, + "tags": { + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" } }, "required": [ @@ -591,6 +942,13 @@ "default": "models", "title": "Unit", "type": "string" + }, + "tags": { + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" } }, "required": [ @@ -619,6 +977,13 @@ "title": "Unit", "type": "string" }, + "tags": { + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" + }, "profile": { "anyOf": [ { @@ -662,6 +1027,13 @@ "title": "Unit", "type": "string" }, + "tags": { + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" + }, "min_version": { "title": "Min Version", "type": "string" @@ -692,6 +1064,13 @@ "default": "columns", "title": "Unit", "type": "string" + }, + "tags": { + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" } }, "required": [ @@ -719,6 +1098,13 @@ "default": "tests", "title": "Unit", "type": "string" + }, + "tags": { + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" } }, "required": [ @@ -746,6 +1132,13 @@ "default": "tests", "title": "Unit", "type": "string" + }, + "tags": { + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" } }, "required": [ @@ -773,6 +1166,13 @@ "default": "version", "title": "Unit", "type": "string" + }, + "tags": { + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" } }, "required": [ @@ -800,6 +1200,13 @@ "default": "days", "title": "Unit", "type": "string" + }, + "tags": { + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" } }, "required": [ @@ -828,6 +1235,13 @@ "title": "Unit", "type": "string" }, + "tags": { + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" + }, "pattern": { "default": "*", "title": "Pattern", @@ -859,6 +1273,13 @@ "default": "version", "title": "Unit", "type": "string" + }, + "tags": { + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" } }, "required": [ @@ -883,10 +1304,17 @@ "type": "string" }, "unit": { - "default": "bool", + "default": "boolean", "title": "Unit", "type": "string" }, + "tags": { + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" + }, "min_version": { "title": "Min Version", "type": "string" @@ -924,6 +1352,33 @@ }, "title": "Group Tags", "type": "array" + }, + "pretty": { + "default": false, + "title": "Pretty", + "type": "boolean" + }, + "columns": { + "anyOf": [ + { + "items": { + "enum": [ + "name", + "description", + "value", + "unit", + "diagnostics" + ], + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Columns" } }, "required": [ @@ -983,6 +1438,50 @@ ], "additionalProperties": false }, + { + "type": "object", + "properties": { + "type": { + "const": "markdown" + }, + "include_indirect": { + "default": false, + "title": "Include Indirect", + "type": "boolean" + }, + "pretty": { + "default": false, + "title": "Pretty", + "type": "boolean" + }, + "columns": { + "anyOf": [ + { + "items": { + "enum": [ + "name", + "description", + "value", + "unit", + "diagnostics" + ], + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Columns" + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, { "type": "object", "properties": { diff --git a/checkup.yaml b/checkup.yaml index 50499d6..273da70 100644 --- a/checkup.yaml +++ b/checkup.yaml @@ -4,19 +4,23 @@ tags: project: checkup providers: -- name: git + - name: git metrics: -- type: git_days_since_last_update -- type: git_tracked_file_count - pattern: src/checkup/* -- type: git_tracked_file_count - name: checkup_yaml_exists - pattern: checkup.yaml -- type: git_tracked_file_count - name: pre_commit_config_exists - pattern: .pre-commit-config.yaml -- type: python_version -- type: python_version_check - min_version: '3.13' - max_version: '3.15' + - type: git_days_since_last_update + - type: git_tracked_file_count + name: checkup_yaml_exists + description: Checkup yaml exists + pattern: checkup.yaml + - type: git_tracked_file_count + name: pre_commit_config_exists + description: Pre-commit config exists + pattern: .pre-commit-config.yaml + - type: python_version + - type: python_version_check + min_version: "3.13" + max_version: "3.15" + +materializer: + type: console + pretty: true diff --git a/plugins/checkup-conveyor/pyproject.toml b/plugins/checkup-conveyor/pyproject.toml index 755a665..91aa427 100644 --- a/plugins/checkup-conveyor/pyproject.toml +++ b/plugins/checkup-conveyor/pyproject.toml @@ -4,10 +4,7 @@ version = "0.4.0" description = "Conveyor metrics for checkup" readme = "README.md" requires-python = ">=3.12" -dependencies = [ - "checkup", - "requests" -] +dependencies = ["checkup", "requests"] [project.urls] Homepage = "https://github.com/datamindedbe/checkup" @@ -22,7 +19,7 @@ conveyor = "checkup_conveyor:ConveyorProvider" [project.entry-points."checkup.metrics"] conveyor_last_deployment_time = "checkup_conveyor.conveyor_metric:ConveyorLastDeploymentTime" -conveyor_is_dirty_deployment = "checkup_conveyor.conveyor_metric:ConveyorIsDirtyDeployment" +conveyor_clean_deployment = "checkup_conveyor.conveyor_metric:ConveyorCleanDeployment" conveyor_last_run_status = "checkup_conveyor.conveyor_metric:ConveyorLastRunStatus" [build-system] diff --git a/plugins/checkup-conveyor/src/checkup_conveyor/conveyor_metric.py b/plugins/checkup-conveyor/src/checkup_conveyor/conveyor_metric.py index 724a83f..2b5fa1e 100644 --- a/plugins/checkup-conveyor/src/checkup_conveyor/conveyor_metric.py +++ b/plugins/checkup-conveyor/src/checkup_conveyor/conveyor_metric.py @@ -5,6 +5,7 @@ from checkup import Context from checkup.measurement import Measurement, Measurements +from checkup.metric import Unit from checkup_conveyor import ConveyorMetric logger = logging.getLogger(__name__) @@ -33,10 +34,12 @@ def calculate(self, context: Context, measurements: Measurements) -> Measurement ) -class ConveyorIsDirtyDeployment(ConveyorMetric): - name: ClassVar[str] = "Conveyor Is Dirty Deployment" - description: ClassVar[str] = "True if the last deployment was dirty" - unit: ClassVar[str] = "boolean" +class ConveyorCleanDeployment(ConveyorMetric): + name: ClassVar[str] = "Conveyor Clean Deployment" + description: ClassVar[str] = ( + "Whether the last deployment was built from a clean git tree" + ) + unit: ClassVar[str] = Unit.BOOLEAN def calculate(self, context: Context, measurements: Measurements) -> Measurement: proj_id = self.get_conveyor_project_id(context) @@ -54,7 +57,7 @@ def calculate(self, context: Context, measurements: Measurements) -> Measurement diagnostic = ( "Commit changes to git, and deploy the project again." if is_dirty else "" ) - return self.measure(value=is_dirty, diagnostic=diagnostic) + return self.measure(value=not is_dirty, diagnostic=diagnostic) class ConveyorLastRunStatus(ConveyorMetric): diff --git a/plugins/checkup-conveyor/test/test_conveyor_integration.py b/plugins/checkup-conveyor/test/test_conveyor_integration.py index 95aa2df..503c4fb 100644 --- a/plugins/checkup-conveyor/test/test_conveyor_integration.py +++ b/plugins/checkup-conveyor/test/test_conveyor_integration.py @@ -3,7 +3,7 @@ import pytest from checkup_conveyor import ConveyorProvider from checkup_conveyor.conveyor_metric import ( - ConveyorIsDirtyDeployment, + ConveyorCleanDeployment, ConveyorLastDeploymentTime, ConveyorLastRunStatus, ) @@ -25,7 +25,7 @@ def test_conveyor_integration(): .with_metrics( [ ConveyorLastDeploymentTime, - ConveyorIsDirtyDeployment, + ConveyorCleanDeployment, ConveyorLastRunStatus, ] ) diff --git a/plugins/checkup-cruft/src/checkup_cruft/metrics.py b/plugins/checkup-cruft/src/checkup_cruft/metrics.py index 37183e5..79cfaa9 100644 --- a/plugins/checkup-cruft/src/checkup_cruft/metrics.py +++ b/plugins/checkup-cruft/src/checkup_cruft/metrics.py @@ -1,7 +1,7 @@ from datetime import UTC, datetime from checkup.measurement import Measurement, Measurements -from checkup.metric import Metric +from checkup.metric import Metric, Unit from checkup.provider import Provider from checkup.types import Context from checkup_cruft.provider import CruftProvider @@ -27,11 +27,11 @@ class CruftLinkedMetric(CruftMetric): name: str = "cruft_linked" description: str = "Whether a .cruft.json template link is present" - unit: str = "boolean" + unit: str = Unit.BOOLEAN def calculate(self, context: Context, measurements: Measurements) -> Measurement: cruft = self.get_context(context) - return self.measure(value=1 if cruft.get("present") else 0) + return self.measure(value=bool(cruft.get("present"))) class CruftDaysSinceUpdateMetric(CruftMetric): @@ -98,14 +98,14 @@ class CruftUpToDateMetric(CruftMetric): name: str = "cruft_up_to_date" description: str = "Whether the project matches the latest template commit" - unit: str = "boolean" + unit: str = Unit.BOOLEAN def calculate(self, context: Context, measurements: Measurements) -> Measurement: cruft = self.get_context(context) behind = cruft.get("commits_behind") if behind is None: return self.measure(value=None, diagnostic="Template not fetched") - return self.measure(value=1 if behind == 0 else 0) + return self.measure(value=behind == 0) class CruftDaysBehindTemplateMetric(CruftMetric): diff --git a/plugins/checkup-cruft/tests/test_cruft_metrics.py b/plugins/checkup-cruft/tests/test_cruft_metrics.py index 3efb7dc..728a655 100644 --- a/plugins/checkup-cruft/tests/test_cruft_metrics.py +++ b/plugins/checkup-cruft/tests/test_cruft_metrics.py @@ -28,12 +28,12 @@ def _measure(repo: Path, metric, *, fetch_template: bool = False): def test_linked_true(make_product, template_repo): repo = make_product(template_repo, "deadbeef") - assert _measure(repo, CruftLinkedMetric()).value == 1 + assert _measure(repo, CruftLinkedMetric()).value is True def test_linked_false(make_product): repo = make_product() - assert _measure(repo, CruftLinkedMetric()).value == 0 + assert _measure(repo, CruftLinkedMetric()).value is False def test_days_since_update_fresh(make_product, template_repo): @@ -96,12 +96,12 @@ def test_up_to_date_when_pinned_to_head(make_product, template_repo): check=True, ).stdout.strip() repo = make_product(template_repo, head) - assert _measure(repo, CruftUpToDateMetric(), fetch_template=True).value == 1 + assert _measure(repo, CruftUpToDateMetric(), fetch_template=True).value is True def test_up_to_date_false_when_behind(make_product, template_repo): repo = make_product(template_repo, _first_commit(template_repo)) - assert _measure(repo, CruftUpToDateMetric(), fetch_template=True).value == 0 + assert _measure(repo, CruftUpToDateMetric(), fetch_template=True).value is False def test_days_behind_template(make_product, template_repo): diff --git a/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/supported_version.py b/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/supported_version.py index 6499d07..1f34857 100644 --- a/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/supported_version.py +++ b/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/supported_version.py @@ -1,7 +1,7 @@ import logging from checkup.measurement import Measurement, Measurements -from checkup.metric import Metric +from checkup.metric import Metric, Unit from checkup.types import Context from checkup_dbt.metrics.base import DbtMetric from checkup_dbt.metrics.quality.version import DbtVersionMetric @@ -16,7 +16,7 @@ class DbtSupportedVersionMetric(DbtMetric): name: str = "dbt_supported_version" description: str = "Whether dbt version meets minimum requirement" - unit: str = "boolean" + unit: str = Unit.BOOLEAN min_version: str @@ -34,12 +34,11 @@ def calculate(self, context: Context, measurements: Measurements) -> Measurement supported = major_version == min_major and minor_version >= min_minor - value = 1 if supported else 0 diagnostic = "" if not supported: diagnostic = ( f"dbt version {version} does not meet minimum requirement of {self.min_version}. " f"Please upgrade dbt to version {self.min_version} or later." ) - logger.info(f"dbt version {version} supported: {bool(value)}") - return self.measure(value=value, diagnostic=diagnostic) + logger.info(f"dbt version {version} supported: {supported}") + return self.measure(value=supported, diagnostic=diagnostic) diff --git a/plugins/checkup-dbt/src/checkup_dbt/metrics/test/column_test_coverage.py b/plugins/checkup-dbt/src/checkup_dbt/metrics/test/column_test_coverage.py index 2f1bd25..096ab30 100644 --- a/plugins/checkup-dbt/src/checkup_dbt/metrics/test/column_test_coverage.py +++ b/plugins/checkup-dbt/src/checkup_dbt/metrics/test/column_test_coverage.py @@ -1,7 +1,7 @@ import logging from checkup.measurement import Measurement, Measurements -from checkup.metric import Metric +from checkup.metric import Metric, Unit from checkup.types import Context from checkup_dbt.metrics.base import DbtMetric from checkup_dbt.metrics.core.columns import DbtColumnsMetric @@ -20,7 +20,7 @@ class DbtColumnTestCoverageMetric(DbtMetric): name: str = "dbt_column_test_coverage" description: str = "Percentage of columns with at least one test" - unit: str = "percent" + unit: str = Unit.PERCENT @classmethod def depends_on(cls) -> list[type[Metric]]: diff --git a/plugins/checkup-dbt/tests/test_quality_metrics.py b/plugins/checkup-dbt/tests/test_quality_metrics.py index 00ca578..f4d98e5 100644 --- a/plugins/checkup-dbt/tests/test_quality_metrics.py +++ b/plugins/checkup-dbt/tests/test_quality_metrics.py @@ -70,7 +70,7 @@ def test_supported_version_metric(sample_manifest_path: Path): m for m in result.measurements if m.metric.name == "dbt_supported_version" ) assert measurement.metric.unit == "boolean" - assert measurement.value == 1 + assert measurement.value is True def test_supported_version_metric_requires_min_version(): diff --git a/plugins/checkup-python/src/checkup_python/metrics/version_check.py b/plugins/checkup-python/src/checkup_python/metrics/version_check.py index 23f80c5..5c0efd3 100644 --- a/plugins/checkup-python/src/checkup_python/metrics/version_check.py +++ b/plugins/checkup-python/src/checkup_python/metrics/version_check.py @@ -1,5 +1,5 @@ from checkup.measurement import Measurement, Measurements -from checkup.metric import Metric +from checkup.metric import Metric, Unit from checkup.types import Context from checkup_python.metrics.utils import parse_semantic_version from checkup_python.metrics.version import PythonVersionMetric @@ -13,7 +13,7 @@ class PythonVersionCheckMetric(Metric): name: str = "python_version_check" description: str = "The Python version adheres to a minimum and maximum boundary" - unit: str = "bool" + unit: str = Unit.BOOLEAN min_version: str max_version: str diff --git a/src/checkup/__init__.py b/src/checkup/__init__.py index 1d7b465..93646f2 100644 --- a/src/checkup/__init__.py +++ b/src/checkup/__init__.py @@ -15,7 +15,7 @@ SQLAlchemyMaterializer, ) from checkup.measurement import Measurement, Measurements -from checkup.metric import ExecutorType, Metric +from checkup.metric import ExecutorType, Metric, Unit from checkup.provider import Provider from checkup.providers.tags import TagProvider from checkup.types import Context @@ -25,30 +25,26 @@ Measurement.model_rebuild() __all__ = [ - # Core "CheckHub", "MeasurementResult", "Metric", "Measurement", "Measurements", "ExecutorType", + "Unit", "Provider", "TagProvider", "Context", - # Executors "ProviderExecutor", "MetricCalculator", - # Materializers "Materializer", "ConsoleMaterializer", "CSVMaterializer", "HTMLMaterializer", "SQLAlchemyMaterializer", - # Exceptions "ProviderError", "MetricPicklingError", "DuplicateMetricNameError", - # Utilities "suppress_subprocess_output", ] diff --git a/src/checkup/cli/executor.py b/src/checkup/cli/executor.py index ba2c644..341cb85 100644 --- a/src/checkup/cli/executor.py +++ b/src/checkup/cli/executor.py @@ -7,6 +7,7 @@ from rich.console import Console +from checkup.cli.utils import parse_cli_item from checkup.configuration import CheckupConfig from checkup.hub import CheckHub from checkup.materializers import ConsoleMaterializer @@ -170,8 +171,7 @@ def _resolve_materializer( """ if override: - mat_type = override - mat_config = {} + mat_type, mat_config = parse_cli_item(override) elif config.materializer: mat_type = config.materializer.type mat_config = config.materializer.config diff --git a/src/checkup/materializers/__init__.py b/src/checkup/materializers/__init__.py index 4408cb9..6b8b51a 100644 --- a/src/checkup/materializers/__init__.py +++ b/src/checkup/materializers/__init__.py @@ -1,15 +1,15 @@ """Materializers for outputting measurements.""" -from checkup.materializers.base import ( - Materializer, - group_measurements_by_tags, - group_measurements_hierarchical, -) +from checkup.materializers.base import Materializer from checkup.materializers.console import ConsoleMaterializer from checkup.materializers.csv_file import CSVMaterializer from checkup.materializers.database import SQLAlchemyMaterializer from checkup.materializers.html_report import HTMLMaterializer from checkup.materializers.markdown import MarkdownMaterializer +from checkup.materializers.utils import ( + group_measurements_by_tags, + group_measurements_hierarchical, +) __all__ = [ "ConsoleMaterializer", diff --git a/src/checkup/materializers/base.py b/src/checkup/materializers/base.py index 406db94..f34760c 100644 --- a/src/checkup/materializers/base.py +++ b/src/checkup/materializers/base.py @@ -1,72 +1,12 @@ -"""Base materializer class and measurement grouping helpers.""" +"""Base materializer class.""" from abc import ABC, abstractmethod -from collections import defaultdict from pydantic import BaseModel from checkup.measurement import Measurement -def group_measurements_by_tags( - measurements: list[Measurement], - tag1: str, - tag2: str, - default_value: str = "Unknown", -) -> dict[tuple[str, str], list[Measurement]]: - """Group measurements by two tag values. - - Args: - measurements: List of measurements to group - tag1: First tag name for grouping - tag2: Second tag name for grouping - default_value: Value to use when tag is missing - - Returns: - Dict mapping (tag1_value, tag2_value) tuples to measurement lists - """ - groups: dict[tuple[str, str], list[Measurement]] = {} - for measurement in measurements: - tag1_value = measurement.tags.get(tag1, default_value) - tag2_value = measurement.tags.get(tag2, default_value) - key = (tag1_value, tag2_value) - - if key not in groups: - groups[key] = [] - groups[key].append(measurement) - - return groups - - -def group_measurements_hierarchical( - measurements: list[Measurement], - tag1: str, - tag2: str, - default_value: str = "Ungrouped", -) -> dict[str, dict[str, list[Measurement]]]: - """Group measurements hierarchically by two tag values. - - Args: - measurements: List of measurements to group - tag1: First tag name for top-level grouping - tag2: Second tag name for nested grouping - default_value: Value to use when tag is missing - - Returns: - Nested dict: {tag1_value: {tag2_value: [measurements]}} - """ - grouped: dict[str, dict[str, list[Measurement]]] = defaultdict( - lambda: defaultdict(list) - ) - - for measurement in measurements: - group1_value = measurement.tags.get(tag1, default_value) - group2_value = measurement.tags.get(tag2, default_value) - grouped[group1_value][group2_value].append(measurement) - - return dict(grouped) - - class Materializer(ABC, BaseModel): """Base class for measurement materializers. diff --git a/src/checkup/materializers/console.py b/src/checkup/materializers/console.py index 91e0502..5fe48d8 100644 --- a/src/checkup/materializers/console.py +++ b/src/checkup/materializers/console.py @@ -4,7 +4,22 @@ from rich.table import Table from checkup.materializers.base import Materializer +from checkup.materializers.utils import ( + TableColumn, + effective_columns, + render_cell, +) from checkup.measurement import Measurement +from checkup.metric import Unit + +# Rich column settings. +COLUMN_SETTINGS: dict[TableColumn, dict] = { + "name": {"header": "Name", "style": "cyan", "no_wrap": True}, + "description": {"header": "Description", "style": "dim"}, + "value": {"header": "Value", "justify": "right", "style": "green"}, + "unit": {"header": "Unit", "style": "yellow"}, + "diagnostics": {"header": "Diagnostics", "style": "red"}, +} class ConsoleMaterializer(Materializer): @@ -17,9 +32,13 @@ class ConsoleMaterializer(Materializer): Args: group_tags: List of tag names to group by. If empty, no grouping. include_indirect: If True, include indirect measurements. + pretty: If True, format values for presentation, well-know ``Metric.Unit`` values are formatted accordingly. + columns: Columns to display, in order. Defaults to all columns. """ group_tags: list[str] = [] + pretty: bool = False + columns: list[TableColumn] | None = None def materialize( self, measurements: list[Measurement], direct_metric_names: set[str] @@ -55,25 +74,42 @@ def _print_table( Print a single table of measurements. """ - table = Table(title=title) + columns = effective_columns(self.columns, self.pretty) - table.add_column("Name", style="cyan", no_wrap=True) - table.add_column("Description", style="dim") - table.add_column("Value", justify="right", style="green") - table.add_column("Unit", style="yellow") - table.add_column("Diagnostics", style="red") + table = Table(title=title) + for column in columns: + settings = COLUMN_SETTINGS[column] + # Without a name column the description is the row identifier, so don't dim it. + if column == "description" and "name" not in columns: + settings = {**settings, "style": None} + table.add_column(**settings) for measurement in measurements: table.add_row( - measurement.metric.name, - measurement.metric.description, - str(measurement.value) if measurement.value is not None else "", - measurement.metric.unit, - measurement.diagnostic, + *(self._cell(measurement, column, columns) for column in columns) ) console.print(table) + def _cell( + self, + measurement: Measurement, + column: TableColumn, + columns: tuple[TableColumn, ...], + ) -> str: + text = render_cell(measurement, column, columns, self.pretty) + + # Pretty booleans are colored. + if ( + self.pretty + and column == "value" + and measurement.value is not None + and measurement.metric.unit == Unit.BOOLEAN + ): + color = "green" if measurement.value else "red" + return f"[{color}]{text}[/{color}]" + return text + def _group_by_tags( self, measurements: list[Measurement], diff --git a/src/checkup/materializers/html_report.py b/src/checkup/materializers/html_report.py index 0b42352..359a27a 100644 --- a/src/checkup/materializers/html_report.py +++ b/src/checkup/materializers/html_report.py @@ -4,7 +4,8 @@ from jinja2 import Environment, FileSystemLoader -from checkup.materializers.base import Materializer, group_measurements_hierarchical +from checkup.materializers.base import Materializer +from checkup.materializers.utils import group_measurements_hierarchical from checkup.measurement import Measurement diff --git a/src/checkup/materializers/markdown.py b/src/checkup/materializers/markdown.py index e47c16a..fda9486 100644 --- a/src/checkup/materializers/markdown.py +++ b/src/checkup/materializers/markdown.py @@ -1,17 +1,43 @@ """Markdown materializer for Markdown table output.""" from checkup.materializers.base import Materializer +from checkup.materializers.utils import ( + TableColumn, + effective_columns, + render_cell, +) from checkup.measurement import Measurement -COLUMNS = ("Name", "Description", "Value", "Unit", "Diagnostics") -ALIGNMENTS = ("---", "---", "---:", "---", "---") # Right-align the Value column. +HEADERS: dict[TableColumn, str] = { + "name": "Name", + "description": "Description", + "value": "Value", + "unit": "Unit", + "diagnostics": "Diagnostics", +} + +ALIGNMENTS: dict[TableColumn, str] = { + "name": "---", + "description": "---", + "value": "---:", # Right-align + "unit": "---", + "diagnostics": "---", +} class MarkdownMaterializer(Materializer): """ Output measurements as a GitHub-flavoured Markdown table. + + Args: + include_indirect: If True, include indirect measurements. + pretty: If True, format values for presentation, well-know ``Metric.Unit`` values are formatted accordingly. + columns: Columns to display, in order. Defaults to all columns. """ + pretty: bool = False + columns: list[TableColumn] | None = None + def materialize( self, measurements: list[Measurement], direct_metric_names: set[str] ) -> None: @@ -20,17 +46,18 @@ def materialize( """ filtered = self._filter_measurements(measurements, direct_metric_names) + columns = effective_columns(self.columns, self.pretty) - rows = [self._row(COLUMNS), self._row(ALIGNMENTS)] + rows = [ + self._row(tuple(HEADERS[column] for column in columns)), + self._row(tuple(ALIGNMENTS[column] for column in columns)), + ] for measurement in filtered: rows.append( self._row( - ( - measurement.metric.name, - measurement.metric.description, - str(measurement.value) if measurement.value is not None else "", - measurement.metric.unit, - measurement.diagnostic, + tuple( + render_cell(measurement, column, columns, self.pretty) + for column in columns ) ) ) diff --git a/src/checkup/materializers/utils.py b/src/checkup/materializers/utils.py new file mode 100644 index 0000000..be600ab --- /dev/null +++ b/src/checkup/materializers/utils.py @@ -0,0 +1,137 @@ +from collections import defaultdict +from collections.abc import Sequence +from typing import Any, Literal + +from checkup.measurement import Measurement +from checkup.metric import Unit + +TableColumn = Literal["name", "description", "value", "unit", "diagnostics"] + +ALL_COLUMNS: tuple[TableColumn, ...] = ( + "name", + "description", + "value", + "unit", + "diagnostics", +) + + +def effective_columns( + columns: Sequence[TableColumn] | None, pretty: bool +) -> tuple[TableColumn, ...]: + """ + Resolve the columns to display. + """ + + if columns is not None: + return tuple(columns) + if pretty: + # Pretty output folds the unit into the value. + return tuple(column for column in ALL_COLUMNS if column != "unit") + return ALL_COLUMNS + + +def format_value_pretty(value: Any, unit: str) -> str: + """ + Format a measurement value for presentation. + """ + + if value is None: + return "" + if unit == Unit.BOOLEAN: + return "✓" if value else "✗" + if isinstance(value, int | float) and not isinstance(value, bool): + number = f"{value:g}" if isinstance(value, float) else str(value) + if unit == Unit.PERCENT: + return f"{number}%" + return f"{number} {unit}".strip() if unit else number + # We assume that for non-numeric values, appending the unit is not desirable. + return str(value) + + +def render_cell( + measurement: Measurement, + column: TableColumn, + columns: Sequence[TableColumn], + pretty: bool, +) -> str: + """ + Render one table cell as plain text. + """ + + metric = measurement.metric + match column: + case "name": + return metric.name + case "description": + # Without a name column, an empty description leaves the row unidentifiable. + if "name" not in columns and not metric.description: + return metric.name + return metric.description + case "value": + if pretty: + return format_value_pretty(measurement.value, metric.unit) + return str(measurement.value) if measurement.value is not None else "" + case "unit": + return metric.unit + case "diagnostics": + return measurement.diagnostic + + +def group_measurements_by_tags( + measurements: list[Measurement], + tag1: str, + tag2: str, + default_value: str = "Unknown", +) -> dict[tuple[str, str], list[Measurement]]: + """Group measurements by two tag values. + + Args: + measurements: List of measurements to group + tag1: First tag name for grouping + tag2: Second tag name for grouping + default_value: Value to use when tag is missing + + Returns: + Dict mapping (tag1_value, tag2_value) tuples to measurement lists + """ + groups: dict[tuple[str, str], list[Measurement]] = {} + for measurement in measurements: + tag1_value = measurement.tags.get(tag1, default_value) + tag2_value = measurement.tags.get(tag2, default_value) + key = (tag1_value, tag2_value) + + if key not in groups: + groups[key] = [] + groups[key].append(measurement) + + return groups + + +def group_measurements_hierarchical( + measurements: list[Measurement], + tag1: str, + tag2: str, + default_value: str = "Ungrouped", +) -> dict[str, dict[str, list[Measurement]]]: + """Group measurements hierarchically by two tag values. + + Args: + measurements: List of measurements to group + tag1: First tag name for top-level grouping + tag2: Second tag name for nested grouping + default_value: Value to use when tag is missing + + Returns: + Nested dict: {tag1_value: {tag2_value: [measurements]}} + """ + grouped: dict[str, dict[str, list[Measurement]]] = defaultdict( + lambda: defaultdict(list) + ) + + for measurement in measurements: + group1_value = measurement.tags.get(tag1, default_value) + group2_value = measurement.tags.get(tag2, default_value) + grouped[group1_value][group2_value].append(measurement) + + return dict(grouped) diff --git a/src/checkup/metric.py b/src/checkup/metric.py index 41f0cba..5ed1ff2 100644 --- a/src/checkup/metric.py +++ b/src/checkup/metric.py @@ -3,7 +3,7 @@ from __future__ import annotations from abc import ABC, abstractmethod -from enum import Enum +from enum import Enum, StrEnum from typing import TYPE_CHECKING, Any from pydantic import BaseModel, Field @@ -30,6 +30,17 @@ class ExecutorType(Enum): ASYNCIO = "asyncio" +class Unit(StrEnum): + """ + Well-known units. + """ + + # Boolean metrics should be phrased positively (True = desirable). + BOOLEAN = "boolean" + # Values are on the 0-100 scale. + PERCENT = "percent" + + class Metric(ABC, BaseModel): """ Base class for all metrics. diff --git a/tests/fixtures.py b/tests/fixtures.py deleted file mode 100644 index ceebe1f..0000000 --- a/tests/fixtures.py +++ /dev/null @@ -1,441 +0,0 @@ -from typing import Any, ClassVar - -from checkup.measurement import Measurement, Measurements -from checkup.metric import Metric -from checkup.provider import Provider -from checkup.types import Context - - -class DummyMetric(Metric): - """Simple test metric with no dependencies.""" - - name: str = "dummy" - description: str = "Test metric" - unit: str = "count" - - expected_value: int = 42 - - def calculate(self, context: Context, measurements: Measurements) -> Measurement: - """Set value to expected_value.""" - return self.measure( - value=self.expected_value, - diagnostic=f"Dummy metric calculated with expected_value={self.expected_value}", - ) - - -class DependentDummyMetric(Metric): - """Test metric that depends on DummyMetric.""" - - name: str = "dependent_dummy" - description: str = "Depends on DummyMetric" - unit: str = "count" - - @classmethod - def depends_on(cls) -> list[type[Metric]]: - """Depends on DummyMetric.""" - return [DummyMetric] - - def calculate(self, context: Context, measurements: Measurements) -> Measurement: - """Double the DummyMetric value.""" - base_value = measurements.get(DummyMetric).value - value = base_value * 2 - return self.measure( - value=value, - diagnostic=f"Doubled DummyMetric value from {base_value} to {value}", - ) - - -class Level2Metric(Metric): - """Test metric at depth 2 in dependency chain.""" - - name: str = "level2" - description: str = "Depth 2 metric" - unit: str = "count" - - @classmethod - def depends_on(cls) -> list[type[Metric]]: - return [DependentDummyMetric] - - def calculate(self, context: Context, measurements: Measurements) -> Measurement: - value = measurements.get(DependentDummyMetric).value + 10 - return self.measure( - value=value, diagnostic=f"Added 10 to DependentDummyMetric value: {value}" - ) - - -class Level3Metric(Metric): - """Test metric at depth 3 in dependency chain.""" - - name: str = "level3" - description: str = "Depth 3 metric" - unit: str = "count" - - @classmethod - def depends_on(cls) -> list[type[Metric]]: - return [Level2Metric] - - def calculate(self, context: Context, measurements: Measurements) -> Measurement: - level2_value = measurements.get(Level2Metric).value - value = level2_value**2 - return self.measure( - value=value, - diagnostic=f"Squared Level2Metric value: {level2_value}^2 = {value}", - ) - - -class CyclicMetricA(Metric): - """Test metric that creates a cycle with CyclicMetricB.""" - - name: str = "cyclic_a" - description: str = "Cyclic test metric A" - unit: str = "count" - - @classmethod - def depends_on(cls) -> list[type[Metric]]: - return [CyclicMetricB] - - def calculate(self, context: Context, measurements: Measurements) -> Measurement: - return self.measure(value=1, diagnostic="CyclicMetricA calculated") - - -class CyclicMetricB(Metric): - """Test metric that creates a cycle with CyclicMetricA.""" - - name: str = "cyclic_b" - description: str = "Cyclic test metric B" - unit: str = "count" - - @classmethod - def depends_on(cls) -> list[type[Metric]]: - return [CyclicMetricA] - - def calculate(self, context: Context, measurements: Measurements) -> Measurement: - return self.measure(value=1, diagnostic="CyclicMetricB calculated") - - -class RootA(Metric): - """Root metric A - no dependencies.""" - - name: str = "root_a" - description: str = "Root A metric" - unit: str = "count" - base_value: int = 10 - - def calculate(self, context: Context, measurements: Measurements) -> Measurement: - return self.measure( - value=self.base_value, - diagnostic=f"RootA calculated with base_value={self.base_value}", - ) - - -class RootB(Metric): - """Root metric B - no dependencies.""" - - name: str = "root_b" - description: str = "Root B metric" - unit: str = "count" - base_value: int = 20 - - def calculate(self, context: Context, measurements: Measurements) -> Measurement: - return self.measure( - value=self.base_value, - diagnostic=f"RootB calculated with base_value={self.base_value}", - ) - - -class RootC(Metric): - """Root metric C - no dependencies (independent subgraph).""" - - name: str = "root_c" - description: str = "Root C metric" - unit: str = "count" - base_value: int = 100 - - def calculate(self, context: Context, measurements: Measurements) -> Measurement: - return self.measure( - value=self.base_value, - diagnostic=f"RootC calculated with base_value={self.base_value}", - ) - - -class SharedAB(Metric): - """Metric with shared ancestors - depends on both RootA and RootB.""" - - name: str = "shared_ab" - description: str = "Shared AB metric" - unit: str = "count" - - @classmethod - def depends_on(cls) -> list[type[Metric]]: - return [RootA, RootB] - - def calculate(self, context: Context, measurements: Measurements) -> Measurement: - root_a_val = measurements.get(RootA).value - root_b_val = measurements.get(RootB).value - value = root_a_val + root_b_val - return self.measure( - value=value, - diagnostic=f"Sum of RootA ({root_a_val}) and RootB ({root_b_val}) = {value}", - ) - - -class BranchB(Metric): - """Branch from RootB only.""" - - name: str = "branch_b" - description: str = "Branch B metric" - unit: str = "count" - - @classmethod - def depends_on(cls) -> list[type[Metric]]: - return [RootB] - - def calculate(self, context: Context, measurements: Measurements) -> Measurement: - root_b_val = measurements.get(RootB).value - value = root_b_val * 3 - return self.measure( - value=value, diagnostic=f"Tripled RootB value: {root_b_val} * 3 = {value}" - ) - - -class LeafC(Metric): - """Leaf in independent subgraph - depends on RootC.""" - - name: str = "leaf_c" - description: str = "Leaf C metric" - unit: str = "count" - - @classmethod - def depends_on(cls) -> list[type[Metric]]: - return [RootC] - - def calculate(self, context: Context, measurements: Measurements) -> Measurement: - root_c_val = measurements.get(RootC).value - value = root_c_val**2 - return self.measure( - value=value, diagnostic=f"Squared RootC value: {root_c_val}^2 = {value}" - ) - - -class MidShared(Metric): - """Middle layer - depends on SharedAB.""" - - name: str = "mid_shared" - description: str = "Mid shared metric" - unit: str = "count" - - @classmethod - def depends_on(cls) -> list[type[Metric]]: - return [SharedAB] - - def calculate(self, context: Context, measurements: Measurements) -> Measurement: - shared_ab_val = measurements.get(SharedAB).value - value = shared_ab_val + 5 - return self.measure( - value=value, - diagnostic=f"Added 5 to SharedAB value: {shared_ab_val} + 5 = {value}", - ) - - -class MidBranch(Metric): - """Middle layer - depends on BranchB.""" - - name: str = "mid_branch" - description: str = "Mid branch metric" - unit: str = "count" - - @classmethod - def depends_on(cls) -> list[type[Metric]]: - return [BranchB] - - def calculate(self, context: Context, measurements: Measurements) -> Measurement: - branch_b_val = measurements.get(BranchB).value - value = branch_b_val * 2 - return self.measure( - value=value, - diagnostic=f"Doubled BranchB value: {branch_b_val} * 2 = {value}", - ) - - -class LeafAB(Metric): - """Leaf with diamond pattern - depends on both MidShared and MidBranch.""" - - name: str = "leaf_ab" - description: str = "Leaf AB metric (diamond convergence)" - unit: str = "count" - - @classmethod - def depends_on(cls) -> list[type[Metric]]: - return [MidShared, MidBranch] - - def calculate(self, context: Context, measurements: Measurements) -> Measurement: - mid_shared_val = measurements.get(MidShared).value - mid_branch_val = measurements.get(MidBranch).value - value = mid_shared_val * mid_branch_val - return self.measure( - value=value, - diagnostic=f"Product of MidShared ({mid_shared_val}) and MidBranch ({mid_branch_val}) = {value}", - ) - - -class DummyProvider(Provider): - """Test provider that adds dummy data to context.""" - - name: ClassVar[str] = "dummy" - - def __init__(self, data: int = 100): - self.data = data - - def provide(self) -> dict[str, Any]: - return {"data": self.data} - - -class ProviderDummyMetric(Metric): - """Test metric that uses a provider.""" - - name: str = "provider_dummy" - description: str = "Uses dummy provider" - unit: str = "count" - - @classmethod - def providers(cls) -> list[type[Provider]]: - return [DummyProvider] - - def calculate(self, context: Context, measurements: Measurements) -> Measurement: - value = context[DummyProvider.name]["data"] - return self.measure( - value=value, diagnostic=f"Retrieved dummy_data from context: {value}" - ) - - -class FailingMetric(Metric): - """Test metric that fails based on context.""" - - name: str = "failing" - description: str = "Fails when should_fail is True" - unit: str = "count" - - def calculate(self, context: Context, measurements: Measurements) -> Measurement: - if context.get("should_fail"): - raise ValueError("Intentional failure") - return self.measure(value=1, diagnostic="Metric calculated successfully") - - -class IntegrationProvider(Provider): - """Provider that adds base_value to context.""" - - name: ClassVar[str] = "integration" - - def __init__(self, base_value: int = 25): - self.base_value = base_value - - def provide(self) -> dict[str, Any]: - return {"base_value": self.base_value} - - -class IntegrationBaseMetric(Metric): - """Base metric for integration tests.""" - - name: str = "base_metric" - description: str = "Base test metric" - unit: str = "units" - threshold: int = 100 - - @classmethod - def providers(cls) -> list[type[Provider]]: - return [IntegrationProvider] - - def calculate(self, context: Context, measurements: Measurements) -> Measurement: - value = context[IntegrationProvider.name]["base_value"] - return self.measure( - value=value, - diagnostic=f"Retrieved base_value from integration provider: {value}", - ) - - -class IntegrationDerivedMetric(Metric): - """Derived metric for integration tests.""" - - name: str = "derived_metric" - description: str = "Derived test metric" - unit: str = "units" - multiplier: int = 2 - - @classmethod - def depends_on(cls): - return [IntegrationBaseMetric] - - def calculate(self, context: Context, measurements: Measurements) -> Measurement: - base_val = measurements.get(IntegrationBaseMetric).value - value = base_val * self.multiplier - return self.measure( - value=value, - diagnostic=f"Multiplied base metric value: {base_val} * {self.multiplier} = {value}", - ) - - -class PathLengthProvider(Provider): - """Provider that calculates path length.""" - - name: ClassVar[str] = "path_length" - - def __init__(self, path: str = "/unknown"): - self.path = path - - def provide(self) -> dict[str, Any]: - return {"length": len(self.path)} - - -class PathMetric(Metric): - """Metric that uses path length from context.""" - - name: str = "path_metric" - description: str = "Calculates based on path" - unit: str = "count" - multiplier: int = 1 - - @classmethod - def providers(cls) -> list[type[Provider]]: - return [PathLengthProvider] - - def calculate(self, context: Context, measurements: Measurements) -> Measurement: - path_len = context[PathLengthProvider.name]["length"] - value = path_len * self.multiplier - return self.measure( - value=value, - diagnostic=f"Path length {path_len} * multiplier {self.multiplier} = {value}", - ) - - -class OtherDummyMetric(Metric): - """Another test metric with a different name.""" - - name: str = "other_metric" - description: str = "Other test metric" - unit: str = "count" - - expected_value: int = 100 - - def calculate(self, context: Context, measurements: Measurements) -> Measurement: - """Set value to expected_value.""" - return self.measure( - value=self.expected_value, - diagnostic=f"Other metric calculated with expected_value={self.expected_value}", - ) - - -class IndirectDummyMetric(Metric): - """Test metric for testing indirect metric filtering.""" - - name: str = "indirect" - description: str = "Indirect test metric" - unit: str = "count" - - expected_value: int = 100 - - def calculate(self, context: Context, measurements: Measurements) -> Measurement: - """Set value to expected_value.""" - return self.measure( - value=self.expected_value, - diagnostic=f"Indirect metric calculated with expected_value={self.expected_value}", - ) diff --git a/tests/fixtures/__init__.py b/tests/fixtures/__init__.py new file mode 100644 index 0000000..9e7a182 --- /dev/null +++ b/tests/fixtures/__init__.py @@ -0,0 +1,61 @@ +from fixtures.capture import capture_output +from fixtures.graph import ( + BranchB, + LeafAB, + LeafC, + MidBranch, + MidShared, + RootA, + RootB, + RootC, + SharedAB, +) +from fixtures.metrics import ( + CyclicMetricA, + CyclicMetricB, + DependentDummyMetric, + DummyMetric, + FailingMetric, + IndirectDummyMetric, + Level2Metric, + Level3Metric, + OtherDummyMetric, +) +from fixtures.providers import ( + DummyProvider, + IntegrationBaseMetric, + IntegrationDerivedMetric, + IntegrationProvider, + PathLengthProvider, + PathMetric, + ProviderDummyMetric, +) + +__all__ = [ + "BranchB", + "CyclicMetricA", + "CyclicMetricB", + "DependentDummyMetric", + "DummyMetric", + "DummyProvider", + "FailingMetric", + "IndirectDummyMetric", + "IntegrationBaseMetric", + "IntegrationDerivedMetric", + "IntegrationProvider", + "LeafAB", + "LeafC", + "Level2Metric", + "Level3Metric", + "MidBranch", + "MidShared", + "OtherDummyMetric", + "PathLengthProvider", + "PathMetric", + "ProviderDummyMetric", + "RootA", + "RootB", + "RootC", + "SharedAB", + "capture_output", +] diff --git a/tests/fixtures/capture.py b/tests/fixtures/capture.py new file mode 100644 index 0000000..cac61af --- /dev/null +++ b/tests/fixtures/capture.py @@ -0,0 +1,18 @@ +import sys +from io import StringIO + +from checkup.materializers import Materializer + + +def capture_output(materializer: Materializer, measurements, direct) -> str: + """ + Capture the stdout a materializer produces, restoring stdout afterwards. + """ + + captured_output = StringIO() + sys.stdout = captured_output + try: + materializer.materialize(measurements, direct) + finally: + sys.stdout = sys.__stdout__ + return captured_output.getvalue() diff --git a/tests/fixtures/graph.py b/tests/fixtures/graph.py new file mode 100644 index 0000000..441f1cd --- /dev/null +++ b/tests/fixtures/graph.py @@ -0,0 +1,168 @@ +from checkup.measurement import Measurement, Measurements +from checkup.metric import Metric +from checkup.types import Context + + +class RootA(Metric): + """Root metric A - no dependencies.""" + + name: str = "root_a" + description: str = "Root A metric" + unit: str = "count" + base_value: int = 10 + + def calculate(self, context: Context, measurements: Measurements) -> Measurement: + return self.measure( + value=self.base_value, + diagnostic=f"RootA calculated with base_value={self.base_value}", + ) + + +class RootB(Metric): + """Root metric B - no dependencies.""" + + name: str = "root_b" + description: str = "Root B metric" + unit: str = "count" + base_value: int = 20 + + def calculate(self, context: Context, measurements: Measurements) -> Measurement: + return self.measure( + value=self.base_value, + diagnostic=f"RootB calculated with base_value={self.base_value}", + ) + + +class RootC(Metric): + """Root metric C - no dependencies (independent subgraph).""" + + name: str = "root_c" + description: str = "Root C metric" + unit: str = "count" + base_value: int = 100 + + def calculate(self, context: Context, measurements: Measurements) -> Measurement: + return self.measure( + value=self.base_value, + diagnostic=f"RootC calculated with base_value={self.base_value}", + ) + + +class SharedAB(Metric): + """Metric with shared ancestors - depends on both RootA and RootB.""" + + name: str = "shared_ab" + description: str = "Shared AB metric" + unit: str = "count" + + @classmethod + def depends_on(cls) -> list[type[Metric]]: + return [RootA, RootB] + + def calculate(self, context: Context, measurements: Measurements) -> Measurement: + root_a_val = measurements.get(RootA).value + root_b_val = measurements.get(RootB).value + value = root_a_val + root_b_val + return self.measure( + value=value, + diagnostic=f"Sum of RootA ({root_a_val}) and RootB ({root_b_val}) = {value}", + ) + + +class BranchB(Metric): + """Branch from RootB only.""" + + name: str = "branch_b" + description: str = "Branch B metric" + unit: str = "count" + + @classmethod + def depends_on(cls) -> list[type[Metric]]: + return [RootB] + + def calculate(self, context: Context, measurements: Measurements) -> Measurement: + root_b_val = measurements.get(RootB).value + value = root_b_val * 3 + return self.measure( + value=value, diagnostic=f"Tripled RootB value: {root_b_val} * 3 = {value}" + ) + + +class LeafC(Metric): + """Leaf in independent subgraph - depends on RootC.""" + + name: str = "leaf_c" + description: str = "Leaf C metric" + unit: str = "count" + + @classmethod + def depends_on(cls) -> list[type[Metric]]: + return [RootC] + + def calculate(self, context: Context, measurements: Measurements) -> Measurement: + root_c_val = measurements.get(RootC).value + value = root_c_val**2 + return self.measure( + value=value, diagnostic=f"Squared RootC value: {root_c_val}^2 = {value}" + ) + + +class MidShared(Metric): + """Middle layer - depends on SharedAB.""" + + name: str = "mid_shared" + description: str = "Mid shared metric" + unit: str = "count" + + @classmethod + def depends_on(cls) -> list[type[Metric]]: + return [SharedAB] + + def calculate(self, context: Context, measurements: Measurements) -> Measurement: + shared_ab_val = measurements.get(SharedAB).value + value = shared_ab_val + 5 + return self.measure( + value=value, + diagnostic=f"Added 5 to SharedAB value: {shared_ab_val} + 5 = {value}", + ) + + +class MidBranch(Metric): + """Middle layer - depends on BranchB.""" + + name: str = "mid_branch" + description: str = "Mid branch metric" + unit: str = "count" + + @classmethod + def depends_on(cls) -> list[type[Metric]]: + return [BranchB] + + def calculate(self, context: Context, measurements: Measurements) -> Measurement: + branch_b_val = measurements.get(BranchB).value + value = branch_b_val * 2 + return self.measure( + value=value, + diagnostic=f"Doubled BranchB value: {branch_b_val} * 2 = {value}", + ) + + +class LeafAB(Metric): + """Leaf with diamond pattern - depends on both MidShared and MidBranch.""" + + name: str = "leaf_ab" + description: str = "Leaf AB metric (diamond convergence)" + unit: str = "count" + + @classmethod + def depends_on(cls) -> list[type[Metric]]: + return [MidShared, MidBranch] + + def calculate(self, context: Context, measurements: Measurements) -> Measurement: + mid_shared_val = measurements.get(MidShared).value + mid_branch_val = measurements.get(MidBranch).value + value = mid_shared_val * mid_branch_val + return self.measure( + value=value, + diagnostic=f"Product of MidShared ({mid_shared_val}) and MidBranch ({mid_branch_val}) = {value}", + ) diff --git a/tests/fixtures/metrics.py b/tests/fixtures/metrics.py new file mode 100644 index 0000000..5ffdde3 --- /dev/null +++ b/tests/fixtures/metrics.py @@ -0,0 +1,157 @@ +from checkup.measurement import Measurement, Measurements +from checkup.metric import Metric +from checkup.types import Context + + +class DummyMetric(Metric): + """Simple test metric with no dependencies.""" + + name: str = "dummy" + description: str = "Test metric" + unit: str = "count" + + expected_value: int = 42 + + def calculate(self, context: Context, measurements: Measurements) -> Measurement: + """Set value to expected_value.""" + return self.measure( + value=self.expected_value, + diagnostic=f"Dummy metric calculated with expected_value={self.expected_value}", + ) + + +class OtherDummyMetric(Metric): + """Another test metric with a different name.""" + + name: str = "other_metric" + description: str = "Other test metric" + unit: str = "count" + + expected_value: int = 100 + + def calculate(self, context: Context, measurements: Measurements) -> Measurement: + """Set value to expected_value.""" + return self.measure( + value=self.expected_value, + diagnostic=f"Other metric calculated with expected_value={self.expected_value}", + ) + + +class IndirectDummyMetric(Metric): + """Test metric for testing indirect metric filtering.""" + + name: str = "indirect" + description: str = "Indirect test metric" + unit: str = "count" + + expected_value: int = 100 + + def calculate(self, context: Context, measurements: Measurements) -> Measurement: + """Set value to expected_value.""" + return self.measure( + value=self.expected_value, + diagnostic=f"Indirect metric calculated with expected_value={self.expected_value}", + ) + + +class DependentDummyMetric(Metric): + """Test metric that depends on DummyMetric.""" + + name: str = "dependent_dummy" + description: str = "Depends on DummyMetric" + unit: str = "count" + + @classmethod + def depends_on(cls) -> list[type[Metric]]: + """Depends on DummyMetric.""" + return [DummyMetric] + + def calculate(self, context: Context, measurements: Measurements) -> Measurement: + """Double the DummyMetric value.""" + base_value = measurements.get(DummyMetric).value + value = base_value * 2 + return self.measure( + value=value, + diagnostic=f"Doubled DummyMetric value from {base_value} to {value}", + ) + + +class Level2Metric(Metric): + """Test metric at depth 2 in dependency chain.""" + + name: str = "level2" + description: str = "Depth 2 metric" + unit: str = "count" + + @classmethod + def depends_on(cls) -> list[type[Metric]]: + return [DependentDummyMetric] + + def calculate(self, context: Context, measurements: Measurements) -> Measurement: + value = measurements.get(DependentDummyMetric).value + 10 + return self.measure( + value=value, diagnostic=f"Added 10 to DependentDummyMetric value: {value}" + ) + + +class Level3Metric(Metric): + """Test metric at depth 3 in dependency chain.""" + + name: str = "level3" + description: str = "Depth 3 metric" + unit: str = "count" + + @classmethod + def depends_on(cls) -> list[type[Metric]]: + return [Level2Metric] + + def calculate(self, context: Context, measurements: Measurements) -> Measurement: + level2_value = measurements.get(Level2Metric).value + value = level2_value**2 + return self.measure( + value=value, + diagnostic=f"Squared Level2Metric value: {level2_value}^2 = {value}", + ) + + +class CyclicMetricA(Metric): + """Test metric that creates a cycle with CyclicMetricB.""" + + name: str = "cyclic_a" + description: str = "Cyclic test metric A" + unit: str = "count" + + @classmethod + def depends_on(cls) -> list[type[Metric]]: + return [CyclicMetricB] + + def calculate(self, context: Context, measurements: Measurements) -> Measurement: + return self.measure(value=1, diagnostic="CyclicMetricA calculated") + + +class CyclicMetricB(Metric): + """Test metric that creates a cycle with CyclicMetricA.""" + + name: str = "cyclic_b" + description: str = "Cyclic test metric B" + unit: str = "count" + + @classmethod + def depends_on(cls) -> list[type[Metric]]: + return [CyclicMetricA] + + def calculate(self, context: Context, measurements: Measurements) -> Measurement: + return self.measure(value=1, diagnostic="CyclicMetricB calculated") + + +class FailingMetric(Metric): + """Test metric that fails based on context.""" + + name: str = "failing" + description: str = "Fails when should_fail is True" + unit: str = "count" + + def calculate(self, context: Context, measurements: Measurements) -> Measurement: + if context.get("should_fail"): + raise ValueError("Intentional failure") + return self.measure(value=1, diagnostic="Metric calculated successfully") diff --git a/tests/fixtures/providers.py b/tests/fixtures/providers.py new file mode 100644 index 0000000..2abecf9 --- /dev/null +++ b/tests/fixtures/providers.py @@ -0,0 +1,122 @@ +from typing import Any, ClassVar + +from checkup.measurement import Measurement, Measurements +from checkup.metric import Metric +from checkup.provider import Provider +from checkup.types import Context + + +class DummyProvider(Provider): + """Test provider that adds dummy data to context.""" + + name: ClassVar[str] = "dummy" + + def __init__(self, data: int = 100): + self.data = data + + def provide(self) -> dict[str, Any]: + return {"data": self.data} + + +class ProviderDummyMetric(Metric): + """Test metric that uses a provider.""" + + name: str = "provider_dummy" + description: str = "Uses dummy provider" + unit: str = "count" + + @classmethod + def providers(cls) -> list[type[Provider]]: + return [DummyProvider] + + def calculate(self, context: Context, measurements: Measurements) -> Measurement: + value = context[DummyProvider.name]["data"] + return self.measure( + value=value, diagnostic=f"Retrieved dummy_data from context: {value}" + ) + + +class IntegrationProvider(Provider): + """Provider that adds base_value to context.""" + + name: ClassVar[str] = "integration" + + def __init__(self, base_value: int = 25): + self.base_value = base_value + + def provide(self) -> dict[str, Any]: + return {"base_value": self.base_value} + + +class IntegrationBaseMetric(Metric): + """Base metric for integration tests.""" + + name: str = "base_metric" + description: str = "Base test metric" + unit: str = "units" + threshold: int = 100 + + @classmethod + def providers(cls) -> list[type[Provider]]: + return [IntegrationProvider] + + def calculate(self, context: Context, measurements: Measurements) -> Measurement: + value = context[IntegrationProvider.name]["base_value"] + return self.measure( + value=value, + diagnostic=f"Retrieved base_value from integration provider: {value}", + ) + + +class IntegrationDerivedMetric(Metric): + """Derived metric for integration tests.""" + + name: str = "derived_metric" + description: str = "Derived test metric" + unit: str = "units" + multiplier: int = 2 + + @classmethod + def depends_on(cls): + return [IntegrationBaseMetric] + + def calculate(self, context: Context, measurements: Measurements) -> Measurement: + base_val = measurements.get(IntegrationBaseMetric).value + value = base_val * self.multiplier + return self.measure( + value=value, + diagnostic=f"Multiplied base metric value: {base_val} * {self.multiplier} = {value}", + ) + + +class PathLengthProvider(Provider): + """Provider that calculates path length.""" + + name: ClassVar[str] = "path_length" + + def __init__(self, path: str = "/unknown"): + self.path = path + + def provide(self) -> dict[str, Any]: + return {"length": len(self.path)} + + +class PathMetric(Metric): + """Metric that uses path length from context.""" + + name: str = "path_metric" + description: str = "Calculates based on path" + unit: str = "count" + multiplier: int = 1 + + @classmethod + def providers(cls) -> list[type[Provider]]: + return [PathLengthProvider] + + def calculate(self, context: Context, measurements: Measurements) -> Measurement: + path_len = context[PathLengthProvider.name]["length"] + value = path_len * self.multiplier + return self.measure( + value=value, + diagnostic=f"Path length {path_len} * multiplier {self.multiplier} = {value}", + ) diff --git a/tests/test_cli_configuration.py b/tests/test_cli_configuration.py index 83d29cf..4d55bae 100644 --- a/tests/test_cli_configuration.py +++ b/tests/test_cli_configuration.py @@ -321,3 +321,17 @@ def test_malformed_pair_is_skipped(self, caplog): assert name == "dbt" assert config == {"project_dir": "./dbt", "other": "value"} assert "malformed" in caplog.text + + +class TestResolveMaterializerOverride: + def test_override_with_config_pairs(self): + from checkup.cli.executor import _resolve_materializer + from checkup.configuration import CheckupConfig + from checkup.registry import get_registry + + materializer = _resolve_materializer( + CheckupConfig.empty(), get_registry(), "console:pretty=true" + ) + + assert type(materializer).__name__ == "ConsoleMaterializer" + assert materializer.pretty is True diff --git a/tests/test_materializer_base.py b/tests/test_materializer_base.py new file mode 100644 index 0000000..e88ab4e --- /dev/null +++ b/tests/test_materializer_base.py @@ -0,0 +1,56 @@ +import pytest +from fixtures import DummyMetric, IndirectDummyMetric, capture_output + +from checkup.materializers import ConsoleMaterializer, Materializer + + +def test_materializer_is_abstract(): + """Test that Materializer cannot be instantiated.""" + with pytest.raises(TypeError): + Materializer() + + +def test_materializer_filters_indirect_by_default(): + """Test that materializers filter out indirect metrics by default.""" + direct_metric = DummyMetric(expected_value=42) + direct_measurement = direct_metric.measure(value=42) + + indirect_metric = IndirectDummyMetric(expected_value=100) + indirect_measurement = indirect_metric.measure(value=100) + + # Only "dummy" is direct, "indirect" is not + output = capture_output( + ConsoleMaterializer(group_tags=["domain", "project"]), + [direct_measurement, indirect_measurement], + {"dummy"}, + ) + + assert "dummy" in output # Direct metric included + assert "indirect" not in output # Indirect metric filtered out + + +def test_materializer_includes_indirect_when_configured(): + """Test that materializers can include indirect metrics.""" + direct_metric = DummyMetric(expected_value=42) + direct_measurement = direct_metric.measure(value=42) + + indirect_metric = IndirectDummyMetric(expected_value=100) + indirect_measurement = indirect_metric.measure(value=100) + + output = capture_output( + ConsoleMaterializer(include_indirect=True, group_tags=["domain", "project"]), + [direct_measurement, indirect_measurement], + {"dummy"}, + ) + + assert "dummy" in output # Direct metric included + assert "indirect" in output # Indirect metric also included + + +def test_materializer_rejects_unknown_column(): + """ + Column names are validated against the known table columns. + """ + + with pytest.raises(ValueError): + ConsoleMaterializer(columns=["nope"]) diff --git a/tests/test_materializer_console.py b/tests/test_materializer_console.py new file mode 100644 index 0000000..d816d24 --- /dev/null +++ b/tests/test_materializer_console.py @@ -0,0 +1,80 @@ +from fixtures import DummyMetric, capture_output + +from checkup.materializers import ConsoleMaterializer + + +def test_console_materializer(): + """Test console output materializer with two-level grouping.""" + metric = DummyMetric(expected_value=42) + measurement = metric.measure(value=42) + + output = capture_output( + ConsoleMaterializer(group_tags=["domain", "project"]), [measurement], {"dummy"} + ) + + assert "dummy" in output + assert "42" in output + + +def test_console_materializer_no_grouping(): + """Test console materializer without grouping.""" + metric = DummyMetric(expected_value=42) + measurement = metric.measure(value=42) + + output = capture_output(ConsoleMaterializer(), [measurement], {"dummy"}) + + assert "dummy" in output + assert "42" in output + + +def test_console_materializer_single_grouping(): + """Test console materializer with single-level grouping.""" + metric = DummyMetric(expected_value=42) + measurement = metric.measure(value=42, tags={"domain": "Analytics"}) + + output = capture_output( + ConsoleMaterializer(group_tags=["domain"]), [measurement], {"dummy"} + ) + + assert "dummy" in output + assert "42" in output + assert "domain: Analytics" in output + + +def test_console_materializer_three_level_grouping(): + """Test console materializer with three-level grouping.""" + metric = DummyMetric(expected_value=42) + measurement = metric.measure( + value=42, tags={"domain": "Analytics", "project": "Core", "env": "prod"} + ) + + output = capture_output( + ConsoleMaterializer(group_tags=["domain", "project", "env"]), + [measurement], + {"dummy"}, + ) + + assert "dummy" in output + assert "42" in output + assert "domain: Analytics" in output + assert "project: Core" in output + assert "env: prod" in output + + +def test_console_pretty_boolean_glyphs(): + """ + Console pretty mode renders booleans as check or cross glyphs. + """ + + measurements = [ + DummyMetric(name="linked", unit="boolean").measure(value=True), + DummyMetric(name="clean", unit="boolean").measure(value=False), + ] + + output = capture_output( + ConsoleMaterializer(pretty=True), measurements, {"linked", "clean"} + ) + + assert "✓" in output + assert "✗" in output + assert "boolean" not in output diff --git a/tests/test_materializer_csv.py b/tests/test_materializer_csv.py new file mode 100644 index 0000000..eaf95a8 --- /dev/null +++ b/tests/test_materializer_csv.py @@ -0,0 +1,90 @@ +from fixtures import DummyMetric, IndirectDummyMetric, OtherDummyMetric + +from checkup.materializers import CSVMaterializer + + +def test_csv_materializer(tmp_path): + """Test CSV file materializer.""" + metric = DummyMetric(expected_value=42) + measurement = metric.measure(value=42) + + output_file = tmp_path / "metrics.csv" + materializer = CSVMaterializer(output_path=output_file) + materializer.materialize([measurement], {"dummy"}) + + # Read and verify CSV content + content = output_file.read_text() + lines = content.strip().split("\n") + + # Check header + assert lines[0] == "name,value,unit,diagnostic,description" + + # Check data row + assert "dummy" in lines[1] + assert "42" in lines[1] + assert "count" in lines[1] + + +def test_csv_materializer_multiple_metrics(tmp_path): + """Test CSV materializer with multiple metrics.""" + metric1 = DummyMetric(expected_value=42) + measurement1 = metric1.measure(value=42) + + metric2 = OtherDummyMetric(expected_value=100) + measurement2 = metric2.measure(value=100) + + output_file = tmp_path / "metrics.csv" + materializer = CSVMaterializer(output_path=output_file) + materializer.materialize([measurement1, measurement2], {"dummy", "other_metric"}) + + content = output_file.read_text() + lines = content.strip().split("\n") + + assert len(lines) == 3 # Header + 2 data rows + assert "dummy" in lines[1] + assert "other_metric" in lines[2] + + +def test_csv_materializer_filters_indirect(tmp_path): + """Test CSV materializer filtering of indirect metrics.""" + direct_metric = DummyMetric(expected_value=42) + direct_measurement = direct_metric.measure(value=42) + + indirect_metric = IndirectDummyMetric(expected_value=100) + indirect_measurement = indirect_metric.measure(value=100) + + output_file = tmp_path / "metrics.csv" + + # Default: filter indirect + materializer = CSVMaterializer(output_path=output_file) + # Only "dummy" is direct + materializer.materialize([direct_measurement, indirect_measurement], {"dummy"}) + + content = output_file.read_text() + lines = content.strip().split("\n") + + assert len(lines) == 2 # Header + 1 direct metric + assert "dummy" in lines[1] + assert "indirect" not in content + + +def test_csv_materializer_includes_indirect(tmp_path): + """Test CSV materializer including indirect metrics.""" + direct_metric = DummyMetric(expected_value=42) + direct_measurement = direct_metric.measure(value=42) + + indirect_metric = IndirectDummyMetric(expected_value=100) + indirect_measurement = indirect_metric.measure(value=100) + + output_file = tmp_path / "metrics.csv" + + # With include_indirect=True + materializer = CSVMaterializer(output_path=output_file, include_indirect=True) + materializer.materialize([direct_measurement, indirect_measurement], {"dummy"}) + + content = output_file.read_text() + lines = content.strip().split("\n") + + assert len(lines) == 3 # Header + 2 metrics + assert "dummy" in content + assert "indirect" in content diff --git a/tests/test_materializer_html.py b/tests/test_materializer_html.py new file mode 100644 index 0000000..a41509a --- /dev/null +++ b/tests/test_materializer_html.py @@ -0,0 +1,309 @@ +from fixtures import DummyMetric, IndirectDummyMetric, OtherDummyMetric + +from checkup.materializers import HTMLMaterializer + + +def test_html_materializer(tmp_path): + """Test HTML materializer with hierarchical grouping.""" + # Create measurements with tags + metric1 = DummyMetric(expected_value=42) + measurement1 = metric1.measure( + value=42, tags={"domain": "Analytics", "project": "Project A"} + ) + + metric2 = DummyMetric(expected_value=100) + measurement2 = metric2.measure( + value=100, tags={"domain": "Analytics", "project": "Project B"} + ) + + metric3 = DummyMetric(expected_value=75) + measurement3 = metric3.measure( + value=75, tags={"domain": "Engineering", "project": "Project C"} + ) + + output_file = tmp_path / "metrics.html" + materializer = HTMLMaterializer( + output_path=output_file, group_tag_1="domain", group_tag_2="project" + ) + materializer.materialize([measurement1, measurement2, measurement3], {"dummy"}) + + # Verify file was created + assert output_file.exists() + + # Read and verify HTML content + content = output_file.read_text() + + # Check HTML structure + assert "" in content + assert "" in content + assert "Metrics Report" in content + + # Check Bootstrap is included + assert "bootstrap" in content + + # Check for group names + assert "Analytics" in content + assert "Engineering" in content + assert "Project A" in content + assert "Project B" in content + assert "Project C" in content + + # Check for accordion components + assert "accordion" in content + assert "accordion-button" in content + assert "accordion-collapse" in content + + # Check metric data is present + assert "dummy" in content + assert "42" in content + assert "100" in content + assert "75" in content + + # Check for table structure + assert "" in content + assert "Value" in content + + +def test_html_materializer_with_diagnostics(tmp_path): + """Test HTML materializer with diagnostic coloring.""" + # Create measurements with different diagnostics + metric1 = DummyMetric(expected_value=42) + measurement1 = metric1.measure( + value=42, + diagnostic="✅ All good", + tags={"domain": "Test", "project": "TestProject"}, + ) + + metric2 = DummyMetric(expected_value=100) + measurement2 = metric2.measure( + value=100, + diagnostic="⚠ Warning: something to check", + tags={"domain": "Test", "project": "TestProject"}, + ) + + output_file = tmp_path / "metrics.html" + materializer = HTMLMaterializer( + output_path=output_file, group_tag_1="domain", group_tag_2="project" + ) + materializer.materialize([measurement1, measurement2], {"dummy"}) + + content = output_file.read_text() + + # Check for diagnostic color classes + assert "table-success" in content + assert "table-warning" in content + + # Check diagnostic text is present + assert "All good" in content + assert "Warning" in content + + +def test_html_materializer_ungrouped_metrics(tmp_path): + """Test HTML materializer with measurements missing tags.""" + # Create measurement without tags + metric = DummyMetric(expected_value=42) + measurement = metric.measure(value=42, tags={}) + + output_file = tmp_path / "metrics.html" + materializer = HTMLMaterializer( + output_path=output_file, group_tag_1="domain", group_tag_2="project" + ) + materializer.materialize([measurement], {"dummy"}) + + content = output_file.read_text() + + # Should default to "Ungrouped" + assert "Ungrouped" in content + assert "dummy" in content + + +def test_html_materializer_filters_indirect(tmp_path): + """Test HTML materializer filtering of indirect metrics.""" + direct_metric = DummyMetric(expected_value=42) + direct_measurement = direct_metric.measure( + value=42, tags={"domain": "Test", "project": "TestProject"} + ) + + indirect_metric = IndirectDummyMetric(expected_value=100) + indirect_measurement = indirect_metric.measure( + value=100, tags={"domain": "Test", "project": "TestProject"} + ) + + output_file = tmp_path / "metrics.html" + + # Default: filter indirect + materializer = HTMLMaterializer( + output_path=output_file, group_tag_1="domain", group_tag_2="project" + ) + materializer.materialize([direct_measurement, indirect_measurement], {"dummy"}) + + content = output_file.read_text() + + # Only direct metric should be present + assert "dummy" in content + assert "indirect" not in content + + +def test_html_materializer_escape_html(tmp_path): + """Test that HTML special characters are escaped.""" + metric = DummyMetric(expected_value=42) + measurement = metric.measure( + value="", + diagnostic="Test & verify ", + tags={"domain": "Test & Dev", "project": "Project "}, + ) + + output_file = tmp_path / "metrics.html" + materializer = HTMLMaterializer( + output_path=output_file, group_tag_1="domain", group_tag_2="project" + ) + materializer.materialize([measurement], {"dummy"}) + + content = output_file.read_text() + + # Check that special characters are escaped + assert "<script>" in content + assert "&" in content + assert "", - diagnostic="Test & verify ", - tags={"domain": "Test & Dev", "project": "Project "}, - ) - - output_file = tmp_path / "metrics.html" - materializer = HTMLMaterializer( - output_path=output_file, group_tag_1="domain", group_tag_2="project" - ) - materializer.materialize([measurement], {"dummy"}) - - content = output_file.read_text() - - # Check that special characters are escaped - assert "<script>" in content - assert "&" in content - assert "