diff --git a/tests/benchmarks/conftest.py b/tests/benchmarks/conftest.py index 646f1b6..932623e 100644 --- a/tests/benchmarks/conftest.py +++ b/tests/benchmarks/conftest.py @@ -1,4 +1,10 @@ -"""Shared fixtures and constants for benchmarks.""" +"""Shared fixtures and constants for benchmarks. + +Benchmark tests require the ``benchmark`` fixture provided by the +``pytest-codspeed`` plugin. When the plugin is absent (e.g. downstream +packaging like conda-forge running ``pytest tests/``), the tests must be +skipped instead of failing on a missing fixture or an unknown mark. +""" import numpy as np import pytest @@ -11,6 +17,8 @@ CONTINUOUS_METRICS = ["lc", "rc", "gv"] +_BENCHMARK_FIXTURES = {"benchmark", "codspeed_benchmark"} + def gen_continuous(n_nodes: int, rng: int = 19425) -> np.ndarray: _, _, ts = gen_delayed_causal_network( @@ -24,14 +32,38 @@ def continuous_data(request) -> np.ndarray: return gen_continuous(request.param) +def pytest_configure(config): + """Register the benchmark markers so they are not "unknown" without pytest-codspeed. + + Without this, pytest raises a ``PytestUnknownMarkWarning`` for + ``@pytest.mark.benchmark`` which is escalated to an error by the + ``filterwarnings = ["error"]`` setting in ``pyproject.toml``. + """ + existing = set() + for entry in config.getini("markers"): + name = entry.split(":", 1)[0].strip() if isinstance(entry, str) else entry[0] + existing.add(name) + for marker in sorted(_BENCHMARK_FIXTURES): + if marker not in existing: + config.addinivalue_line( + "markers", + f"{marker}: micro-benchmark; run with --codspeed", + ) + + +def _is_benchmark_item(item: pytest.Item) -> bool: + return bool(_BENCHMARK_FIXTURES.intersection(getattr(item, "fixturenames", []))) + + def pytest_collection_modifyitems(config, items): + """Skip benchmark tests unless explicitly run with ``--codspeed``. + + Works with or without the ``pytest-codspeed`` plugin installed: benchmark + tests are identified by the ``benchmark`` fixture they request, so they are + skipped even when the plugin (and thus the fixture) is unavailable. + """ if not config.getoption("codspeed", False): - try: - from pytest_codspeed.plugin import has_benchmark_fixture - - msg = pytest.mark.skip(reason="use --codspeed to run benchmarks") - for item in items: - if has_benchmark_fixture(item): - item.add_marker(msg) - except ImportError: - pass + msg = pytest.mark.skip(reason="use --codspeed to run benchmarks") + for item in items: + if _is_benchmark_item(item): + item.add_marker(msg) diff --git a/tests/benchmarks/test_conftest_skip.py b/tests/benchmarks/test_conftest_skip.py new file mode 100644 index 0000000..3e7b1a8 --- /dev/null +++ b/tests/benchmarks/test_conftest_skip.py @@ -0,0 +1,66 @@ +"""Regression tests for the benchmark skip logic in ``tests/benchmarks/conftest.py``. + +Benchmark tests must be skipped when ``--codspeed`` is not passed, and this must +work even when the ``pytest-codspeed`` plugin (which provides the ``benchmark`` +fixture and registers the ``benchmark`` mark) is not installed -- e.g. when a +downstream consumer such as conda-forge runs ``pytest tests/``. +""" + +import pytest + +from .conftest import _is_benchmark_item, pytest_collection_modifyitems + + +class _StubItem: + """Minimal stand-in for a pytest item to exercise the collection hook.""" + + def __init__(self, fixturenames): + self.fixturenames = fixturenames + self.own_markers = [] + + def add_marker(self, marker): + self.own_markers.append(marker) + + +def test_is_benchmark_item_detects_benchmark_fixture(): + """A test requesting the ``benchmark`` fixture is identified as a benchmark.""" + # Arrange + item = _StubItem(["benchmark", "continuous_data"]) + # Act + is_benchmark = _is_benchmark_item(item) + # Assert + assert is_benchmark + + +def test_is_benchmark_item_false_without_benchmark_fixture(): + """A test without a benchmark fixture is not identified as a benchmark.""" + # Arrange + item = _StubItem(["continuous_data"]) + # Act + is_benchmark = _is_benchmark_item(item) + # Assert + assert not is_benchmark + + +def test_benchmark_item_skipped_without_codspeed(pytestconfig, monkeypatch): + """Without ``--codspeed``, benchmark tests are marked to be skipped.""" + # Arrange + monkeypatch.setattr(pytestconfig, "getoption", lambda name, default=None: default) + item = _StubItem(["benchmark", "continuous_data"]) + # Act + pytest_collection_modifyitems(pytestconfig, [item]) + # Assert + assert any(m.mark.name == "skip" for m in item.own_markers) + + +def test_benchmark_item_not_skipped_with_codspeed(pytestconfig, monkeypatch): + """With ``--codspeed``, benchmark tests are not skipped.""" + # Arrange + monkeypatch.setattr( + pytestconfig, "getoption", lambda name, default=None: name == "codspeed" + ) + item = _StubItem(["benchmark", "continuous_data"]) + # Act + pytest_collection_modifyitems(pytestconfig, [item]) + # Assert + assert not any(m.mark.name == "skip" for m in item.own_markers)