From 074c6b19cd9993c97f984c092703ae5eb62f5c6e Mon Sep 17 00:00:00 2001 From: Katarzyna Kaczmarska Date: Mon, 10 Aug 2026 16:44:13 +0200 Subject: [PATCH] [UR][CI] Add unified test result reporting for UR tests Introduce common tooling for collecting and reporting results from UR conformance and adapter-specific test executions. The implementation: - provides a common execution and reporting path for UR test suites, - collects detailed test statuses and execution statistics, - parses LIT output and JUnit XML to obtain test-level information, - generates human-readable console output and GitHub Step Summary, - preserves full execution logs and XML results as CI artifacts, - separates test execution, result parsing, and summary generation to support future consumers of test result data. This is the first implementation step towards a broader test management and test health monitoring system. Database integration, historical analysis, and automated PR triage are outside the scope of this change. --- .ci/requirements.txt | 1 + .github/workflows/ur-build-hw.yml | 67 +++--- devops/actions/run-tests/ur/action.yml | 127 ++++++++++ devops/scripts/ur-test | 8 + devops/scripts/ur_test_tools/__init__.py | 36 +++ devops/scripts/ur_test_tools/cli.py | 149 ++++++++++++ devops/scripts/ur_test_tools/constants.py | 36 +++ .../scripts/ur_test_tools/models/__init__.py | 25 ++ devops/scripts/ur_test_tools/models/config.py | 48 ++++ .../scripts/ur_test_tools/models/test_data.py | 50 ++++ .../scripts/ur_test_tools/outputs/__init__.py | 9 + .../scripts/ur_test_tools/outputs/console.py | 119 ++++++++++ .../ur_test_tools/outputs/github_actions.py | 20 ++ .../scripts/ur_test_tools/parsers/__init__.py | 19 ++ .../ur_test_tools/parsers/log_parser.py | 151 ++++++++++++ .../ur_test_tools/parsers/stats_parser.py | 16 ++ .../ur_test_tools/parsers/xml_parser.py | 90 +++++++ .../ur_test_tools/summary_generator.py | 224 ++++++++++++++++++ devops/scripts/ur_test_tools/test_runner.py | 160 +++++++++++++ .../ur_test_tools/validation/__init__.py | 9 + .../validation/data_validator.py | 36 +++ .../validation/path_validator.py | 96 ++++++++ unified-runtime/test/CMakeLists.txt | 3 +- .../test/conformance/CMakeLists.txt | 6 +- .../third_party/requirements_testing.txt | 3 + 25 files changed, 1476 insertions(+), 32 deletions(-) create mode 100644 devops/actions/run-tests/ur/action.yml create mode 100755 devops/scripts/ur-test create mode 100644 devops/scripts/ur_test_tools/__init__.py create mode 100644 devops/scripts/ur_test_tools/cli.py create mode 100644 devops/scripts/ur_test_tools/constants.py create mode 100644 devops/scripts/ur_test_tools/models/__init__.py create mode 100644 devops/scripts/ur_test_tools/models/config.py create mode 100644 devops/scripts/ur_test_tools/models/test_data.py create mode 100644 devops/scripts/ur_test_tools/outputs/__init__.py create mode 100644 devops/scripts/ur_test_tools/outputs/console.py create mode 100644 devops/scripts/ur_test_tools/outputs/github_actions.py create mode 100644 devops/scripts/ur_test_tools/parsers/__init__.py create mode 100644 devops/scripts/ur_test_tools/parsers/log_parser.py create mode 100644 devops/scripts/ur_test_tools/parsers/stats_parser.py create mode 100644 devops/scripts/ur_test_tools/parsers/xml_parser.py create mode 100644 devops/scripts/ur_test_tools/summary_generator.py create mode 100644 devops/scripts/ur_test_tools/test_runner.py create mode 100644 devops/scripts/ur_test_tools/validation/__init__.py create mode 100644 devops/scripts/ur_test_tools/validation/data_validator.py create mode 100644 devops/scripts/ur_test_tools/validation/path_validator.py diff --git a/.ci/requirements.txt b/.ci/requirements.txt index 45eb253548496..6c6abcada3eaf 100644 --- a/.ci/requirements.txt +++ b/.ci/requirements.txt @@ -1,3 +1,4 @@ junitparser==3.2.0 google-cloud-storage==3.3.0 PyGithub==2.8.1 +defusedxml==0.7.1 diff --git a/.github/workflows/ur-build-hw.yml b/.github/workflows/ur-build-hw.yml index ef5ce3bae5712..192874089e5db 100644 --- a/.github/workflows/ur-build-hw.yml +++ b/.github/workflows/ur-build-hw.yml @@ -72,22 +72,9 @@ jobs: # - switch to Ninja generator in CMake # - downloading DPC++ should be integrated somehow; most likely use nightly release. # - - name: Clean workspace - run: | - find $GITHUB_WORKSPACE -mindepth 1 -delete 2>/dev/null || true - mkdir -p $GITHUB_WORKSPACE - - name: Checkout LLVM uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Verify checkout - run: | - if [ ! -d "unified-runtime" ]; then - echo "ERROR: unified-runtime directory not found after checkout" - ls -la - exit 1 - fi - # for some reason it's required to re-configure python for venv to work properly. - name: Set up Python 3.12 if: ${{ inputs.docker_image == 'ghcr.io/intel/llvm/ubuntu2404_intel_drivers:latest' }} @@ -96,12 +83,13 @@ jobs: python-version: '3.12' - name: Install UR python dependencies in venv + working-directory: ./unified-runtime run: | python3 -m venv .venv . .venv/bin/activate echo "$PATH" >> $GITHUB_PATH - pip install -r unified-runtime/third_party/requirements.txt - pip install -r unified-runtime/third_party/requirements_testing.txt + pip install -r third_party/requirements.txt + pip install -r third_party/requirements_testing.txt - name: Download DPC++ run: | @@ -125,10 +113,6 @@ jobs: run: | sudo -E bash devops/scripts/install_drivers.sh devops/dependencies.json --igfx - - name: Check NVML version compatibility - if: ${{ inputs.adapter_name == 'CUDA' }} - run: python3 devops/scripts/check_nvml_version.py - - name: Configure Unified Runtime project # ">" is used to avoid adding "\" at the end of each line; this command is quite long run: > @@ -157,21 +141,46 @@ jobs: # This is to check that install command does not fail run: cmake --install build - - name: Test adapter specific + - name: Build artifact name suffix + id: artifact_suffix + shell: bash env: - ZE_ENABLE_LOADER_DEBUG_TRACE: 1 - LIT_OPTS: "--timeout 120 -j 50" - # These tests cause timeouts on CI - LIT_FILTER_OUT: "(adapters/level_zero/memcheck.test|adapters/level_zero/v2/deferred_kernel_memcheck.test)" - run: cmake --build build -j $(($(nproc)/3)) -- check-unified-runtime-adapter + RUNNER_NAME: ${{ inputs.runner_name }} + STATIC_LOADER: ${{ matrix.adapter.static_Loader }} + OTHER_NAME: ${{ matrix.adapter.other_name }} + run: | + RUNNER_SUFFIX=$(echo "${RUNNER_NAME}" | sed 's/^UR_//') + SUFFIX="${RUNNER_SUFFIX}" + + if [ "${STATIC_LOADER}" = "ON" ]; then + SUFFIX="${SUFFIX}-static" + fi + + if [ -n "${OTHER_NAME}" ]; then + SUFFIX="${SUFFIX}-${OTHER_NAME}" + fi + + echo "suffix=${SUFFIX}" >> $GITHUB_OUTPUT + + - name: Check NVML version compatibility + if: ${{ inputs.adapter_name == 'CUDA' }} + run: python3 devops/scripts/check_nvml_version.py + + - name: Test adapter specific + uses: ./devops/actions/run-tests/ur + with: + test_type: 'adapter-specific' + build_dir: build + artifact_name: 'ur-${{matrix.adapter.name}}-${{steps.artifact_suffix.outputs.suffix}}-adapter-specific' # Don't run adapter specific tests when building multiple adapters if: ${{ matrix.adapter.other_name == '' }} - name: Test adapters - env: - ZE_ENABLE_LOADER_DEBUG_TRACE: 1 - LIT_OPTS: "--timeout 120 -j 50" - run: cmake --build build -j $(($(nproc)/3)) -- check-unified-runtime-conformance + uses: ./devops/actions/run-tests/ur + with: + test_type: 'conformance' + build_dir: build + artifact_name: 'ur-${{matrix.adapter.name}}-${{steps.artifact_suffix.outputs.suffix}}-conformance' - name: Debug CI platform information if: ${{ always() }} diff --git a/devops/actions/run-tests/ur/action.yml b/devops/actions/run-tests/ur/action.yml new file mode 100644 index 0000000000000..5a03e0d6c6482 --- /dev/null +++ b/devops/actions/run-tests/ur/action.yml @@ -0,0 +1,127 @@ +name: 'Run UR Tests' + +inputs: + test_type: + description: 'Type of tests to run (adapter-specific or conformance)' + required: true + build_dir: + required: false + default: 'build' + artifact_name: + description: 'Name for uploaded test artifacts (logs, XML)' + required: false + default: '' + +runs: + using: "composite" + steps: + - name: Run ${{ inputs.test_type }} tests + id: run_tests + continue-on-error: true + shell: bash + env: + BUILD_DIR: ${{ inputs.build_dir }} + TEST_TYPE: ${{ inputs.test_type }} + run: | + # Run tests via unified CLI (handles validation, config, cmake execution) + # Use temp file instead of command substitution to preserve outputs even on error + OUTPUTS_FILE=$(mktemp) + set +e # Don't exit on error + devops/scripts/ur-test run "$TEST_TYPE" "$BUILD_DIR" "$GITHUB_WORKSPACE" > "$OUTPUTS_FILE" 2>&1 + EXIT_CODE=$? + set -e # Re-enable exit on error + + # Parse Python output and set GitHub Actions outputs (not env for security) + while IFS='=' read -r key value; do + if [ -n "$key" ] && [ -n "$value" ]; then + echo "$key=$value" >> $GITHUB_OUTPUT + echo " Set output: $key=$value" + fi + done < <(grep -E "^(log-file|xml-file|skip-artifacts)=" "$OUTPUTS_FILE") + + rm -f "$OUTPUTS_FILE" + exit $EXIT_CODE + + - name: Filter test log + if: ${{ always() && steps.run_tests.outputs['skip-artifacts'] != '1' }} + shell: bash + env: + LOG_FILE: ${{ steps.run_tests.outputs['log-file'] || 'adapter_tests.log' }} + run: | + if [ ! -f "$LOG_FILE" ]; then + echo "::warning::Test log not found" + exit 0 + fi + + echo "::group::Test Log" + devops/scripts/ur-test filter-log "$LOG_FILE" || { echo "::error::Filter failed"; head -1000 "$LOG_FILE"; } + echo "::endgroup::" + + - name: Report test failures + if: steps.run_tests.outcome != 'success' + shell: bash + env: + LOG_FILE: ${{ steps.run_tests.outputs['log-file'] || 'adapter_tests.log' }} + run: | + devops/scripts/ur-test extract-errors "$LOG_FILE" + exit 1 + + - name: Test summary + if: ${{ always() && steps.run_tests.outputs['skip-artifacts'] != '1' }} + shell: bash + env: + LOG_FILE: ${{ steps.run_tests.outputs['log-file'] || 'adapter_tests.log' }} + XML_FILE: ${{ steps.run_tests.outputs['xml-file'] || 'adapter_tests_results.xml' }} + ARTIFACT_NAME: ${{ inputs.artifact_name }} + WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + if [ ! -f "$LOG_FILE" ]; then + echo "::warning::Test log not found" + exit 0 + fi + + if [ ! -s "$LOG_FILE" ]; then + echo "::warning::Test log is empty" + exit 0 + fi + + devops/scripts/ur-test summary "$LOG_FILE" "${XML_FILE:-}" + + # Add link to artifacts if they will be uploaded + if [ -n "$ARTIFACT_NAME" ]; then + echo "" + echo "---" + echo "" + echo "Test artifacts (full logs, XML) will be available at:" + echo "$WORKFLOW_URL" + fi + + - name: Copy artifacts + if: ${{ always() && inputs.artifact_name != '' && (inputs.test_type != 'adapter-specific' || steps.run_tests.outputs['skip-artifacts'] != '1') }} + shell: bash + env: + LOG_FILE: ${{ steps.run_tests.outputs['log-file'] || 'adapter_tests.log' }} + XML_FILE: ${{ steps.run_tests.outputs['xml-file'] || 'adapter_tests_results.xml' }} + run: | + set -f # Disable glob expansion + + # Clean artifacts directory to avoid mixing files from different test runs + rm -rf test_artifacts + mkdir -p test_artifacts + + if [ -f "$LOG_FILE" ]; then + cp "$LOG_FILE" test_artifacts/ + fi + + if [ -n "$XML_FILE" ] && [ -f "$XML_FILE" ]; then + cp "$XML_FILE" test_artifacts/ + fi + + - name: Upload test artifacts + if: ${{ always() && inputs.artifact_name != '' && (inputs.test_type != 'adapter-specific' || steps.run_tests.outputs['skip-artifacts'] != '1') }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ inputs.artifact_name }} + path: test_artifacts/ + retention-days: 7 + if-no-files-found: warn diff --git a/devops/scripts/ur-test b/devops/scripts/ur-test new file mode 100755 index 0000000000000..fcb4aaf4af43a --- /dev/null +++ b/devops/scripts/ur-test @@ -0,0 +1,8 @@ +#!/usr/bin/env python3 +"""Unified Runtime test management CLI.""" + +import sys +from ur_test_tools.cli import main + +if __name__ == "__main__": + sys.exit(main()) diff --git a/devops/scripts/ur_test_tools/__init__.py b/devops/scripts/ur_test_tools/__init__.py new file mode 100644 index 0000000000000..c8d7cdb6a98ac --- /dev/null +++ b/devops/scripts/ur_test_tools/__init__.py @@ -0,0 +1,36 @@ +"""UR Test Tools - Modular test orchestration and summary generation.""" + +__version__ = "1.0.0" +__author__ = "Unified Runtime Team" + +from .models import ( + TestLists, + TestCounts, + TimingSummary, + TestConfig, + SummaryConfigFromLines, +) +from .test_runner import TestRunner +from .summary_generator import SummaryReporter +from .validation import ( + PathValidator, +) +from .outputs import ( + ConsoleOutput, + GitHubActionsOutput, +) + +__all__ = [ + "__version__", + "__author__", + "TestLists", + "TestCounts", + "TimingSummary", + "TestConfig", + "SummaryConfigFromLines", + "TestRunner", + "SummaryReporter", + "PathValidator", + "ConsoleOutput", + "GitHubActionsOutput", +] diff --git a/devops/scripts/ur_test_tools/cli.py b/devops/scripts/ur_test_tools/cli.py new file mode 100644 index 0000000000000..76db046db4f70 --- /dev/null +++ b/devops/scripts/ur_test_tools/cli.py @@ -0,0 +1,149 @@ +"""CLI entry points for UR test tools.""" + +import sys +import os +from pathlib import Path + +from .models.config import SummaryConfigFromLines, TestConfig, TestExecutionContext +from .validation.path_validator import PathValidator +from .parsers.log_parser import ( + LITLogParser, + read_log_file, +) +from .outputs.console import ConsoleOutput +from .summary_generator import SummaryReporter +from .test_runner import ( + TestRunner, + get_test_config, +) +from .outputs.github_actions import GitHubActionsOutput + + +def main() -> int: + """Unified CLI entry point.""" + if len(sys.argv) < 2: + print("Error: Missing command", file=sys.stderr) + return 1 + + command = sys.argv[1] + + if command == "run": + return main_ci_utils("run-tests") + + elif command in ("summary", "extract-errors", "filter-log"): + internal_cmd = "show-summary" if command == "summary" else command + return main_test_summary(internal_cmd) + + else: + print(f"Error: Unknown command '{command}'", file=sys.stderr) + return 1 + + +def main_test_summary(command: str) -> int: + """Entry point for ur_test_summary CLI.""" + try: + if len(sys.argv) < 3: + print( + f"Error: {sys.argv[0]} [xml_file]", + file=sys.stderr, + ) + return 1 + + log_file = sys.argv[2] + PathValidator.validate_log_path(log_file) + lines = read_log_file(log_file) + parser = LITLogParser(lines) + + if command == "extract-errors": + for line in parser.extract_error_details(): + print(line, end="") + + elif command == "filter-log": + for line in ConsoleOutput.filter_log_for_display(lines): + print(line, end="") + + elif command == "show-summary": + xml_file = PathValidator.validate_optional_path( + sys.argv[3] if len(sys.argv) > 3 else "", "XML", allow_absolute=True + ) + config = SummaryConfigFromLines(log_lines=lines, xml_file=xml_file or None) + SummaryReporter(config).generate() + + else: + print(f"Error: Unknown command '{command}'", file=sys.stderr) + return 1 + + return 0 + + except (OSError, ValueError) as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + + +def main_ci_utils(command: str) -> int: + """Entry point for ur_ci_utils CLI.""" + if command == "run-tests": + return _run_tests_command() + + else: + print(f"Error: Unknown command '{command}'", file=sys.stderr) + return 1 + + +def _run_tests_command() -> int: + """Execute run-tests command.""" + if len(sys.argv) < 5: + print( + f"Error: run-tests ", + file=sys.stderr, + ) + return 1 + + test_type = sys.argv[2] + build_dir = sys.argv[3] + workspace = sys.argv[4] + + # Validate inputs + gha = GitHubActionsOutput() + if not PathValidator.validate_build_dir(build_dir, workspace): + gha.print_error("Invalid build_dir") + return 1 + + try: + config = get_test_config(test_type) + except ValueError as e: + gha.print_error(str(e)) + return 1 + + # Convert to paths and create context + workspace_path = Path(workspace).resolve() + build_dir_path = workspace_path / build_dir + + xml_output_name = f"{test_type.replace('-', '_')}_results.xml" + xml_output_path = (build_dir_path / xml_output_name).absolute() + xml_output_path.parent.mkdir(parents=True, exist_ok=True) + + log_file_path = workspace_path / config.log_file + + env = os.environ.copy() + + context = TestExecutionContext( + test_type=test_type, + build_dir=build_dir_path, + workspace=workspace_path, + xml_output_path=xml_output_path, + log_file_path=log_file_path, + config=config, + env=env, + ) + + # Validate context + try: + context.validate() + except ValueError as e: + gha.print_error(str(e)) + return 1 + + # Run tests + runner = TestRunner(context) + return runner.run() diff --git a/devops/scripts/ur_test_tools/constants.py b/devops/scripts/ur_test_tools/constants.py new file mode 100644 index 0000000000000..11610e6d5e894 --- /dev/null +++ b/devops/scripts/ur_test_tools/constants.py @@ -0,0 +1,36 @@ +"""Constants for UR test tools.""" + +import re + +# File I/O +MAX_LINES_TO_SCAN = 1000 + +SEPARATOR_WIDTH = 70 + +# Job Calculation +MAX_JOBS = 16 + +# LIT Configuration +DEFAULT_LIT_TIMEOUT = 120 +DEFAULT_LIT_JOBS = 50 + +# Test Type Identifiers +TEST_TYPE_ADAPTER_SPECIFIC = "adapter-specific" + +# Constants +TEST_NOT_SELECTED_MSG = "Test not selected" +SLOWEST_TESTS_HEADER = "Slowest Tests:" +TEST_TIMES_HEADERS = ("Tests Times:", "Test Times:") + +# LIT Output Patterns +FAIL_TIMEOUT_PATTERN = re.compile(r"^(FAIL|TIMEOUT):") +TEST_LIST_HEADER_PATTERN = re.compile( + r"^(Passed|Unsupported|Failed|Expectedly Failed|" + r"Timed Out|Unexpectedly Passed|Unresolved) Tests \(" +) +STATS_PATTERN = re.compile( + r"^\s*(Total Discovered|Expected Passes|Expectedly Failed|" + r"Excluded|Unsupported|Skipped|Passed|Passed With Retry|" + r"Failed|Timed Out|Unexpectedly Passed|Unresolved)(\s+Tests)?\s*:" +) +TEST_CATEGORY_PATTERN = re.compile(r"^([A-Za-z]+(?: [A-Za-z]+)*) Tests \((\d+)\):") diff --git a/devops/scripts/ur_test_tools/models/__init__.py b/devops/scripts/ur_test_tools/models/__init__.py new file mode 100644 index 0000000000000..f242b3bc63400 --- /dev/null +++ b/devops/scripts/ur_test_tools/models/__init__.py @@ -0,0 +1,25 @@ +"""Models package - Data structures for UR test tools.""" + +from .test_data import ( + TestLists, + TestCounts, + TimingSummary, + SkippedTestsResult, + ExcludedTestsResult, +) +from .config import ( + TestConfig, + TestExecutionContext, + SummaryConfigFromLines, +) + +__all__ = [ + "TestLists", + "TestCounts", + "TimingSummary", + "SkippedTestsResult", + "ExcludedTestsResult", + "TestConfig", + "TestExecutionContext", + "SummaryConfigFromLines", +] diff --git a/devops/scripts/ur_test_tools/models/config.py b/devops/scripts/ur_test_tools/models/config.py new file mode 100644 index 0000000000000..dbdcfb97df9e4 --- /dev/null +++ b/devops/scripts/ur_test_tools/models/config.py @@ -0,0 +1,48 @@ +"""Configuration dataclasses for UR test tools.""" + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Optional, Dict, List + + +@dataclass +class TestConfig: + """Test execution configuration.""" + + target: str + log_file: str + lit_filter_out: Optional[str] = None + + def __post_init__(self): + """Validate configuration on creation.""" + if not all([self.target, self.log_file]): + raise ValueError("target and log_file are required") + + +@dataclass(frozen=True) +class TestExecutionContext: + """Context for test execution (immutable).""" + + test_type: str + build_dir: Path + workspace: Path + xml_output_path: Path + log_file_path: Path + config: TestConfig + env: Dict[str, str] = field(default_factory=dict) + + def validate(self) -> None: + try: + workspace_resolved = self.workspace.resolve() + for path in [self.log_file_path, self.xml_output_path, self.build_dir]: + path.resolve().relative_to(workspace_resolved) + except ValueError as e: + raise ValueError(f"Path outside workspace: {e}") from e + + +@dataclass +class SummaryConfigFromLines: + """Configuration for summary generation from parsed log lines.""" + + log_lines: List[str] + xml_file: Optional[str] = None diff --git a/devops/scripts/ur_test_tools/models/test_data.py b/devops/scripts/ur_test_tools/models/test_data.py new file mode 100644 index 0000000000000..782c475609dc8 --- /dev/null +++ b/devops/scripts/ur_test_tools/models/test_data.py @@ -0,0 +1,50 @@ +"""Type definitions for test data structures.""" + +from typing import TypedDict, List + + +class TestLists(TypedDict, total=False): + """Type definition for test list dictionary.""" + + Passed: List[str] + Failed: List[str] + Skipped: List[str] + Unsupported: List[str] + Excluded: List[str] + Unresolved: List[str] + + +class TestCounts(TypedDict, total=False): + """Type definition for test count dictionary.""" + + Passed: int + Failed: int + Skipped: int + Unsupported: int + Excluded: int + Unresolved: int + + +class TimingSummary(TypedDict): + """Type definition for test timing summary.""" + + slowest: List[str] + histogram: List[str] + + +class SkippedTestsResult(TypedDict): + """Result of skipped tests analysis.""" + + tests: List[str] + count: int + source: str + note: str + + +class ExcludedTestsResult(TypedDict): + """Result of excluded tests analysis.""" + + tests: List[str] + count: int + source: str + note: str diff --git a/devops/scripts/ur_test_tools/outputs/__init__.py b/devops/scripts/ur_test_tools/outputs/__init__.py new file mode 100644 index 0000000000000..9273918957af5 --- /dev/null +++ b/devops/scripts/ur_test_tools/outputs/__init__.py @@ -0,0 +1,9 @@ +"""Outputs package - Console and GitHub Actions output generation.""" + +from .console import ConsoleOutput +from .github_actions import GitHubActionsOutput + +__all__ = [ + "ConsoleOutput", + "GitHubActionsOutput", +] diff --git a/devops/scripts/ur_test_tools/outputs/console.py b/devops/scripts/ur_test_tools/outputs/console.py new file mode 100644 index 0000000000000..a0b5f98fb2d86 --- /dev/null +++ b/devops/scripts/ur_test_tools/outputs/console.py @@ -0,0 +1,119 @@ +"""Console output formatting for test results.""" + +from typing import List, Optional + +from ..constants import ( + SEPARATOR_WIDTH, + SLOWEST_TESTS_HEADER, + STATS_PATTERN, + TEST_CATEGORY_PATTERN, + TEST_TIMES_HEADERS, +) +from ..models.test_data import TimingSummary +from ..parsers.log_parser import LITLogParser + + +class ConsoleOutput: + """Format test results for console output.""" + + @staticmethod + def print_test_group( + title: str, tests: List[str], note: str = "", count: Optional[int] = None + ) -> None: + """Print GitHub Actions collapsible group with test list.""" + test_count = count if count is not None else len(tests) + print(f"::group::{title} ({test_count})") + if note: + print(note) + print() + for test in tests: + print(test) + print("::endgroup::") + + @staticmethod + def print_statistics(stats: List[str]) -> None: + """Print statistics section.""" + if stats: + print("=== Test Statistics ===") + for stat in stats: + print(stat.rstrip()) + print() + + @staticmethod + def print_timing_summary(lines: List[str]) -> None: + """Print timing information section.""" + parser = LITLogParser(lines) + time_info = parser.extract_time_summary() + + testing_time = None + for line in lines: + if line.strip().startswith("Testing Time:"): + testing_time = line.strip() + break + + if not (time_info["slowest"] or time_info["histogram"] or testing_time): + return + + print("::group::Test Timing Summary") + + if testing_time: + print(testing_time) + print() + + if time_info["slowest"]: + print(SLOWEST_TESTS_HEADER) + print("-" * SEPARATOR_WIDTH) + for line in time_info["slowest"]: + print(line) + print() + + if time_info["histogram"]: + print("Test Times Distribution:") + print("-" * SEPARATOR_WIDTH) + for line in time_info["histogram"]: + print(line) + + print("::endgroup::") + + @staticmethod + def filter_log_for_display(lines: List[str]) -> List[str]: + """Remove statistics, test lists, and timing from log.""" + result = [] + skip_until_empty = False + in_timing = False + + for line in lines: + stripped = line.strip() + + # Skip statistics lines + if STATS_PATTERN.match(line): + continue + + # Skip test category sections + if TEST_CATEGORY_PATTERN.match(line): + skip_until_empty = True + continue + + # Skip timing sections + if stripped == SLOWEST_TESTS_HEADER or stripped in TEST_TIMES_HEADERS: + in_timing = True + continue + + if in_timing and stripped.replace("*", "") == "": + in_timing = False + continue + + if in_timing: + continue + + if stripped.startswith("Testing Time:"): + continue + + if skip_until_empty: + if not stripped: + skip_until_empty = False + continue + + result.append(line) + + return result diff --git a/devops/scripts/ur_test_tools/outputs/github_actions.py b/devops/scripts/ur_test_tools/outputs/github_actions.py new file mode 100644 index 0000000000000..e892ad3f2f036 --- /dev/null +++ b/devops/scripts/ur_test_tools/outputs/github_actions.py @@ -0,0 +1,20 @@ +"""GitHub Actions-specific output formatting.""" + +import sys + + +class GitHubActionsOutput: + """Format output for GitHub Actions.""" + + @staticmethod + def print_error(message: str) -> None: + print(f"::error::{message}", file=sys.stderr) + + @staticmethod + def print_warning(message: str) -> None: + print(f"::warning::{message}", file=sys.stderr) + + @staticmethod + def set_output(name: str, value: str) -> None: + print(f"{name}={value}", flush=True) + sys.stdout.flush() diff --git a/devops/scripts/ur_test_tools/parsers/__init__.py b/devops/scripts/ur_test_tools/parsers/__init__.py new file mode 100644 index 0000000000000..55721e06f10c6 --- /dev/null +++ b/devops/scripts/ur_test_tools/parsers/__init__.py @@ -0,0 +1,19 @@ +"""Parsers package - Data extraction from logs and XML.""" + +from .log_parser import ( + LITLogParser, + read_log_file, +) +from .xml_parser import ( + JUnitXMLParser, + ParsedXMLTests, +) +from .stats_parser import get_count_from_stats + +__all__ = [ + "LITLogParser", + "read_log_file", + "JUnitXMLParser", + "ParsedXMLTests", + "get_count_from_stats", +] diff --git a/devops/scripts/ur_test_tools/parsers/log_parser.py b/devops/scripts/ur_test_tools/parsers/log_parser.py new file mode 100644 index 0000000000000..2291973b0ebea --- /dev/null +++ b/devops/scripts/ur_test_tools/parsers/log_parser.py @@ -0,0 +1,151 @@ +"""Parse LIT text output for test information.""" + +import sys +from pathlib import Path +from typing import Iterator, List, Tuple + +from ..constants import ( + FAIL_TIMEOUT_PATTERN, + TEST_LIST_HEADER_PATTERN, + STATS_PATTERN, + TEST_CATEGORY_PATTERN, + SLOWEST_TESTS_HEADER, + TEST_TIMES_HEADERS, +) +from ..models.test_data import TestLists, TestCounts, TimingSummary + + +def _read_with_utf8_fallback(path: str, read_func): + try: + with open(path, "r", encoding="utf-8", errors="strict") as f: + return read_func(f) + except UnicodeDecodeError: + print( + f"Warning: File contains non-UTF-8 characters, replacing with U+FFFD", + file=sys.stderr, + ) + with open(path, "r", encoding="utf-8", errors="replace") as f: + return read_func(f) + + +def read_log_file(log_path: str) -> List[str]: + path = Path(log_path) + file_size = path.stat().st_size + + if file_size > 10 * 1024 * 1024: # 10 MB + print( + f"Large log file: {file_size / (1024 * 1024):.1f} MB. " + f"This may indicate a test problem.", + file=sys.stderr, + ) + + try: + return _read_with_utf8_fallback(log_path, lambda f: f.readlines()) + except OSError as e: + raise OSError(f"Cannot read log file: {e}") from e + + +class LITLogParser: + """Parse LIT (llvm-lit) text output for test information.""" + + def __init__(self, lines: List[str]): + self.lines = lines + + def extract_error_details(self) -> List[str]: + result = [] + in_error = False + + for line in self.lines: + if FAIL_TIMEOUT_PATTERN.match(line): + in_error = True + + # Stop at test list headers or timing summaries + if in_error and ( + TEST_LIST_HEADER_PATTERN.match(line) + or line.strip() == SLOWEST_TESTS_HEADER + or line.strip() in TEST_TIMES_HEADERS + ): + break + + if in_error: + result.append(line) + + return result + + def extract_statistics(self) -> List[str]: + return [line for line in self.lines if STATS_PATTERN.match(line)] + + def extract_time_summary(self) -> TimingSummary: + result: TimingSummary = {"slowest": [], "histogram": []} + current_section = None + skip_next_hr = False + + for line in self.lines: + stripped = line.strip() + + if stripped == SLOWEST_TESTS_HEADER: + current_section = "slowest" + skip_next_hr = True + continue + elif stripped in TEST_TIMES_HEADERS: + current_section = "histogram" + skip_next_hr = True + continue + + if skip_next_hr and stripped.startswith("---"): + skip_next_hr = False + continue + + if current_section == "slowest": + if not stripped: + current_section = None + elif not stripped.startswith("---"): + result["slowest"].append(line.rstrip()) + elif current_section == "histogram": + if not stripped: + current_section = None + elif stripped.replace("*", "") == "": + current_section = None + elif stripped.startswith("[") or stripped.replace("-", "") == "": + result["histogram"].append(line.rstrip()) + else: + current_section = None + + return result + + def extract_test_lists(self) -> Tuple[TestLists, TestCounts]: + categories: TestLists = {} + declared_counts: TestCounts = {} + current_category = None + current_tests = [] + current_declared_count = 0 + + for line in self.lines: + match = TEST_CATEGORY_PATTERN.match(line) + if match: + if current_category: + categories[current_category] = current_tests + declared_counts[current_category] = current_declared_count + + current_category = match.group(1) + current_declared_count = int(match.group(2)) + current_tests = [] + continue + + if current_category: + if not line.strip(): + categories[current_category] = current_tests + declared_counts[current_category] = current_declared_count + current_category = None + current_tests = [] + current_declared_count = 0 + else: + test_name = line.strip() + if test_name: + current_tests.append(test_name) + + if current_category: + categories[current_category] = current_tests + declared_counts[current_category] = current_declared_count + + return categories, declared_counts diff --git a/devops/scripts/ur_test_tools/parsers/stats_parser.py b/devops/scripts/ur_test_tools/parsers/stats_parser.py new file mode 100644 index 0000000000000..5424e6c084405 --- /dev/null +++ b/devops/scripts/ur_test_tools/parsers/stats_parser.py @@ -0,0 +1,16 @@ +"""Parse statistics from LIT output.""" + +import re +from typing import List + + +def get_count_from_stats(stats: List[str], keywords: List[str]) -> int: + # Build regex pattern from keywords (compile once per call) + pattern = re.compile("|".join(re.escape(kw) for kw in keywords)) + + for stat in stats: + if pattern.search(stat): + match = re.search(r"(\d+)", stat) + if match: + return int(match.group(1)) + return 0 diff --git a/devops/scripts/ur_test_tools/parsers/xml_parser.py b/devops/scripts/ur_test_tools/parsers/xml_parser.py new file mode 100644 index 0000000000000..ac2eb41e83700 --- /dev/null +++ b/devops/scripts/ur_test_tools/parsers/xml_parser.py @@ -0,0 +1,90 @@ +"""Parse JUnit XML test results.""" + +import sys +from pathlib import Path +from typing import List, NamedTuple, Optional + +import defusedxml.ElementTree as ET + +from ..constants import TEST_NOT_SELECTED_MSG + + +class ParsedXMLTests(NamedTuple): + """Skipped and excluded tests from XML parsing.""" + + skipped: List[str] + excluded: List[str] + + +def _format_test_name(classname: str, name: str) -> str: + if classname and name: + return f"{classname}.{name}" + return name + + +class JUnitXMLParser: + """Parse JUnit XML from LIT --xunit-xml-output.""" + + def __init__(self, xml_path: Optional[str]): + self.xml_path = xml_path + self._root = None + + def parse(self) -> bool: + if not self.xml_path: + return False + + xml_file = Path(self.xml_path) + if not xml_file.exists(): + return False + + try: + tree = ET.parse(self.xml_path) + self._root = tree.getroot() + return True + except ET.ParseError as e: + print( + f"Warning: Failed to parse XML file {self.xml_path}: {e}", + file=sys.stderr, + ) + return False + except (OSError, ValueError) as e: + print( + f"Warning: Error reading XML file {self.xml_path}: {e}", file=sys.stderr + ) + return False + + def extract_tests_from_xml(self) -> ParsedXMLTests: + if not self.parse(): + return ParsedXMLTests([], []) + + skipped = [] + excluded = [] + + for testcase in self._root.findall(".//testcase"): + skipped_elem = testcase.find("skipped") + if skipped_elem is None: + continue + + message = skipped_elem.get("message", "") + test_name = _format_test_name( + testcase.get("classname", ""), testcase.get("name", "") + ) + + if not test_name: + continue + + # Separate by message type + if TEST_NOT_SELECTED_MSG in message: + excluded.append(test_name) + else: + skipped.append(test_name) + + return ParsedXMLTests(skipped=skipped, excluded=excluded) + + def extract_skipped_tests(self) -> List[str]: + skipped, _ = self.extract_tests_from_xml() + return skipped + + def extract_excluded_tests(self) -> List[str]: + _, excluded = self.extract_tests_from_xml() + return excluded diff --git a/devops/scripts/ur_test_tools/summary_generator.py b/devops/scripts/ur_test_tools/summary_generator.py new file mode 100644 index 0000000000000..e4ccac6c8f8fa --- /dev/null +++ b/devops/scripts/ur_test_tools/summary_generator.py @@ -0,0 +1,224 @@ +"""Generate test summary reports.""" + +import sys +from typing import List + +from .models.config import SummaryConfigFromLines +from .models.test_data import ( + TestLists, + TestCounts, + SkippedTestsResult, + ExcludedTestsResult, +) +from .parsers.log_parser import LITLogParser +from .parsers.xml_parser import JUnitXMLParser +from .parsers.stats_parser import get_count_from_stats +from .outputs.console import ConsoleOutput +from .validation.data_validator import validate_test_counts + + +class SummaryReporter: + """Generate comprehensive test summary.""" + + def __init__(self, config: SummaryConfigFromLines): + self.config = config + + def generate(self) -> None: + parser = LITLogParser(self.config.log_lines) + stats = parser.extract_statistics() + test_lists, declared_counts = parser.extract_test_lists() + total_discovered = get_count_from_stats(stats, ["Total Discovered"]) + + xml_parser = JUnitXMLParser(self.config.xml_file) + parsed_xml = xml_parser.extract_tests_from_xml() + skipped_xml = parsed_xml.skipped + excluded_xml = parsed_xml.excluded + + ConsoleOutput.print_statistics(stats) + + skipped_result = self._analyze_skipped_tests(test_lists, stats, skipped_xml) + self._validate_skipped_counts(skipped_result, declared_counts, stats) + self._display_skipped_tests(skipped_result) + if skipped_result["count"] > 0: + self._cleanup_skipped_from_test_lists(test_lists) + + excluded_result = self._analyze_excluded_tests(test_lists, stats, excluded_xml) + self._validate_excluded_counts(excluded_result, declared_counts, stats) + self._display_excluded_tests(excluded_result) + if excluded_result["count"] > 0: + self._cleanup_excluded_from_test_lists(test_lists) + + self._display_remaining_categories(test_lists) + + validate_test_counts( + total_discovered, + test_lists, + skipped_result["count"], + excluded_result["count"], + ) + + ConsoleOutput.print_timing_summary(self.config.log_lines) + + def _analyze_skipped_tests( + self, test_lists: TestLists, stats: List[str], skipped_xml: List[str] + ) -> SkippedTestsResult: + """Analyze skipped tests (priority: XML > Log > Stats).""" + skipped_from_log = test_lists.get("Skipped", test_lists.get("Unsupported", [])) + stats_count = get_count_from_stats(stats, ["Skipped", "Unsupported"]) + + # Priority 1: XML data (most reliable - structured output) + if skipped_xml: + return SkippedTestsResult( + tests=skipped_xml, + count=len(skipped_xml), + source="xml", + note="", + ) + + # Priority 2: Log data + if skipped_from_log: + return SkippedTestsResult( + tests=skipped_from_log, + count=len(skipped_from_log), + source="log", + note="", + ) + + # Priority 3: Stats only (no individual test names) + if stats_count: + return SkippedTestsResult( + tests=[], + count=stats_count, + source="stats", + note="Warning: Test names not available", + ) + + # No data available + return SkippedTestsResult(tests=[], count=0, source="none", note="") + + def _validate_skipped_counts( + self, result: SkippedTestsResult, declared_counts: TestCounts, stats: List[str] + ) -> None: + """Validate skipped counts (warns on mismatch).""" + actual_count = result["count"] + if actual_count == 0: + return # Nothing to validate + + declared_count = declared_counts.get( + "Skipped", declared_counts.get("Unsupported", 0) + ) + stats_count = get_count_from_stats(stats, ["Skipped", "Unsupported"]) + + # Build list of mismatches + mismatches = [] + + if declared_count and declared_count != actual_count: + mismatches.append(f"log header: {declared_count}") + + if stats_count and stats_count != actual_count: + mismatches.append(f"statistics: {stats_count}") + + # Display warning only if mismatches found + if mismatches: + sources_str = ", ".join(mismatches) + print( + f"Warning: Skipped test count mismatch. " + f"Using {actual_count} from {result['source']}, " + f"but found {sources_str}", + file=sys.stderr, + ) + + def _display_skipped_tests(self, result: SkippedTestsResult) -> None: + if result["count"] > 0: + ConsoleOutput.print_test_group( + "Skipped Tests", + result["tests"], + note=result["note"], + count=result["count"] if not result["tests"] else None, + ) + + def _cleanup_skipped_from_test_lists(self, test_lists: TestLists) -> None: + test_lists.pop("Skipped", None) + test_lists.pop("Unsupported", None) + + def _analyze_excluded_tests( + self, test_lists: TestLists, stats: List[str], excluded_xml: List[str] + ) -> ExcludedTestsResult: + """Analyze excluded tests (priority: Log > XML > Stats).""" + excluded_from_log = test_lists.get("Excluded", []) + stats_count = get_count_from_stats(stats, ["Excluded"]) + + # Priority 1: Log data + if excluded_from_log: + return ExcludedTestsResult( + tests=excluded_from_log, + count=len(excluded_from_log), + source="log", + note="", + ) + + # Priority 2: XML data + if excluded_xml: + return ExcludedTestsResult( + tests=excluded_xml, count=len(excluded_xml), source="xml", note="" + ) + + # Priority 3: Stats only (no individual test names) + if stats_count: + return ExcludedTestsResult( + tests=[], + count=stats_count, + source="stats", + note="Warning: Test names not available", + ) + + # No data available + return ExcludedTestsResult(tests=[], count=0, source="none", note="") + + def _validate_excluded_counts( + self, result: ExcludedTestsResult, declared_counts: TestCounts, stats: List[str] + ) -> None: + """Validate excluded counts (warns on mismatch).""" + actual_count = result["count"] + if actual_count == 0: + return # Nothing to validate + + declared_count = declared_counts.get("Excluded", 0) + stats_count = get_count_from_stats(stats, ["Excluded"]) + + # Build list of mismatches + mismatches = [] + + if declared_count and declared_count != actual_count: + mismatches.append(f"log header: {declared_count}") + + if stats_count and stats_count != actual_count: + mismatches.append(f"statistics: {stats_count}") + + # Display warning only if mismatches found + if mismatches: + sources_str = ", ".join(mismatches) + print( + f"Warning: Excluded test count mismatch. " + f"Using {actual_count} from {result['source']}, " + f"but found {sources_str}", + file=sys.stderr, + ) + + def _display_excluded_tests(self, result: ExcludedTestsResult) -> None: + if result["count"] > 0: + ConsoleOutput.print_test_group( + "Excluded Tests", + result["tests"], + note=result["note"], + count=result["count"] if not result["tests"] else None, + ) + + def _cleanup_excluded_from_test_lists(self, test_lists: TestLists) -> None: + test_lists.pop("Excluded", None) + + def _display_remaining_categories(self, test_lists: TestLists) -> None: + for category, tests in test_lists.items(): + if tests: + ConsoleOutput.print_test_group(f"{category} Tests", tests) + diff --git a/devops/scripts/ur_test_tools/test_runner.py b/devops/scripts/ur_test_tools/test_runner.py new file mode 100644 index 0000000000000..136a43800412f --- /dev/null +++ b/devops/scripts/ur_test_tools/test_runner.py @@ -0,0 +1,160 @@ +"""Test execution.""" + +import os +import sys +import subprocess # nosec B404 - Used safely with list args, no shell=True +from pathlib import Path +from typing import List, Optional + +from .constants import ( + DEFAULT_LIT_TIMEOUT, + DEFAULT_LIT_JOBS, + TEST_TYPE_ADAPTER_SPECIFIC, + MAX_LINES_TO_SCAN, + MAX_JOBS, +) +from .models.config import TestConfig, TestExecutionContext +from .outputs.github_actions import GitHubActionsOutput +from .parsers.log_parser import _read_with_utf8_fallback + + +def get_test_config(test_type: str) -> TestConfig: + """Get test configuration for test type.""" + if test_type == "adapter-specific": + return TestConfig( + target="check-unified-runtime-adapter", + log_file="adapter_tests.log", + lit_filter_out=( + "(adapters/level_zero/memcheck.test|" + "adapters/level_zero/v2/deferred_kernel_memcheck.test)" + ), + ) + elif test_type == "conformance": + return TestConfig( + target="check-unified-runtime-conformance", + log_file="conformance_tests.log", + ) + else: + raise ValueError(f"Invalid test_type: {test_type}") + + +def calculate_jobs() -> int: + """Calculate parallel jobs (nproc/3 capped at MAX_JOBS).""" + try: + nproc = os.cpu_count() or 4 + return min(nproc // 3, MAX_JOBS) + except (OSError, AttributeError): + return 4 + + +def check_log_has_tests(log_file: str) -> bool: + """Check if log contains test results.""" + + def _scan_for_testing(f): + for _ in range(MAX_LINES_TO_SCAN): + line = f.readline() + if not line: + break + if "Testing:" in line: + return True + return False + + try: + return _read_with_utf8_fallback(log_file, _scan_for_testing) + except OSError: + return False + + +class TestRunner: + """Execute UR tests.""" + + def __init__(self, context: TestExecutionContext): + self.context = context + self.github_output = GitHubActionsOutput() + self.jobs = calculate_jobs() + + def run(self) -> int: + """Run tests and return exit code.""" + self._setup_environment() + + result = self._execute_tests() + if result is None: + return 1 + + if not self._validate_output(): + return 1 + + self._publish_outputs(result) + return result.returncode + + def _setup_environment(self) -> None: + lit_opts = ( + f"--show-unsupported --show-pass --show-xfail --no-progress-bar " + f"-v --timeout {DEFAULT_LIT_TIMEOUT} -j {DEFAULT_LIT_JOBS} " + f"--time-tests --show-flakypass --show-skipped " + f"--xunit-xml-output {self.context.xml_output_path}" + ) + self.context.env["LIT_OPTS"] = lit_opts + + if self.context.config.lit_filter_out: + self.context.env["LIT_FILTER_OUT"] = self.context.config.lit_filter_out + + self.context.env["ZE_ENABLE_LOADER_DEBUG_TRACE"] = "1" + + def _build_cmake_command(self) -> List[str]: + return [ + "cmake", + "--build", + str(self.context.build_dir), + "-j", + str(self.jobs), + "--", + self.context.config.target, + ] + + def _execute_tests(self) -> Optional[subprocess.CompletedProcess]: + cmd = self._build_cmake_command() + + print(f"Running: {' '.join(cmd)}", file=sys.stderr) + print(f"Log: {self.context.log_file_path}, Jobs: {self.jobs}", file=sys.stderr) + print(f"Expected XML: {self.context.xml_output_path}", file=sys.stderr) + + try: + with open(self.context.log_file_path, "w", encoding="utf-8") as log: + return subprocess.run( # nosec B603 B607 + cmd, + stdout=log, + stderr=subprocess.STDOUT, + env=self.context.env, + cwd=self.context.workspace, + ) + except (OSError, PermissionError) as e: + self.github_output.print_error(f"Test execution failed: {e}") + return None + + def _validate_output(self) -> bool: + log_path = self.context.log_file_path + + if not log_path.exists() or log_path.stat().st_size == 0: + self.github_output.print_error("No log generated") + return False + + return True + + def _publish_outputs(self, result: subprocess.CompletedProcess) -> None: + self.github_output.set_output("log-file", str(self.context.log_file_path)) + + if ( + self.context.test_type == TEST_TYPE_ADAPTER_SPECIFIC + and not check_log_has_tests(str(self.context.log_file_path)) + ): + print("No adapter-specific tests found", file=sys.stderr) + self.github_output.set_output("skip-artifacts", "1") + return + + if self.context.xml_output_path.exists(): + self.github_output.set_output("xml-file", str(self.context.xml_output_path)) + else: + self.github_output.print_warning( + f"Expected XML file not found at {self.context.xml_output_path}" + ) diff --git a/devops/scripts/ur_test_tools/validation/__init__.py b/devops/scripts/ur_test_tools/validation/__init__.py new file mode 100644 index 0000000000000..081e0c05ae8f7 --- /dev/null +++ b/devops/scripts/ur_test_tools/validation/__init__.py @@ -0,0 +1,9 @@ +"""Validation package - Security and data validation.""" + +from .path_validator import PathValidator +from .data_validator import validate_test_counts + +__all__ = [ + "PathValidator", + "validate_test_counts", +] diff --git a/devops/scripts/ur_test_tools/validation/data_validator.py b/devops/scripts/ur_test_tools/validation/data_validator.py new file mode 100644 index 0000000000000..b63cc715f685c --- /dev/null +++ b/devops/scripts/ur_test_tools/validation/data_validator.py @@ -0,0 +1,36 @@ +"""Test data validation.""" + +import sys +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from ..models.test_data import TestLists + + +def validate_test_counts( + total_discovered: int, + test_lists: "TestLists", + displayed_skipped: int, + displayed_excluded: int, +) -> None: + """Validate test counts match (prints warnings on mismatch).""" + if not total_discovered: + return + + sum_categories = sum(len(tests) for tests in test_lists.values()) + + # Add skipped tests if they were displayed but not in test_lists + if displayed_skipped > 0 and "Skipped" not in test_lists: + sum_categories += displayed_skipped + + # Add excluded tests if they were displayed but not in test_lists + if displayed_excluded > 0 and "Excluded" not in test_lists: + sum_categories += displayed_excluded + + if total_discovered != sum_categories: + print() + print( + f"::warning::Test count mismatch: Total Discovered = " + f"{total_discovered}, but sum of all categories = {sum_categories}" + ) + print() diff --git a/devops/scripts/ur_test_tools/validation/path_validator.py b/devops/scripts/ur_test_tools/validation/path_validator.py new file mode 100644 index 0000000000000..c2094e142ca62 --- /dev/null +++ b/devops/scripts/ur_test_tools/validation/path_validator.py @@ -0,0 +1,96 @@ +"""Path validation for security.""" + +from pathlib import Path +from typing import Optional + + +class PathValidator: + """Validate paths for security and correctness.""" + + @staticmethod + def validate_build_dir(build_dir: str, workspace: Optional[str] = None) -> bool: + """Validate build directory is safe and within workspace.""" + if not build_dir or ".." in build_dir or build_dir.startswith("/"): + return False + + # Block shell metacharacters, quotes, and control characters + # to prevent injection in f-strings, env vars, and logs + dangerous_chars = {";", "&", "#", "$", "|", "`", "\\", "'", '"', "\n", "\r"} + if any(c in build_dir for c in dangerous_chars): + return False + + if workspace: + try: + build_path = Path(build_dir).resolve(strict=False) + workspace_path = Path(workspace).resolve(strict=False) + build_path.relative_to(workspace_path) + return True + except (ValueError, OSError): + return False + return True + + @staticmethod + def validate_log_path(path: str) -> None: + """Validate log file path (detects path traversal).""" + try: + # Resolve path to detect encoded forms of path traversal (e.g., %2e%2e) + resolved = Path(path).resolve(strict=False) + + # Check for path traversal in original string (simple check) + if ".." in path: + raise ValueError( + f"Invalid log file path (path traversal not allowed): {path}" + ) + + # Verify file exists + if not resolved.exists(): + raise ValueError(f"Log file not found: {path}") + except (OSError, ValueError) as e: + if isinstance(e, ValueError): + raise + raise OSError(f"Invalid log file path: {path} ({e})") from e + + @staticmethod + def validate_optional_path( + path: str, path_type: str, allow_absolute: bool = False + ) -> str: + """Validate optional file path.""" + if not path: + return "" + + try: + # Resolve path to detect encoded forms of path traversal + Path(path).resolve(strict=False) + + # Check for path traversal in original string + if ".." in path: + raise ValueError( + f"Invalid {path_type} file path (path traversal): {path}" + ) + + # Check absolute path restriction + if not allow_absolute and path.startswith("/"): + raise ValueError( + f"Invalid {path_type} file path " + f"(absolute paths not allowed): {path}" + ) + except (OSError, ValueError) as e: + if isinstance(e, ValueError): + raise + raise OSError(f"Invalid {path_type} file path: {path} ({e})") from e + + return path + + @staticmethod + def ensure_within_workspace(path: Path, workspace: Path) -> Path: + """Ensure path is within workspace.""" + resolved = path.resolve() + workspace_resolved = workspace.resolve() + + try: + resolved.relative_to(workspace_resolved) + return resolved + except ValueError as e: + raise ValueError( + f"Path outside workspace: {path} not in {workspace}" + ) from e diff --git a/unified-runtime/test/CMakeLists.txt b/unified-runtime/test/CMakeLists.txt index 2d3e36efbc12f..30f700ca4355f 100644 --- a/unified-runtime/test/CMakeLists.txt +++ b/unified-runtime/test/CMakeLists.txt @@ -86,7 +86,8 @@ function(add_ur_lit_testsuite suite) if(UR_STANDALONE_BUILD) add_custom_target(${TARGET} - COMMAND "${URLIT_LIT_BINARY}" "${CMAKE_CURRENT_BINARY_DIR}" -sv + COMMAND "${URLIT_LIT_BINARY}" "${CMAKE_CURRENT_BINARY_DIR}" + --show-unsupported --show-pass --show-xfail --no-progress-bar --succinct --timeout 120 -j 50 --time-tests --show-flakypass --show-skipped USES_TERMINAL ) else() diff --git a/unified-runtime/test/conformance/CMakeLists.txt b/unified-runtime/test/conformance/CMakeLists.txt index 57b39ddbe7c4e..444805b22045a 100644 --- a/unified-runtime/test/conformance/CMakeLists.txt +++ b/unified-runtime/test/conformance/CMakeLists.txt @@ -41,8 +41,10 @@ foreach(adapter ${UR_ADAPTERS_LIST}) if(NOT "${adapter}" STREQUAL "mock") if(UR_STANDALONE_BUILD) add_custom_target(check-unified-runtime-conformance-${adapter} - COMMAND "${URLIT_LIT_BINARY}" "${CMAKE_CURRENT_BINARY_DIR}" - -v -Dselector=${adapter}:* + COMMAND ${CMAKE_COMMAND} -E env "LIT_OPTS=$ENV{LIT_OPTS}" + "${URLIT_LIT_BINARY}" "${CMAKE_CURRENT_BINARY_DIR}" + --show-unsupported --show-pass --show-xfail --no-progress-bar --succinct --timeout 120 -j 50 --time-tests --show-flakypass --show-skipped + -Dselector=${adapter}:* DEPENDS deps_check-unified-runtime-conformance ) else() diff --git a/unified-runtime/third_party/requirements_testing.txt b/unified-runtime/third_party/requirements_testing.txt index c41dbabde3fe3..2ac825e67cfbd 100644 --- a/unified-runtime/third_party/requirements_testing.txt +++ b/unified-runtime/third_party/requirements_testing.txt @@ -4,3 +4,6 @@ lit==18.1.8 # For timeouts in lit psutil==7.0.0 + +# For secure XML parsing in test result processing +defusedxml==0.7.1