diff --git a/.bazelrc b/.bazelrc index baf91fc72..17a6d839d 100644 --- a/.bazelrc +++ b/.bazelrc @@ -29,7 +29,9 @@ build --protocopt=--experimental_allow_proto3_optional build --incompatible_require_linker_input_cc_api=false build:macos --apple_platform_type=macos +build:macos --repo_env=BAZEL_USE_CPP_ONLY_TOOLCHAIN=1 build:macos_arm64 --cpu=darwin_arm64 +build:macos_arm64 --repo_env=BAZEL_USE_CPP_ONLY_TOOLCHAIN=1 build:macos_arm64 --linkopt=-Wl,-undefined,dynamic_lookup build:macos_arm64 --host_linkopt=-Wl,-undefined,dynamic_lookup build --conlyopt=-std=c11 diff --git a/.github/reusable-build/action.yml b/.github/reusable-build/action.yml index 8997fb4a6..1e68d11ba 100644 --- a/.github/reusable-build/action.yml +++ b/.github/reusable-build/action.yml @@ -9,7 +9,7 @@ runs: steps: - name: Set up Python ${{ inputs.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 with: python-version: ${{ inputs.python-version }} @@ -20,14 +20,15 @@ runs: - name: Build the package for Python ${{ inputs.python-version }} shell: bash + env: + PYTHON_VERSION: ${{ inputs.python-version }} run: | - version="${{ inputs.python-version }}" - DOCKER_SERVICE=manylinux-python$(echo "$version" | sed 's/\.//') + DOCKER_SERVICE=manylinux-python$(echo "$PYTHON_VERSION" | sed 's/\.//') docker compose build ${DOCKER_SERVICE} docker compose run ${DOCKER_SERVICE} - name: Upload wheel artifact for Python ${{ inputs.python-version }} - uses: actions/upload-artifact@v4.4.0 + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 with: name: ml-metadata-wheel-py${{ inputs.python-version }} path: dist/*.whl diff --git a/.github/workflows/cd-docs.yml b/.github/workflows/cd-docs.yml index cc529ce4c..0c6f0428e 100644 --- a/.github/workflows/cd-docs.yml +++ b/.github/workflows/cd-docs.yml @@ -4,29 +4,38 @@ on: push: branches: - 'master' - pull_request: + permissions: - contents: write + contents: read + jobs: deploy: + name: Deploy to GitHub Pages runs-on: ubuntu-latest + # zizmor: ignore[excessive-permissions] + permissions: + # Required to deploy documentation to GitHub Pages (gh-pages branch) + contents: write steps: - name: Checkout repo - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false - name: Set up Python 3.12 - uses: actions/setup-python@v5 + uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 with: python-version: '3.12' cache: 'pip' cache-dependency-path: | setup.py + requirements-docs.txt - name: Save time for cache for mkdocs run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV - name: Caching - uses: actions/cache@v4 + uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 with: key: mkdocs-material-${{ env.cache_id }} path: .cache @@ -37,12 +46,7 @@ jobs: run: pip install -r requirements-docs.txt - name: Deploy to GitHub Pages - if: (github.event_name != 'pull_request') run: | git config user.name github-actions[bot] git config user.email 41898282+github-actions[bot]@users.noreply.github.com mkdocs gh-deploy --force - - - name: Build docs to check for errors - run: mkdocs build - if: (github.event_name == 'pull_request') diff --git a/.github/workflows/ci-docs.yml b/.github/workflows/ci-docs.yml new file mode 100644 index 000000000..00753c820 --- /dev/null +++ b/.github/workflows/ci-docs.yml @@ -0,0 +1,47 @@ +name: Check docs + +on: + pull_request: + branches: + - 'master' + +permissions: + contents: read + +jobs: + build: + name: Build docs check + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout repo + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + + - name: Set up Python 3.12 + uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 + with: + python-version: '3.12' + cache: 'pip' + cache-dependency-path: | + setup.py + requirements-docs.txt + + - name: Save time for cache for mkdocs + run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV + + - name: Caching + uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 + with: + key: mkdocs-material-${{ env.cache_id }} + path: .cache + restore-keys: | + mkdocs-material- + + - name: Install Dependencies + run: pip install -r requirements-docs.txt + + - name: Build docs to check for errors + run: mkdocs build diff --git a/.github/workflows/conda-build.yml b/.github/workflows/conda-build.yml index 21e7ebac1..ede6e6780 100644 --- a/.github/workflows/conda-build.yml +++ b/.github/workflows/conda-build.yml @@ -11,6 +11,9 @@ on: types: [published] workflow_dispatch: +permissions: + contents: read + jobs: build: runs-on: ${{ matrix.os }} @@ -22,10 +25,12 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false - name: Setup Micromamba - uses: mamba-org/setup-micromamba@v1 + uses: mamba-org/setup-micromamba@4b9113af4fba0e9e1124b252dd6497a419e7396d # v1.11.0 with: environment-file: ${{ matrix.os == 'macos-latest' && 'ci/environment-macos.yml' || 'ci/environment.yml' }} cache-environment: true @@ -42,14 +47,24 @@ jobs: shell: bash -l {0} run: | # Install Bazelisk (manages Bazel versions) + ARCH="amd64" + if [ "$(uname -m)" == "arm64" ]; then + ARCH="arm64" + fi + + mkdir -p "$HOME/.local/bin" if [ "$RUNNER_OS" == "Linux" ]; then - curl -Lo /tmp/bazelisk https://github.com/bazelbuild/bazelisk/releases/download/v1.20.0/bazelisk-linux-amd64 + curl -Lo "$HOME/.local/bin/bazel" "https://github.com/bazelbuild/bazelisk/releases/download/v1.25.0/bazelisk-linux-${ARCH}" elif [ "$RUNNER_OS" == "macOS" ]; then - curl -Lo /tmp/bazelisk https://github.com/bazelbuild/bazelisk/releases/download/v1.20.0/bazelisk-darwin-amd64 + curl -Lo "$HOME/.local/bin/bazel" "https://github.com/bazelbuild/bazelisk/releases/download/v1.25.0/bazelisk-darwin-${ARCH}" + fi + chmod +x "$HOME/.local/bin/bazel" + if [ -n "$CONDA_PREFIX" ]; then + cp "$HOME/.local/bin/bazel" "$CONDA_PREFIX/bin/bazel" fi - chmod +x /tmp/bazelisk - sudo mv /tmp/bazelisk /usr/local/bin/bazel + echo "$HOME/.local/bin" >> $GITHUB_PATH echo "USE_BAZEL_VERSION=7.7.0" >> $GITHUB_ENV + echo "BAZEL_USE_CPP_ONLY_TOOLCHAIN=1" >> $GITHUB_ENV bazel --version - name: Install build tooling @@ -60,6 +75,7 @@ jobs: - name: Build the package shell: bash -l {0} run: | + unset APPLE_SDK_VERSION_OVERRIDE XCODE_VERSION_OVERRIDE rm -rf build build_mlmd_tmp dist python -m build --wheel --no-isolation @@ -73,7 +89,7 @@ jobs: rm "${WHEEL_PATH}" - name: Upload wheel artifact - uses: actions/upload-artifact@v4.4.0 + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 with: name: ml-metadata-wheel-${{ matrix.os }}-py${{ matrix.python-version }} path: dist/*.whl @@ -86,9 +102,11 @@ jobs: environment: name: pypi url: https://pypi.org/p/ml-metadata/ + permissions: + contents: read steps: - name: Retrieve wheels - uses: actions/download-artifact@v4.1.8 + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 with: merge-multiple: true path: wheels @@ -98,7 +116,8 @@ jobs: ls -lAs wheels/ - name: Upload to PyPI - uses: pypa/gh-action-pypi-publish@release/v1.9 + # zizmor: ignore[use-trusted-publishing] + uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # release/v1 with: packages_dir: wheels/ user: __token__ diff --git a/.github/workflows/conda-test.yml b/.github/workflows/conda-test.yml index 4284fee22..ec2b98ae7 100644 --- a/.github/workflows/conda-test.yml +++ b/.github/workflows/conda-test.yml @@ -9,6 +9,9 @@ on: - master workflow_dispatch: +permissions: + contents: read + jobs: test: runs-on: ${{ matrix.os }} @@ -20,10 +23,12 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false - name: Setup Micromamba - uses: mamba-org/setup-micromamba@v1 + uses: mamba-org/setup-micromamba@4b9113af4fba0e9e1124b252dd6497a419e7396d # v1.11.0 with: environment-file: ${{ matrix.os == 'macos-latest' && 'ci/environment-macos.yml' || 'ci/environment.yml' }} cache-environment: true @@ -40,19 +45,30 @@ jobs: shell: bash -l {0} run: | # Install Bazelisk (manages Bazel versions) + ARCH="amd64" + if [ "$(uname -m)" == "arm64" ]; then + ARCH="arm64" + fi + + mkdir -p "$HOME/.local/bin" if [ "$RUNNER_OS" == "Linux" ]; then - curl -Lo /tmp/bazelisk https://github.com/bazelbuild/bazelisk/releases/download/v1.20.0/bazelisk-linux-amd64 + curl -Lo "$HOME/.local/bin/bazel" "https://github.com/bazelbuild/bazelisk/releases/download/v1.25.0/bazelisk-linux-${ARCH}" elif [ "$RUNNER_OS" == "macOS" ]; then - curl -Lo /tmp/bazelisk https://github.com/bazelbuild/bazelisk/releases/download/v1.20.0/bazelisk-darwin-amd64 + curl -Lo "$HOME/.local/bin/bazel" "https://github.com/bazelbuild/bazelisk/releases/download/v1.25.0/bazelisk-darwin-${ARCH}" + fi + chmod +x "$HOME/.local/bin/bazel" + if [ -n "$CONDA_PREFIX" ]; then + cp "$HOME/.local/bin/bazel" "$CONDA_PREFIX/bin/bazel" fi - chmod +x /tmp/bazelisk - sudo mv /tmp/bazelisk /usr/local/bin/bazel + echo "$HOME/.local/bin" >> $GITHUB_PATH echo "USE_BAZEL_VERSION=7.7.0" >> $GITHUB_ENV + echo "BAZEL_USE_CPP_ONLY_TOOLCHAIN=1" >> $GITHUB_ENV bazel --version - name: Build the package shell: bash -l {0} run: | + unset APPLE_SDK_VERSION_OVERRIDE XCODE_VERSION_OVERRIDE rm -rf build build_mlmd_tmp dist python setup.py bdist_wheel diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 0a1eb0c8e..ed883a314 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -5,10 +5,15 @@ on: push: branches: [master] +permissions: + contents: read + jobs: pre-commit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - - uses: pre-commit/action@v3.0.1 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 + - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 diff --git a/ci/environment-macos.yml b/ci/environment-macos.yml index 1e6ef593e..fc138c5ee 100644 --- a/ci/environment-macos.yml +++ b/ci/environment-macos.yml @@ -14,9 +14,5 @@ dependencies: - six # Required by ZetaSQL build tools - libiconv # Required by libmysqlclient for character encoding conversions - # C/C++ compilers - - clang - - clangxx - - pip: - delocate # For macOS wheel repair diff --git a/setup.py b/setup.py index f2e809278..e0e66e9e2 100644 --- a/setup.py +++ b/setup.py @@ -99,9 +99,16 @@ def finalize_options(self): # This flag determines the platform qualifier of the macos wheel. if platform.machine() == 'arm64': self._additional_build_options = ['--macos_minimum_os=11.0', - '--config=macos_arm64'] + '--config=macos_arm64', + '--repo_env=BAZEL_USE_CPP_ONLY_TOOLCHAIN=1'] else: - self._additional_build_options = ['--macos_minimum_os=10.14'] + self._additional_build_options = ['--macos_minimum_os=10.14', + '--repo_env=BAZEL_USE_CPP_ONLY_TOOLCHAIN=1'] + + # Remove any overrides from conda environment that break Bazel on macOS + os.environ.pop('APPLE_SDK_VERSION_OVERRIDE', None) + os.environ.pop('XCODE_VERSION_OVERRIDE', None) + os.environ['BAZEL_USE_CPP_ONLY_TOOLCHAIN'] = '1' if 'ICONV_LIBRARIES' in os.environ: self._additional_build_options.append(