diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 97d9d5c..7060213 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -11,160 +11,91 @@ on: jobs: build-wheels: - name: Build wheels on ${{ matrix.os }} + name: Wheels on ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: - os: [ubuntu-latest, windows-latest, macos-latest] - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] - + # ubuntu -> manylinux2014 x86_64 (glibc 2.17 baseline, runs on any HPC) + # macos-14 -> Apple Silicon runner; builds fat universal2 (x86_64+arm64) + # windows -> AMD64 + os: [ubuntu-latest, macos-14, windows-latest] + steps: - uses: actions/checkout@v4 - - - name: Free disk space (Ubuntu only) - if: runner.os == 'Linux' - run: | - df -h - sudo rm -rf /usr/local/lib/android - sudo rm -rf /usr/share/dotnet - sudo rm -rf /opt/ghc - df -h - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies (Ubuntu) - if: runner.os == 'Linux' - run: | - sudo apt-get update - sudo apt-get install -y cmake libeigen3-dev build-essential - pip install --upgrade pip setuptools wheel pybind11 numpy scipy scikit-learn - - - name: Install dependencies (macOS) - if: runner.os == 'macOS' - run: | - brew install cmake eigen - pip install --upgrade pip setuptools wheel pybind11 numpy scipy scikit-learn - - - name: Install dependencies (Windows) - if: runner.os == 'Windows' - run: | - # Install cmake via choco - choco install cmake -y - # Do NOT upgrade pip here: GHA Windows images often block in-place pip - # self-upgrades ("ERROR: To modify pip, please run ..."), which fails - # the whole job. Build deps are enough for `pip wheel`. - python -m pip install --upgrade setuptools wheel pybind11 numpy scipy scikit-learn - - - name: Setup MSVC (Windows only) - if: runner.os == 'Windows' - uses: ilammy/msvc-dev-cmd@v1 - with: - arch: x64 - - - name: Build wheel - if: runner.os != 'Linux' - run: python -m pip wheel . --no-deps -w dist/ -v - - name: Build manylinux wheels - if: runner.os == 'Linux' - uses: docker://quay.io/pypa/manylinux2014_x86_64 + # cibuildwheel builds + repairs (auditwheel/delocate/delvewheel) every + # selected CPython ABI in one go. All build selection / manylinux image / + # skip rules live in [tool.cibuildwheel] in pyproject.toml so the exact + # same build is reproducible locally via `pipx run cibuildwheel`. + - name: Build wheels + uses: pypa/cibuildwheel@v2.23.3 + env: + # The latest scipy (transitive via scikit-learn) has no manylinux2014 + # wheel; without this the post-build smoke test compiles it from source + # (no OpenBLAS) and fails. Force scipy to install from a binary wheel. + # Scoped to scipy so it can't block building hapc itself from source. + # CIBW_ENVIRONMENT_PASS_LINUX forwards it into the manylinux container, + # where it must apply to the wheel-under-test install (not test-requires, + # which runs too late). + PIP_ONLY_BINARY: scipy + CIBW_ENVIRONMENT_PASS_LINUX: PIP_ONLY_BINARY + + - uses: actions/upload-artifact@v4 with: - entrypoint: /bin/bash - args: > - -c " - set -e -x && - mkdir -p /github/workspace/dist && - yum install -y eigen3-devel && - export CMAKE_PREFIX_PATH=/usr/include/eigen3 && - for PYBIN in /opt/python/*/bin; do - if [[ \${PYBIN} == *cp38* || \${PYBIN} == *cp39* || \${PYBIN} == *cp310* || \${PYBIN} == *cp311* || \${PYBIN} == *cp312* ]]; then - \${PYBIN}/pip wheel /github/workspace --no-deps -w /github/workspace/dist/ && - \${PYBIN}/pip install auditwheel && - auditwheel repair /github/workspace/dist/hapc-*-linux_x86_64.whl -w /github/workspace/dist/ && - rm /github/workspace/dist/hapc-*-linux_x86_64.whl - fi - done - " - - - name: List built wheels - shell: bash - run: ls -R dist/ - - - name: Clean up build artifacts - if: always() && runner.os != 'Windows' - shell: bash - run: | - rm -rf build/ - rm -rf *.egg-info - rm -rf CMakeCache.txt - - - name: Clean up build artifacts (Windows) - if: always() && runner.os == 'Windows' - shell: powershell - run: | - if (Test-Path "build") { Remove-Item -Recurse -Force "build" -ErrorAction SilentlyContinue } - if (Test-Path "*.egg-info") { Remove-Item -Recurse -Force "*.egg-info" -ErrorAction SilentlyContinue } - if (Test-Path "CMakeCache.txt") { Remove-Item -Force "CMakeCache.txt" -ErrorAction SilentlyContinue } - - - name: Upload wheels to artifact - uses: actions/upload-artifact@v4 + name: cibw-wheels-${{ matrix.os }} + path: wheelhouse/*.whl + if-no-files-found: error # never silently publish a release with missing wheels + + build-sdist: + name: Build source distribution + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Build sdist + run: pipx run build --sdist + + - uses: actions/upload-artifact@v4 with: - name: wheels-${{ matrix.os }}-${{ matrix.python-version }} - path: dist/ + name: cibw-sdist + path: dist/*.tar.gz + if-no-files-found: error publish-to-pypi: name: Publish to PyPI - needs: build-wheels + needs: [build-wheels, build-sdist] runs-on: ubuntu-latest if: startsWith(github.ref, 'refs/tags/') - + steps: - - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.12' - - - name: Install dependencies - run: | - pip install --upgrade pip setuptools wheel twine build - - - name: Build source distribution - run: python -m build --sdist - - - name: Download all wheels + - name: Download all artifacts uses: actions/download-artifact@v4 with: - path: wheels-temp - - - name: Organize wheels - run: | - mkdir -p dist - if [ -d wheels-temp ]; then - echo "Copying wheel artifacts from wheels-temp to dist/" - find wheels-temp -name "*.whl" -exec cp {} dist/ \; - else - echo "No wheels-temp directory found; proceeding with sdist only." - fi + pattern: cibw-* + path: dist + merge-multiple: true - name: List files to publish + run: ls -R dist/ + + # Guard against the exact failure mode that shipped 2.3.0 without Linux + # wheels: refuse to publish unless a manylinux wheel is present. + - name: Verify Linux wheels are present run: | - echo "Contents of dist/:" - ls -R dist/ || echo "dist/ directory is empty or missing" - + if ! ls dist/*manylinux*_x86_64.whl >/dev/null 2>&1; then + echo "::error::No manylinux wheels found in dist/ — aborting publish." + exit 1 + fi + - name: Publish to PyPI env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} - run: | - twine upload dist/* --skip-existing --verbose - + run: pipx run twine upload dist/* --skip-existing --verbose + - name: Create GitHub Release - uses: softprops/action-gh-release@v1 + uses: softprops/action-gh-release@v2 with: files: dist/* draft: false diff --git a/CMakeLists.txt b/CMakeLists.txt index 4adda53..136dc57 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.15) +cmake_minimum_required(VERSION 3.18) project(hapc) set(CMAKE_CXX_STANDARD 17) @@ -15,7 +15,13 @@ endif() # Python3_EXECUTABLE from setup.py so the build always targets the *same* # interpreter that pip is using. Without this CMake may discover a newer/ # older system Python and produce a .so tagged for the wrong ABI. -find_package(Python3 COMPONENTS Interpreter Development REQUIRED) +# +# Use Development.Module (headers only), NOT the full Development component: +# the latter also requires Development.Embed -> libpython, which manylinux +# images deliberately do not ship (extension modules must not link libpython). +# Requiring full Development makes the manylinux build fail with +# "Could NOT find Python3 (missing: Python3_LIBRARIES Development.Embed)". +find_package(Python3 COMPONENTS Interpreter Development.Module REQUIRED) message(STATUS "Python3_EXECUTABLE: ${Python3_EXECUTABLE}") message(STATUS "Python3_VERSION: ${Python3_VERSION}") diff --git a/README.md b/README.md index 4e6f11e..24bdfcb 100644 --- a/README.md +++ b/README.md @@ -17,16 +17,41 @@ A fast and flexible machine learning library for nonparametric high-dimensional pip install hapc ``` +Prebuilt wheels are published for Linux (manylinux2014, x86_64), macOS +(Intel + Apple Silicon) and Windows, for CPython 3.8–3.12. No compiler, +CMake or Eigen is needed when a wheel is available. + +### Linux / HPC clusters + +The Linux wheels use the **manylinux2014** baseline (glibc 2.17), so +`pip install hapc` works out of the box on HPC login/compute nodes — +no `conda` toolchain, `devtoolset`, or sysroot setup required: + +```bash +pip install hapc +``` + +If you must build from the source distribution (niche architecture, very +old Python, or an air-gapped node), provide a C++17 compiler and either +let CMake fetch Eigen automatically (needs network) or install Eigen and +let `find_package(Eigen3)` find it: + +```bash +# with conda compilers (recommended on HPC) +conda install -c conda-forge cxx-compiler cmake eigen +pip install hapc --no-binary hapc +``` + ### Install from GitHub (latest development version) ```bash -pip install git+https://github.com/yourusername/hapc.git +pip install git+https://github.com/meixide/hapc.git ``` Or with editable install for development: ```bash -git clone https://github.com/yourusername/hapc.git +git clone https://github.com/meixide/hapc.git cd hapc pip install -e . ``` @@ -167,7 +192,7 @@ Cross-validation to select lambda. Contributions welcome! The C++ core is shared between R and Python packages. ```bash -git clone https://github.com/yourusername/hapc.git +git clone https://github.com/meixide/hapc.git cd hapc pip install -e . pytest diff --git a/pyproject.toml b/pyproject.toml index 3f65775..9bb1365 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,10 @@ [build-system] -requires = ["setuptools>=65", "wheel", "cmake>=3.15", "pybind11>=2.6"] +requires = ["setuptools>=65", "wheel", "cmake>=3.18", "pybind11>=2.6"] build-backend = "setuptools.build_meta" [project] name = "hapc" -version = "2.3.0" +version = "2.3.1" description = "Highly Adaptive Principal Components" readme = "README.md" requires-python = ">=3.8" @@ -34,3 +34,37 @@ Homepage = "https://github.com/meixide/hapc" Documentation = "https://github.com/meixide/hapc#readme" Repository = "https://github.com/meixide/hapc.git" Issues = "https://github.com/meixide/hapc/issues" + +[tool.cibuildwheel] +# Build CPython 3.8–3.12 only; skip PyPy and musl (HPC/desktop targets are glibc). +build = "cp38-* cp39-* cp310-* cp311-* cp312-*" +skip = ["pp*", "*-musllinux*"] +build-verbosity = 1 +# Smoke-test every wheel: install it (pulling numpy/scikit-learn) and import the +# compiled extension. Catches wrong-ABI / unresolved-symbol wheels before publish. +test-command = "python -c \"import hapc; print(hapc.__version__)\"" +# NOTE: the latest scipy (transitive via scikit-learn) no longer ships a +# manylinux2014 wheel, so the in-container smoke test would try to compile it +# from source and fail. We force scipy to resolve from a binary wheel via +# PIP_ONLY_BINARY=scipy, set in the workflow and passed into the Linux container +# (see .github/workflows/build-and-publish.yml). It must apply to the wheel +# install itself, which is why it lives in the env rather than test-requires. + +[tool.cibuildwheel.linux] +archs = ["x86_64"] +# manylinux2014 -> glibc 2.17 baseline + bundled libstdc++ via auditwheel, so the +# wheel installs and runs unmodified on any HPC cluster (glibc >= 2.17), no +# compiler / conda toolchain / sysroot required. +manylinux-x86_64-image = "manylinux2014" + +[tool.cibuildwheel.macos] +# Build fat universal2 wheels (x86_64 + arm64) from a single runner. CMake does +# not honour the interpreter's arch flags, so the arch is forced explicitly via +# CMAKE_ARGS below (setup.py appends $CMAKE_ARGS to the cmake invocation). +# delocate then verifies both slices are present, which is what caught the old +# single-arch-but-universal2-tagged wheels. +archs = ["universal2"] +environment = { CMAKE_ARGS = "-DCMAKE_OSX_ARCHITECTURES=arm64;x86_64" } + +[tool.cibuildwheel.windows] +archs = ["AMD64"] diff --git a/python/hapc/__init__.py b/python/hapc/__init__.py index 256e463..1bf28fe 100644 --- a/python/hapc/__init__.py +++ b/python/hapc/__init__.py @@ -19,7 +19,7 @@ - :func:`ate_hapc` — ATE estimate + Wald CI via HAPC + outcome undersmoothing. """ -__version__ = "2.3.0" +__version__ = "2.3.1" from .core import ( DesignOutput, diff --git a/setup.py b/setup.py index bdde9ed..695bc45 100644 --- a/setup.py +++ b/setup.py @@ -3,6 +3,7 @@ from setuptools import setup, find_packages, Extension from setuptools.command.build_ext import build_ext import os +import shlex import subprocess import sys from pathlib import Path @@ -44,6 +45,15 @@ def build_extension(self, ext): build_args = ['--config', cfg] cmake_args += ['-DCMAKE_BUILD_TYPE=' + cfg] + + # Honour the conventional CMAKE_ARGS env var (set by cibuildwheel/conda). + # Used to force universal2 macOS builds via + # CMAKE_ARGS="-DCMAKE_OSX_ARCHITECTURES=arm64;x86_64", which CMake cannot + # infer from the (single-arch) build interpreter on its own. + extra_cmake_args = os.environ.get('CMAKE_ARGS') + if extra_cmake_args: + cmake_args += shlex.split(extra_cmake_args) + # Add parallel build flag only on non-Windows platforms # On Windows, MSBuild doesn't support -j flag and handles parallelization automatically if sys.platform != 'win32':