Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
191 changes: 61 additions & 130 deletions .github/workflows/build-and-publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,160 +11,91 @@ on:

jobs:
build-wheels:
name: Build wheels on ${{ matrix.os }}
name: Wheels on ${{ matrix.os }}
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, windows-latest, macos-latest]
python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']

# ubuntu -> manylinux2014 x86_64 (glibc 2.17 baseline, runs on any HPC)
# macos-14 -> Apple Silicon runner; builds fat universal2 (x86_64+arm64)
# windows -> AMD64
os: [ubuntu-latest, macos-14, windows-latest]

steps:
- uses: actions/checkout@v4

- name: Free disk space (Ubuntu only)
if: runner.os == 'Linux'
run: |
df -h
sudo rm -rf /usr/local/lib/android
sudo rm -rf /usr/share/dotnet
sudo rm -rf /opt/ghc
df -h

- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}

- name: Install dependencies (Ubuntu)
if: runner.os == 'Linux'
run: |
sudo apt-get update
sudo apt-get install -y cmake libeigen3-dev build-essential
pip install --upgrade pip setuptools wheel pybind11 numpy scipy scikit-learn

- name: Install dependencies (macOS)
if: runner.os == 'macOS'
run: |
brew install cmake eigen
pip install --upgrade pip setuptools wheel pybind11 numpy scipy scikit-learn

- name: Install dependencies (Windows)
if: runner.os == 'Windows'
run: |
# Install cmake via choco
choco install cmake -y
# Do NOT upgrade pip here: GHA Windows images often block in-place pip
# self-upgrades ("ERROR: To modify pip, please run ..."), which fails
# the whole job. Build deps are enough for `pip wheel`.
python -m pip install --upgrade setuptools wheel pybind11 numpy scipy scikit-learn

- name: Setup MSVC (Windows only)
if: runner.os == 'Windows'
uses: ilammy/msvc-dev-cmd@v1
with:
arch: x64

- name: Build wheel
if: runner.os != 'Linux'
run: python -m pip wheel . --no-deps -w dist/ -v

- name: Build manylinux wheels
if: runner.os == 'Linux'
uses: docker://quay.io/pypa/manylinux2014_x86_64
# cibuildwheel builds + repairs (auditwheel/delocate/delvewheel) every
# selected CPython ABI in one go. All build selection / manylinux image /
# skip rules live in [tool.cibuildwheel] in pyproject.toml so the exact
# same build is reproducible locally via `pipx run cibuildwheel`.
- name: Build wheels
uses: pypa/cibuildwheel@v2.23.3
env:
# The latest scipy (transitive via scikit-learn) has no manylinux2014
# wheel; without this the post-build smoke test compiles it from source
# (no OpenBLAS) and fails. Force scipy to install from a binary wheel.
# Scoped to scipy so it can't block building hapc itself from source.
# CIBW_ENVIRONMENT_PASS_LINUX forwards it into the manylinux container,
# where it must apply to the wheel-under-test install (not test-requires,
# which runs too late).
PIP_ONLY_BINARY: scipy
CIBW_ENVIRONMENT_PASS_LINUX: PIP_ONLY_BINARY

- uses: actions/upload-artifact@v4
with:
entrypoint: /bin/bash
args: >
-c "
set -e -x &&
mkdir -p /github/workspace/dist &&
yum install -y eigen3-devel &&
export CMAKE_PREFIX_PATH=/usr/include/eigen3 &&
for PYBIN in /opt/python/*/bin; do
if [[ \${PYBIN} == *cp38* || \${PYBIN} == *cp39* || \${PYBIN} == *cp310* || \${PYBIN} == *cp311* || \${PYBIN} == *cp312* ]]; then
\${PYBIN}/pip wheel /github/workspace --no-deps -w /github/workspace/dist/ &&
\${PYBIN}/pip install auditwheel &&
auditwheel repair /github/workspace/dist/hapc-*-linux_x86_64.whl -w /github/workspace/dist/ &&
rm /github/workspace/dist/hapc-*-linux_x86_64.whl
fi
done
"

- name: List built wheels
shell: bash
run: ls -R dist/

- name: Clean up build artifacts
if: always() && runner.os != 'Windows'
shell: bash
run: |
rm -rf build/
rm -rf *.egg-info
rm -rf CMakeCache.txt

- name: Clean up build artifacts (Windows)
if: always() && runner.os == 'Windows'
shell: powershell
run: |
if (Test-Path "build") { Remove-Item -Recurse -Force "build" -ErrorAction SilentlyContinue }
if (Test-Path "*.egg-info") { Remove-Item -Recurse -Force "*.egg-info" -ErrorAction SilentlyContinue }
if (Test-Path "CMakeCache.txt") { Remove-Item -Force "CMakeCache.txt" -ErrorAction SilentlyContinue }

- name: Upload wheels to artifact
uses: actions/upload-artifact@v4
name: cibw-wheels-${{ matrix.os }}
path: wheelhouse/*.whl
if-no-files-found: error # never silently publish a release with missing wheels

build-sdist:
name: Build source distribution
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4

- name: Build sdist
run: pipx run build --sdist

- uses: actions/upload-artifact@v4
with:
name: wheels-${{ matrix.os }}-${{ matrix.python-version }}
path: dist/
name: cibw-sdist
path: dist/*.tar.gz
if-no-files-found: error

publish-to-pypi:
name: Publish to PyPI
needs: build-wheels
needs: [build-wheels, build-sdist]
runs-on: ubuntu-latest
if: startsWith(github.ref, 'refs/tags/')

steps:
- uses: actions/checkout@v4

- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.12'

- name: Install dependencies
run: |
pip install --upgrade pip setuptools wheel twine build

- name: Build source distribution
run: python -m build --sdist

- name: Download all wheels
- name: Download all artifacts
uses: actions/download-artifact@v4
with:
path: wheels-temp

- name: Organize wheels
run: |
mkdir -p dist
if [ -d wheels-temp ]; then
echo "Copying wheel artifacts from wheels-temp to dist/"
find wheels-temp -name "*.whl" -exec cp {} dist/ \;
else
echo "No wheels-temp directory found; proceeding with sdist only."
fi
pattern: cibw-*
path: dist
merge-multiple: true

- name: List files to publish
run: ls -R dist/

# Guard against the exact failure mode that shipped 2.3.0 without Linux
# wheels: refuse to publish unless a manylinux wheel is present.
- name: Verify Linux wheels are present
run: |
echo "Contents of dist/:"
ls -R dist/ || echo "dist/ directory is empty or missing"

if ! ls dist/*manylinux*_x86_64.whl >/dev/null 2>&1; then
echo "::error::No manylinux wheels found in dist/ — aborting publish."
exit 1
fi

- name: Publish to PyPI
env:
TWINE_USERNAME: __token__
TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
run: |
twine upload dist/* --skip-existing --verbose

run: pipx run twine upload dist/* --skip-existing --verbose

- name: Create GitHub Release
uses: softprops/action-gh-release@v1
uses: softprops/action-gh-release@v2
with:
files: dist/*
draft: false
Expand Down
10 changes: 8 additions & 2 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
cmake_minimum_required(VERSION 3.15)
cmake_minimum_required(VERSION 3.18)
project(hapc)

set(CMAKE_CXX_STANDARD 17)
Expand All @@ -15,7 +15,13 @@ endif()
# Python3_EXECUTABLE from setup.py so the build always targets the *same*
# interpreter that pip is using. Without this CMake may discover a newer/
# older system Python and produce a .so tagged for the wrong ABI.
find_package(Python3 COMPONENTS Interpreter Development REQUIRED)
#
# Use Development.Module (headers only), NOT the full Development component:
# the latter also requires Development.Embed -> libpython, which manylinux
# images deliberately do not ship (extension modules must not link libpython).
# Requiring full Development makes the manylinux build fail with
# "Could NOT find Python3 (missing: Python3_LIBRARIES Development.Embed)".
find_package(Python3 COMPONENTS Interpreter Development.Module REQUIRED)
message(STATUS "Python3_EXECUTABLE: ${Python3_EXECUTABLE}")
message(STATUS "Python3_VERSION: ${Python3_VERSION}")

Expand Down
31 changes: 28 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,41 @@ A fast and flexible machine learning library for nonparametric high-dimensional
pip install hapc
```

Prebuilt wheels are published for Linux (manylinux2014, x86_64), macOS
(Intel + Apple Silicon) and Windows, for CPython 3.8–3.12. No compiler,
CMake or Eigen is needed when a wheel is available.

### Linux / HPC clusters

The Linux wheels use the **manylinux2014** baseline (glibc 2.17), so
`pip install hapc` works out of the box on HPC login/compute nodes —
no `conda` toolchain, `devtoolset`, or sysroot setup required:

```bash
pip install hapc
```

If you must build from the source distribution (niche architecture, very
old Python, or an air-gapped node), provide a C++17 compiler and either
let CMake fetch Eigen automatically (needs network) or install Eigen and
let `find_package(Eigen3)` find it:

```bash
# with conda compilers (recommended on HPC)
conda install -c conda-forge cxx-compiler cmake eigen
pip install hapc --no-binary hapc
```

### Install from GitHub (latest development version)

```bash
pip install git+https://github.com/yourusername/hapc.git
pip install git+https://github.com/meixide/hapc.git
```

Or with editable install for development:

```bash
git clone https://github.com/yourusername/hapc.git
git clone https://github.com/meixide/hapc.git
cd hapc
pip install -e .
```
Expand Down Expand Up @@ -167,7 +192,7 @@ Cross-validation to select lambda.
Contributions welcome! The C++ core is shared between R and Python packages.

```bash
git clone https://github.com/yourusername/hapc.git
git clone https://github.com/meixide/hapc.git
cd hapc
pip install -e .
pytest
Expand Down
38 changes: 36 additions & 2 deletions pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
[build-system]
requires = ["setuptools>=65", "wheel", "cmake>=3.15", "pybind11>=2.6"]
requires = ["setuptools>=65", "wheel", "cmake>=3.18", "pybind11>=2.6"]
build-backend = "setuptools.build_meta"

[project]
name = "hapc"
version = "2.3.0"
version = "2.3.1"
description = "Highly Adaptive Principal Components"
readme = "README.md"
requires-python = ">=3.8"
Expand Down Expand Up @@ -34,3 +34,37 @@ Homepage = "https://github.com/meixide/hapc"
Documentation = "https://github.com/meixide/hapc#readme"
Repository = "https://github.com/meixide/hapc.git"
Issues = "https://github.com/meixide/hapc/issues"

[tool.cibuildwheel]
# Build CPython 3.8–3.12 only; skip PyPy and musl (HPC/desktop targets are glibc).
build = "cp38-* cp39-* cp310-* cp311-* cp312-*"
skip = ["pp*", "*-musllinux*"]
build-verbosity = 1
# Smoke-test every wheel: install it (pulling numpy/scikit-learn) and import the
# compiled extension. Catches wrong-ABI / unresolved-symbol wheels before publish.
test-command = "python -c \"import hapc; print(hapc.__version__)\""
# NOTE: the latest scipy (transitive via scikit-learn) no longer ships a
# manylinux2014 wheel, so the in-container smoke test would try to compile it
# from source and fail. We force scipy to resolve from a binary wheel via
# PIP_ONLY_BINARY=scipy, set in the workflow and passed into the Linux container
# (see .github/workflows/build-and-publish.yml). It must apply to the wheel
# install itself, which is why it lives in the env rather than test-requires.

[tool.cibuildwheel.linux]
archs = ["x86_64"]
# manylinux2014 -> glibc 2.17 baseline + bundled libstdc++ via auditwheel, so the
# wheel installs and runs unmodified on any HPC cluster (glibc >= 2.17), no
# compiler / conda toolchain / sysroot required.
manylinux-x86_64-image = "manylinux2014"

[tool.cibuildwheel.macos]
# Build fat universal2 wheels (x86_64 + arm64) from a single runner. CMake does
# not honour the interpreter's arch flags, so the arch is forced explicitly via
# CMAKE_ARGS below (setup.py appends $CMAKE_ARGS to the cmake invocation).
# delocate then verifies both slices are present, which is what caught the old
# single-arch-but-universal2-tagged wheels.
archs = ["universal2"]
environment = { CMAKE_ARGS = "-DCMAKE_OSX_ARCHITECTURES=arm64;x86_64" }

[tool.cibuildwheel.windows]
archs = ["AMD64"]
2 changes: 1 addition & 1 deletion python/hapc/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
- :func:`ate_hapc` — ATE estimate + Wald CI via HAPC + outcome undersmoothing.
"""

__version__ = "2.3.0"
__version__ = "2.3.1"

from .core import (
DesignOutput,
Expand Down
10 changes: 10 additions & 0 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
from setuptools import setup, find_packages, Extension
from setuptools.command.build_ext import build_ext
import os
import shlex
import subprocess
import sys
from pathlib import Path
Expand Down Expand Up @@ -44,6 +45,15 @@ def build_extension(self, ext):
build_args = ['--config', cfg]

cmake_args += ['-DCMAKE_BUILD_TYPE=' + cfg]

# Honour the conventional CMAKE_ARGS env var (set by cibuildwheel/conda).
# Used to force universal2 macOS builds via
# CMAKE_ARGS="-DCMAKE_OSX_ARCHITECTURES=arm64;x86_64", which CMake cannot
# infer from the (single-arch) build interpreter on its own.
extra_cmake_args = os.environ.get('CMAKE_ARGS')
if extra_cmake_args:
cmake_args += shlex.split(extra_cmake_args)

# Add parallel build flag only on non-Windows platforms
# On Windows, MSBuild doesn't support -j flag and handles parallelization automatically
if sys.platform != 'win32':
Expand Down
Loading