Migrate CI environment to pixi

.github/actions/setup-ci-tools/action.yml

@@ -2,52 +2,14 @@ name: 'Setup nmaci CI Tools'
 description: 'Install the Neuromatch nmaci package'
 
 inputs:
-  branch:
-    description: 'nmaci branch to use'
-    required: false
-    default: 'main'
-  commit-message:
-    description: 'Commit message to parse for branch override'
-    required: false
-    default: ''
   stub-widgets:
     description: 'Install ipywidgets stub for headless execution (disable for book builds)'
     required: false
     default: 'true'
 
-outputs:
-  nmaci-branch:
-    description: 'nmaci branch used'
-    value: ${{ steps.detect-branch.outputs.branch }}
-
 runs:
   using: 'composite'
   steps:
-    - name: Detect nmaci branch
-      id: detect-branch
-      shell: bash
-      env:
-        COMMIT_MESSAGE: ${{ inputs.commit-message }}
-      run: |
-        BRANCH="${{ inputs.branch }}"
-        if [ -n "$COMMIT_MESSAGE" ]; then
-          OVERRIDE=$(python3 -c "import os, re; m = re.search(r'nmaci:([\w-]+)', os.environ.get('COMMIT_MESSAGE', '')); print(m.group(1) if m else '')")
-          if [ -n "$OVERRIDE" ]; then
-            BRANCH="$OVERRIDE"
-          fi
-        fi
-        echo "branch=$BRANCH" >> $GITHUB_OUTPUT
-        echo "Using nmaci branch: $BRANCH"
-
-    - name: Install nmaci package
-      shell: bash
-      run: |
-        set -e
-        pip install --upgrade pip
-        BRANCH=${{ steps.detect-branch.outputs.branch }}
-        pip install "git+https://github.com/neuromatch/nmaci@$BRANCH"
-        python -c "import nbformat; print(f'nbformat version: {nbformat.__version__}')"
-
     - name: Stub ipywidgets for headless kernel execution
       if: inputs.stub-widgets == 'true'
       shell: bash

.github/actions/setup-python-env/action.yml

@@ -1,32 +1,39 @@
 name: 'Setup Python Environment'
-description: 'Set up Python with caching and install base dependencies'
+description: 'Set up the pixi Python environment'
 
 inputs:
-  python-version:
-    description: 'Python version to install'
+  environment:
+    description: 'Pixi environment to install and add to PATH'
     required: false
-    default: '3.10'
+    default: 'ci'
 
 outputs:
   cache-hit:
-    description: 'Whether pip cache was hit'
-    value: ${{ steps.setup-python.outputs.cache-hit }}
+    description: 'Whether the pixi environment cache was hit'
+    value: ${{ steps.cache-pixi.outputs.cache-hit }}
 
 runs:
   using: 'composite'
   steps:
-    - name: Set up Python
-      id: setup-python
-      uses: actions/setup-python@v5
-      with:
-        python-version: ${{ inputs.python-version }}
-        cache: 'pip'
-        cache-dependency-path: requirements.txt
-
-    - name: Upgrade pip
+    - name: Install pixi
       shell: bash
-      run: pip install --upgrade pip wheel
+      run: |
+        if ! command -v pixi >/dev/null 2>&1; then
+          curl -fsSL https://pixi.sh/install.sh | bash
+        fi
+        echo "$HOME/.pixi/bin" >> "$GITHUB_PATH"
+
+    - name: Cache pixi environment
+      id: cache-pixi
+      uses: actions/cache@v4
+      with:
+        path: |
+          .pixi
+          ~/.cache/rattler
+        key: pixi-${{ runner.os }}-${{ inputs.environment }}-${{ hashFiles('pixi.toml', 'pixi.lock') }}
 
-    - name: Install requirements
+    - name: Install pixi environment
       shell: bash
-      run: pip install -r requirements.txt
+      run: |
+        "$HOME/.pixi/bin/pixi" install --locked -e "${{ inputs.environment }}"
+        echo "$GITHUB_WORKSPACE/.pixi/envs/${{ inputs.environment }}/bin" >> "$GITHUB_PATH"

.github/actions/setup-rendering-deps/action.yml

@@ -1,52 +1,41 @@
 name: 'Setup Rendering Dependencies'
-description: 'Install fonts, backend libraries, and graphviz for notebook rendering'
+description: 'Install fonts, backend libraries, and configure rendering for notebooks'
 
 inputs:
   skip-fonts:
     description: 'Skip XKCD font installation'
     required: false
     default: 'false'
-  skip-backend:
-    description: 'Skip backend libraries'
-    required: false
-    default: 'false'
-  skip-graphviz:
-    description: 'Skip graphviz installation'
-    required: false
-    default: 'false'
-
 runs:
   using: 'composite'
   steps:
     - name: Cache fonts
       id: cache-fonts
       uses: actions/cache@v4
       with:
-        path: /usr/share/fonts/truetype/humor-sans
+        path: ~/.local/share/fonts/humor-sans
         key: fonts-${{ runner.os }}-humor-sans-v1
 
     - name: Install XKCD fonts
       if: ${{ inputs.skip-fonts != 'true' }}
       shell: bash
       run: |
-        if [ ! -f /usr/share/fonts/truetype/humor-sans/Humor-Sans.ttf ]; then
-          sudo apt-get update -yq
-          wget -q http://archive.ubuntu.com/ubuntu/pool/universe/f/fonts-humor-sans/fonts-humor-sans_1.0-4_all.deb
-          sudo dpkg -i --force-all fonts-humor-sans_1.0-4_all.deb <<< 'yes' 2>/dev/null || true
-          sudo apt install -fy
-          rm -f fonts-humor-sans_1.0-4_all.deb $HOME/.matplotlib/fontList.cache
+        font_dir="$HOME/.local/share/fonts/humor-sans"
+        font_file="$font_dir/Humor-Sans.ttf"
+        if [ ! -f "$font_file" ]; then
+          mkdir -p "$font_dir"
+          curl -fsSL \
+            -o "$font_file" \
+            https://raw.githubusercontent.com/shreyankg/xkcd-desktop/master/Humor-Sans.ttf
+          fc-cache -f "$HOME/.local/share/fonts"
+          rm -f "$HOME"/.matplotlib/fontlist-v*.json "$HOME/.matplotlib/fontList.cache"
         else
           echo "XKCD fonts already cached"
         fi
 
-    - name: Install backend libraries
-      if: ${{ inputs.skip-backend != 'true' }}
+    - name: Configure headless rendering
       shell: bash
       run: |
         sudo apt-get update -yq || true
         sudo apt-get install -y libglew-dev libglfw3 ffmpeg
         echo "MUJOCO_GL=egl" >> $GITHUB_ENV
-
-    - name: Install Graphviz
-      if: ${{ inputs.skip-graphviz != 'true' }}
-      uses: tlylt/install-graphviz@v1

.github/workflows/notebook-pr.yaml

@@ -27,8 +27,9 @@ jobs:
       skip_ci: ${{ steps.check-skip.outputs.skip }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v5
         with:
+          repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
           persist-credentials: false
           fetch-depth: 0
           ref: ${{ github.head_ref || github.ref_name }}
@@ -57,8 +58,6 @@ jobs:
       - name: Setup CI tools
         if: steps.check-skip.outputs.skip != 'true'
         uses: ./.github/actions/setup-ci-tools
-        with:
-          commit-message: ${{ steps.get-commit.outputs.message }}
 
       - name: Get changed files
         if: steps.check-skip.outputs.skip != 'true' && github.event_name == 'pull_request'
@@ -113,8 +112,9 @@ jobs:
         notebook: ${{ fromJson(needs.setup.outputs.notebooks) }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v5
         with:
+          repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
           persist-credentials: false
           fetch-depth: 0
           ref: ${{ github.head_ref || github.ref_name }}
@@ -124,13 +124,9 @@ jobs:
 
       - name: Setup CI tools
         uses: ./.github/actions/setup-ci-tools
-        with:
-          commit-message: ${{ needs.setup.outputs.commit_message }}
 
       - name: Setup rendering dependencies
         uses: ./.github/actions/setup-rendering-deps
-        with:
-          skip-backend: 'true'
 
       - name: Process notebook
         env:
@@ -189,22 +185,18 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v5
         with:
+          repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
           persist-credentials: false
           fetch-depth: 0
           ref: ${{ github.head_ref }}
 
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.10'
-          cache: 'pip'
+      - name: Setup Python environment
+        uses: ./.github/actions/setup-python-env
 
       - name: Setup CI tools
         uses: ./.github/actions/setup-ci-tools
-        with:
-          commit-message: ${{ needs.setup.outputs.commit_message }}
 
       - name: Download all artifacts
         uses: actions/download-artifact@v4

.github/workflows/publish-book-v2.yml

@@ -40,19 +40,17 @@ jobs:
 
       - name: Setup Python environment
         uses: ./.github/actions/setup-python-env
+        with:
+          environment: book-v2
 
       - name: Setup Node.js (required by MyST theme engine)
         uses: actions/setup-node@v4
         with:
           node-version: '20'
 
-      - name: Install Jupyter Book 2
-        run: pip install "jupyter-book>=2.1"
-
       - name: Setup CI tools
         uses: ./.github/actions/setup-ci-tools
         with:
-          commit-message: ${{ env.COMMIT_MESSAGE }}
           stub-widgets: 'false'
 
       - name: Setup rendering dependencies
@@ -67,7 +65,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: book/_build/execute
-          key: jb2-exec-v2-${{ steps.cache-date.outputs.date }}-${{ hashFiles('tutorials/**/*.ipynb', 'requirements.txt') }}
+          key: jb2-exec-v2-${{ steps.cache-date.outputs.date }}-${{ hashFiles('tutorials/**/*.ipynb', 'pixi.toml', 'pixi.lock') }}
           restore-keys: |
             jb2-exec-v2-${{ steps.cache-date.outputs.date }}-
             jb2-exec-v2-

.github/workflows/publish-book.yml

@@ -32,14 +32,12 @@ jobs:
 
       - name: Setup Python environment
         uses: ./.github/actions/setup-python-env
-
-      - name: Install Jupyter Book
-        run: pip install jupyter-book==0.14.0 ghp-import
+        with:
+          environment: book
 
       - name: Setup CI tools
         uses: ./.github/actions/setup-ci-tools
         with:
-          commit-message: ${{ env.COMMIT_MESSAGE }}
           stub-widgets: 'false'
 
       - name: Setup rendering dependencies
@@ -67,7 +65,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: book/.jupyter-cache
-          key: jupyter-exec-${{ steps.cache-date.outputs.date }}-${{ hashFiles('tutorials/**/*.ipynb', 'requirements.txt') }}
+          key: jupyter-exec-${{ steps.cache-date.outputs.date }}-${{ hashFiles('tutorials/**/*.ipynb', 'pixi.toml', 'pixi.lock') }}
           restore-keys: |
             jupyter-exec-${{ steps.cache-date.outputs.date }}-
             jupyter-exec-

.github/workflows/tutorial-readme-pr.yaml

@@ -19,22 +19,14 @@ jobs:
           fetch-depth: 0
           ref: ${{ github.head_ref }}
 
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: 3.9
+      - name: Setup Python environment
+        uses: ./.github/actions/setup-python-env
 
-      - name: Install CI tools
-        run: |
-          BRANCH=`python -c 'import os, re; m = re.search(r"nmaci:([\w-]+)", os.environ["COMMIT_MESSAGE"]); print("main" if m is None else m.group(1))'`
-          wget https://github.com/NeuromatchAcademy/nmaci/archive/refs/heads/$BRANCH.tar.gz
-          tar -xzf $BRANCH.tar.gz
-          mv nmaci-$BRANCH/scripts/ ci/
-          rm -r nmaci-$BRANCH
-          echo ci/ >> .gitignore
+      - name: Setup CI tools
+        uses: ./.github/actions/setup-ci-tools
 
       - name: Update READMEs
-        run: python ci/generate_tutorial_readmes.py
+        run: nmaci generate-readmes
 
       - name: Commit post-processed files
         if: ${{ success() }}

README.md

@@ -15,6 +15,26 @@ Please check out [expected prerequisites here](https://github.com/neuromatch/Neu
 
 ---
 
+## Development Environment
+
+This repository uses [pixi](https://pixi.sh/) to manage the course environment.
+
+Install pixi by following the [official installation instructions](https://pixi.sh/latest/installation/), or on Linux/macOS run:
+
+```bash
+curl -fsSL https://pixi.sh/install.sh | bash
+```
+
+Then install the locked course environment from the repository root:
+
+```bash
+pixi install --locked
+```
+
+Use plain `pixi install` only when intentionally updating the environment, and commit both `pixi.toml` and `pixi.lock` after changing dependencies.
+
+---
+
 ## Licensing
 
 [![CC BY 4.0][cc-by-image]][cc-by]

book/README.md

@@ -26,13 +26,15 @@ In order to build the book locally, you will need to do the following:
 
 `cd course-content`
 
-`pip install -r ../nmaci-main/requirements.txt`
+Install pixi by following the [official installation instructions](https://pixi.sh/latest/installation/), or on Linux/macOS run:
 
-`pip install -r requirements.txt`
+`curl -fsSL https://pixi.sh/install.sh | bash`
 
-`pip install jupyter-book==0.10.2`
+Install the locked legacy book environment:
 
-**Important:** Do not install jupyter-book 0.11 or later at this point, as there are breaking changes in how it handles the table of contents file we generate.
+`pixi install --locked -e book`
+
+**Important:** Use the pinned Jupyter Book version from the `book` pixi environment instead of installing a different version manually.
 
 5. Copy files from precourse to `tutorials/`
 
@@ -54,7 +56,7 @@ In order to build the book locally, you will need to do the following:
 
 7. Prepare repo for book building 
 
-`python ../nmaci-main/scripts/generate_book.py arg`
+`pixi run -e book python ../nmaci-main/scripts/generate_book.py arg`
 
 where `arg` can take either `student` or `instructor` as a value.
 
@@ -64,6 +66,6 @@ This will use the modified tutorials/materials.yml to create the `_toc.yml` file
 
 8. Build the book
 
-`jupyter-book build book`
+`pixi run -e book jupyter-book build book`
 
 This will create a `book/_build` directory. You can open the `index.html` in any browser to verify the book.