diff --git a/.github/workflows/repository-policy.yml b/.github/workflows/repository-policy.yml index 11bcdb2..66e2d73 100644 --- a/.github/workflows/repository-policy.yml +++ b/.github/workflows/repository-policy.yml @@ -19,3 +19,7 @@ jobs: - name: Check tracked text file integrity run: python3 scripts/tests/check_text_integrity.py + + - name: Check PR base is current + if: github.event_name == 'pull_request' + run: python3 scripts/tests/check_pr_base_current.py diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml index d229718..6ff6e26 100644 --- a/.github/workflows/shellcheck.yml +++ b/.github/workflows/shellcheck.yml @@ -56,3 +56,21 @@ jobs: echo "Checking ${#files[@]} shell files at -S error level" shellcheck -S error -f gcc "${files[@]}" echo "shellcheck error-level: 0" + + - name: Run warning-level shellcheck on changed app scripts + if: github.event_name == 'pull_request' + shell: bash + run: | + set -euo pipefail + git fetch --no-tags --depth=1 origin "+refs/heads/${GITHUB_BASE_REF}:refs/remotes/origin/${GITHUB_BASE_REF}" + mapfile -t files < <( + git diff --name-only --diff-filter=ACMRT "origin/${GITHUB_BASE_REF}...HEAD" -- ':(glob)programs/**/*.sh' | + sort + ) + if [ "${#files[@]}" -eq 0 ]; then + echo "No changed app shell files found; warning-level shellcheck skipped." + exit 0 + fi + echo "Checking ${#files[@]} changed app shell files at -S warning level" + shellcheck -S warning -f gcc "${files[@]}" + echo "changed app shellcheck warning-level: 0" diff --git a/config/queue.csv b/config/queue.csv index 3991079..631299b 100644 --- a/config/queue.csv +++ b/config/queue.csv @@ -1,6 +1,6 @@ queue,submit_cmd,template SLURM_RIKYU,sbatch,"-p ${queue_group} ${scheduler_extra_args} -t ${elapse} -N ${nodes} --ntasks-per-node=${numproc_node} --cpus-per-task=${nthreads} --gpus=${proc}" -FJ,pjsub,"-L rscunit=rscunit_ft01,rscgrp=${queue_group},elapse=${elapse},node=${nodes} --mpi max-proc-per-node=${numproc_node} -x PJM_LLIO_GFSCACHE=/vol0002:/vol0003:/vol0004:/vol0005" +FJ,pjsub,"${scheduler_extra_args} -L rscunit=rscunit_ft01,rscgrp=${queue_group},elapse=${elapse},node=${nodes} --mpi max-proc-per-node=${numproc_node} -x PJM_LLIO_GFSCACHE=/vol0002:/vol0003:/vol0004:/vol0005" PJM_GENKAI,pjsub,"-L rscgrp=${queue_group},elapse=${elapse},node=${nodes} --mpi proc=${proc}" SLURM_RC,sbatch,"-p ${queue_group} ${scheduler_extra_args} -t ${elapse} -N ${nodes} --ntasks-per-node=${numproc_node} --cpus-per-task=${nthreads}" PBS_Miyabi,qsub,"-q ${queue_group} -l select=${nodes}:mpiprocs=${numproc_node}:ompthreads=${nthreads} -l walltime=${elapse} -W group_list=jh260034" diff --git a/docs/ci.md b/docs/ci.md index b08763a..60ca654 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -72,7 +72,7 @@ The workflow accepts these inputs: | `target_ref` | Branch, tag, or SHA in the upstream repository to test / upstreamリポジトリ内でテストするbranch、tag、SHA | `feature/my-change`, `ci/pr-123`, `develop` | | `code` | Benchkit program filter / Benchkitプログラムのフィルタ | `qws,genesis` | | `system` | Benchkit system filter. Legacy Benchpark bridge jobs in this repo do not honor this as a general system selector. / Benchkit systemフィルタ。このrepo内のlegacy Benchpark bridge jobは汎用system selectorとしては扱いません | `Fugaku,MiyabiG` | -| `allocation_project_id` | Optional semantic project/allocation ID passed to GitLab as `BK_ALLOCATION_PROJECT_ID`. Benchkit translates it to scheduler syntax only for systems that require it, for example Slurm `--account=` on RIKYU. / GitLab へ `BK_ALLOCATION_PROJECT_ID` として渡す任意の意味的な project/allocation ID。Benchkit は必要な system に限って scheduler 書式へ変換します。例: RIKYU の Slurm `--account=` | `rkp00010` | +| `allocation_project_id` | Optional semantic project/allocation ID passed to GitLab as `BK_ALLOCATION_PROJECT_ID`. Benchkit translates it to scheduler syntax only for systems that require it, for example Slurm `--account=` on RIKYU and PJM `-g ` on Fugaku. / GitLab へ `BK_ALLOCATION_PROJECT_ID` として渡す任意の意味的な project/allocation ID。Benchkit は必要な system に限って scheduler 書式へ変換します。例: RIKYU の Slurm `--account=`、Fugaku の PJM `-g ` | `rkp00010` | | `app` | Legacy Benchpark bridge app filter. Active Benchpark CI/CD/CB result handling is maintained in a separate project. / legacy Benchpark bridge appフィルタ。現行Benchpark CI/CD/CB結果受け取りは別プロジェクト側で管理します | `osu-micro-benchmarks` | | `benchpark` | Enable the legacy Benchpark bridge path together with Benchkit / legacy Benchpark bridge pathも有効化 | `true` | | `park_only` | Run only the legacy Benchpark bridge path / legacy Benchpark bridgeのみ実行 | `true` | @@ -156,7 +156,7 @@ The recommended mechanism is pipeline variables. `GitLab Manual CI` uses pipelin |---|---|---| | `system` | Benchkit system filter. Legacy Benchpark bridge jobs in this repo are not a general multi-system Benchpark runner. / Benchkit systemフィルタ。このrepo内のlegacy Benchpark bridge jobは汎用multi-system Benchpark runnerではありません | `MiyabiG,MiyabiC,RC_GENOA` | | `code` | Benchkit program filter / Benchkit programフィルタ | `qws,genesis` | -| `BK_ALLOCATION_PROJECT_ID` | Optional semantic project/allocation ID. Benchkit validates the value and derives scheduler arguments only for systems that support it. / 任意の意味的な project/allocation ID。Benchkit は値を検証し、対応 system に限って scheduler 引数へ変換します | `rkp00010` | +| `BK_ALLOCATION_PROJECT_ID` | Optional semantic project/allocation ID. Benchkit validates the value and derives scheduler arguments only for systems that support it: `--account=` on RIKYU and `-g ` on Fugaku/FugakuCN. / 任意の意味的な project/allocation ID。Benchkit は値を検証し、対応 system に限って scheduler 引数へ変換します。RIKYU では `--account=`、Fugaku/FugakuCN では `-g ` になります | `rkp00010` | | `app` | Legacy Benchpark bridge app filter. Active Benchpark CI/CD/CB result handling has moved to a separate project. / legacy Benchpark bridge appフィルタ。現行Benchpark CI/CD/CB結果受け取りは別プロジェクト側へ移行済み | `osu-micro-benchmarks` | | `benchpark` | Enable the legacy Benchpark bridge path / legacy Benchpark bridge pathを有効化 | `true` | | `park_only` | Run the legacy Benchpark bridge and skip the normal Benchkit matrix / legacy Benchpark bridgeのみ実行し通常Benchkit matrixをスキップ | `true` | diff --git a/docs/guides/add-app.md b/docs/guides/add-app.md index 03a8800..5e011c3 100644 --- a/docs/guides/add-app.md +++ b/docs/guides/add-app.md @@ -472,6 +472,9 @@ cat programs//list.csv # 1行目の設定でテスト実行 bash scripts/test_submit.sh 1 + +# Fugakuでdefault group以外を使う場合 +BK_ALLOCATION_PROJECT_ID=ra000009 bash scripts/test_submit.sh 1 ``` ### test_submit.sh の機能 diff --git a/scripts/job_functions.sh b/scripts/job_functions.sh index 64febd7..4013212 100644 --- a/scripts/job_functions.sh +++ b/scripts/job_functions.sh @@ -99,6 +99,9 @@ scheduler_args_from_allocation_project() { RIKYU) printf '%s\n' "--account=${allocation_project_id}" ;; + Fugaku|FugakuCN) + printf '%s\n' "-g ${allocation_project_id}" + ;; esac return 0 } diff --git a/scripts/test_submit.sh b/scripts/test_submit.sh index c24a223..a8ec425 100644 --- a/scripts/test_submit.sh +++ b/scripts/test_submit.sh @@ -96,7 +96,7 @@ echo "Parsed values:" echo " system=$system, enable=$enable, mode=$mode (from system.csv), queue_group=$queue_group (from system.csv)" echo " nodes=$nodes, numproc_node=$numproc_node, nthreads=$nthreads, elapse=$elapse" if [[ -n "$scheduler_extra_args" ]]; then - echo " scheduler_extra_args=$scheduler_extra_args (from BK_SCHEDULER_EXTRA_ARGS or BK_SCHEDULER_EXTRA_ARGS_${system})" + echo " scheduler_extra_args=$scheduler_extra_args (from BK_SCHEDULER_EXTRA_ARGS*, or BK_ALLOCATION_PROJECT_ID for supported systems)" fi # --- 投入用スクリプト作成 --- @@ -106,12 +106,12 @@ echo bash programs/$code/run.sh $system $nodes $numproc_node $nthreads >> script # --- システム別ジョブ投入 --- case "$system" in Fugaku|FugakuCN) - echo pjsub -L rscunit=rscunit_ft01,rscgrp=$queue_group,node=$nodes,elapse=$elapse \ + echo pjsub "${scheduler_extra_args_array[@]}" -L rscunit=rscunit_ft01,rscgrp=$queue_group,node=$nodes,elapse=$elapse \ --mpi max-proc-per-node=$numproc_node \ -S -x PJM_LLIO_GFSCACHE=/vol0002:/vol0003:/vol0004:/vol0005 \ script.sh - pjsub -L rscunit=rscunit_ft01,rscgrp=$queue_group,node=$nodes,elapse=$elapse \ + pjsub "${scheduler_extra_args_array[@]}" -L rscunit=rscunit_ft01,rscgrp=$queue_group,node=$nodes,elapse=$elapse \ --mpi max-proc-per-node=$numproc_node \ -S -x PJM_LLIO_GFSCACHE=/vol0002:/vol0003:/vol0004:/vol0005 \ script.sh diff --git a/scripts/tests/check_pr_base_current.py b/scripts/tests/check_pr_base_current.py new file mode 100644 index 0000000..af8b25b --- /dev/null +++ b/scripts/tests/check_pr_base_current.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""Fail pull-request policy checks when the tested base SHA is stale.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + + +def run_git(*args: str) -> str: + return subprocess.check_output(["git", *args], text=True).strip() + + +def main() -> int: + if os.environ.get("GITHUB_EVENT_NAME") != "pull_request": + print("Not a pull_request event; PR base freshness check skipped.") + return 0 + + event_path = os.environ.get("GITHUB_EVENT_PATH") + if not event_path: + print("GITHUB_EVENT_PATH is not set", file=sys.stderr) + return 1 + + with Path(event_path).open(encoding="utf-8") as handle: + event = json.load(handle) + + pull_request = event.get("pull_request") or {} + base = pull_request.get("base") or {} + base_ref = base.get("ref") + tested_base_sha = base.get("sha") + if not base_ref or not tested_base_sha: + print("pull_request.base.ref or pull_request.base.sha is missing", file=sys.stderr) + return 1 + + subprocess.run( + [ + "git", + "fetch", + "--no-tags", + "--depth=1", + "origin", + f"+refs/heads/{base_ref}:refs/remotes/origin/{base_ref}", + ], + check=True, + ) + current_base_sha = run_git("rev-parse", f"refs/remotes/origin/{base_ref}") + + if current_base_sha != tested_base_sha: + print( + f"PR base is stale: this run tested {base_ref} at {tested_base_sha}, " + f"but origin/{base_ref} is now {current_base_sha}.", + file=sys.stderr, + ) + print("Update/rebase the PR branch or rerun CI after refreshing the base.", file=sys.stderr) + return 1 + + print(f"PR base is current: {base_ref} {tested_base_sha}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/tests/test_scheduler_extra_args.sh b/scripts/tests/test_scheduler_extra_args.sh index 4966ed1..9fa79d5 100644 --- a/scripts/tests/test_scheduler_extra_args.sh +++ b/scripts/tests/test_scheduler_extra_args.sh @@ -13,8 +13,9 @@ unset BK_SCHEDULER_EXTRA_ARGS_RIKYU unset BK_SCHEDULER_EXTRA_ARGS_RC_GH200 test "$(get_scheduler_extra_args RIKYU)" = "--account=rkp00010" +test "$(get_scheduler_extra_args Fugaku)" = "-g rkp00010" +test "$(get_scheduler_extra_args FugakuCN)" = "-g rkp00010" test "$(get_scheduler_extra_args RC_GH200)" = "" -test "$(get_scheduler_extra_args Fugaku)" = "" export BK_ALLOCATION_PROJECT_ID="rkp00010 --qos=debug" if get_scheduler_extra_args RIKYU >/dev/null 2>&1; then @@ -29,6 +30,7 @@ test "$(get_scheduler_extra_args RIKYU)" = "--account=explicit-rikyu" unset BK_SCHEDULER_EXTRA_ARGS_RIKYU export BK_SCHEDULER_EXTRA_ARGS="--account=global" test "$(get_scheduler_extra_args RIKYU)" = "--account=global" +test "$(get_scheduler_extra_args Fugaku)" = "--account=global" tmpdir="" estimate_tmpdir=$(mktemp -d) @@ -60,6 +62,7 @@ export BK_TEST_SBATCH_ARGS_FILE="$tmpdir/sbatch.args" export BK_ALLOCATION_PROJECT_ID="rkp00010" unset BK_SCHEDULER_EXTRA_ARGS unset BK_SCHEDULER_EXTRA_ARGS_RIKYU +unset BK_SCHEDULER_EXTRA_ARGS_Fugaku unset BK_SCHEDULER_EXTRA_ARGS_RC_GH200 bash scripts/test_submit_build.sh qws 5 >/dev/null @@ -72,6 +75,27 @@ export BK_SCHEDULER_EXTRA_ARGS_RC_GH200="--account=explicit-rc" bash scripts/test_submit_build.sh qws 5 >/dev/null grep -q -- "--account=explicit-rc" "$BK_TEST_SBATCH_ARGS_FILE" +cat >"$tmpdir/pjsub" <<'SCRIPT' +#!/bin/bash +printf '%s\n' "$*" >"${BK_TEST_PJSUB_ARGS_FILE:?}" +SCRIPT +chmod +x "$tmpdir/pjsub" + +export BK_TEST_PJSUB_ARGS_FILE="$tmpdir/pjsub.args" +export BK_ALLOCATION_PROJECT_ID="ra000009" +unset BK_SCHEDULER_EXTRA_ARGS +unset BK_SCHEDULER_EXTRA_ARGS_Fugaku + +bash scripts/test_submit.sh qws 2 >/dev/null +grep -q -- "-g ra000009" "$BK_TEST_PJSUB_ARGS_FILE" + +unset BK_ALLOCATION_PROJECT_ID +bash scripts/test_submit.sh qws 2 >/dev/null +if grep -q -- "-g" "$BK_TEST_PJSUB_ARGS_FILE"; then + echo "Fugaku must not pass -g when BK_ALLOCATION_PROJECT_ID is unset" >&2 + exit 1 +fi + popd >/dev/null echo "scheduler extra args test passed"