diff --git a/.github/workflows/repository-policy.yml b/.github/workflows/repository-policy.yml index 66e2d73..18e4d11 100644 --- a/.github/workflows/repository-policy.yml +++ b/.github/workflows/repository-policy.yml @@ -16,10 +16,15 @@ jobs: steps: - name: Check out repository uses: actions/checkout@v4 + with: + fetch-depth: 0 - name: Check tracked text file integrity run: python3 scripts/tests/check_text_integrity.py + - name: Check commit messages + run: python3 scripts/tests/check_commit_messages.py + - name: Check PR base is current if: github.event_name == 'pull_request' run: python3 scripts/tests/check_pr_base_current.py diff --git a/docs/ci.md b/docs/ci.md index 60ca654..00417a2 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -16,8 +16,8 @@ Benchkit uses GitHub as the public development repository and GitLab CI for benc | `Sync protected branches to GitLab` | Pushes to `develop` or `main` / `develop`または`main`へのpush | Mirrors protected branches to GitLab without starting GitLab CI / GitLab CIを発火させずに保護ブランチをGitLabへ同期する | | `Guard main PR source` | Pull requests to `main` / `main`宛PR | Allows only upstream `develop` to target `main` / upstreamの`develop`から`main`へのPRだけを許可する | | `Result Server Tests` | All pull requests and pushes to `develop` or `main` / 全pull requestと`develop`または`main`へのpush | Runs site config preflight, result server tests, and profiler/profile-data/estimation shell tests / site config preflight、result server test、profiler/profile-data/estimation shell testを実行する | -| `Shellcheck` | Shell script changes under `scripts/`, `programs/`, or `benchpark-bridge/scripts/` / `scripts/`、`programs/`、または`benchpark-bridge/scripts/`配下のshell script変更 | Gates `bash -n` syntax failures and shellcheck error-level issues / `bash -n`構文エラーとshellcheckのerror級をgateする | -| `Repository Policy` | All pull requests and pushes to `develop` or `main` / 全pull requestと`develop`または`main`へのpush | Validates tracked text files as UTF-8 without replacement characters / 管理下text fileがUTF-8かつreplacement characterなしであることを検証する | +| `Shellcheck` | Shell script changes under `scripts/`, `programs/`, or `benchpark-bridge/scripts/` / `scripts/`、`programs/`、または`benchpark-bridge/scripts/`配下のshell script変更 | Gates `bash -n` syntax failures and shellcheck error-level issues; pull requests also run warning-level shellcheck for changed app scripts / `bash -n`構文エラーとshellcheckのerror級をgateする。pull requestでは変更されたapp scriptをwarning級でも確認する | +| `Repository Policy` | All pull requests and pushes to `develop` or `main` / 全pull requestと`develop`または`main`へのpush | Validates tracked text files, commit-message metadata, and current PR base / 管理下text file、commit message metadata、PR baseが最新であることを検証する | ## GitLab Secrets / GitLab secret @@ -284,9 +284,9 @@ cache miss の場合は通常の `programs//build.sh` 経路に戻り、 ## Lightweight Repository Policy / 軽量repository policy -`Repository Policy` runs on every pull request and on pushes to `develop` or `main`. It runs `scripts/tests/check_text_integrity.py` to ensure tracked text-like files decode as UTF-8 and do not contain the U+FFFD replacement character. +`Repository Policy` runs on every pull request and on pushes to `develop` or `main`. It runs `scripts/tests/check_text_integrity.py` to ensure tracked text-like files decode as UTF-8 and do not contain the U+FFFD replacement character. It also runs `scripts/tests/check_commit_messages.py` to reject external AI tool session URLs and private tooling session identifiers in newly introduced commit messages. On pull requests, `scripts/tests/check_pr_base_current.py` confirms that the CI run used the current base branch SHA. -`Repository Policy` は全pull requestと `develop` / `main` へのpushで動きます。`scripts/tests/check_text_integrity.py` を実行し、管理下のtext系fileがUTF-8としてdecodeでき、U+FFFD replacement characterを含まないことを確認します。 +`Repository Policy` は全pull requestと `develop` / `main` へのpushで動きます。`scripts/tests/check_text_integrity.py` を実行し、管理下のtext系fileがUTF-8としてdecodeでき、U+FFFD replacement characterを含まないことを確認します。さらに `scripts/tests/check_commit_messages.py` で、新規に入るcommit messageに外部AI toolのsession URLやprivate tooling session識別子が含まれないことを確認します。pull requestでは `scripts/tests/check_pr_base_current.py` により、CI runが最新のbase branch SHAを使っていることも確認します。 The periodic review should therefore focus on semantic checks that are hard to encode in CI: stale documentation, implementation/design mismatches, workflow review, unused-code candidates, ownership-boundary drift, and research-snapshot consistency. It should not need to repeat the lightweight executable checks unless a CI result itself looks suspicious. @@ -300,7 +300,7 @@ The periodic review should therefore focus on semantic checks that are hard to e | `result_server/**/*` / `result_server/**/*` | `Result Server Tests` | Skipped by `.gitlab-ci.yml` rules / `.gitlab-ci.yml` rulesでskip | Portal regressions should be caught by lightweight Python tests / portal回帰はlightweight Python testで捕捉する | | Public site config or portal metadata `config/system.csv`, `config/queue.csv`, `config/system_info.csv` / 公開site configまたはportal表示メタデータ`config/system.csv`、`config/queue.csv`、`config/system_info.csv` | `Result Server Tests`, including site config preflight / site config preflightを含む`Result Server Tests` | `config/system.csv` and `config/queue.csv` run by `.gitlab-ci.yml`; `config/system_info.csv` is skipped / `config/system.csv`と`config/queue.csv`は`.gitlab-ci.yml`で実行、`config/system_info.csv`はskip | Public systems listed in `system_info.csv` must also exist in `system.csv` and reference a queue defined in `queue.csv` / `system_info.csv`に載せる公開systemは`system.csv`にも存在し、`queue.csv`定義済みqueueを参照する必要がある | | Portal upload or profile-data helper `scripts/bk_functions.sh`, `scripts/result.sh`, `scripts/result_server/**` / portal uploadまたはprofile-data helper `scripts/bk_functions.sh`、`scripts/result.sh`、`scripts/result_server/**` | `Result Server Tests`; `Shellcheck` for `.sh` changes / `Result Server Tests`; `.sh`変更は`Shellcheck` | GitHub pull requests do not start GitLab by default; if a direct/manual GitLab pipeline is started, `scripts/**/*` is treated as benchmark-affecting and runs / GitHub pull requestでは既定でGitLabは起動しない。直接/手動GitLab pipelineを起動した場合、`scripts/**/*` はbenchmark影響ありとして実行される | These helpers shape result JSON / upload behavior. Use lightweight tests first, then start `GitLab Manual CI` when benchmark-side behavior needs validation / これらのhelperはResult JSONやupload挙動へ影響する。まずlightweight testで確認し、benchmark側挙動の検証が必要な場合は`GitLab Manual CI`を起動する | -| Benchmark app entrypoints `programs/**/build.sh`, `programs/**/estimate.sh`, `programs/**/run.sh`, or app matrix `programs/**/list.csv` / benchmark app entrypoint `programs/**/build.sh`、`programs/**/estimate.sh`、`programs/**/run.sh`、またはapp matrix `programs/**/list.csv` | `Result Server Tests` for portal app-support visibility and diagnostics; `Shellcheck` for `.sh` changes / portal app-support表示とdiagnostics向けに`Result Server Tests`; `.sh`変更は`Shellcheck` | Run through `GitLab Manual CI` when maintainer starts it / maintainerが`GitLab Manual CI`を起動した場合に実行 | App support and diagnostics read list/build/estimate/run for `/results/usage`; these checks provide visibility, not a readiness gate / app supportとdiagnosticsは`/results/usage`用にlist/build/estimate/runを読む。これはvisibilityでありreadiness gateではない | +| Benchmark app entrypoints `programs/**/build.sh`, `programs/**/estimate.sh`, `programs/**/run.sh`, or app matrix `programs/**/list.csv` / benchmark app entrypoint `programs/**/build.sh`、`programs/**/estimate.sh`、`programs/**/run.sh`、またはapp matrix `programs/**/list.csv` | `Result Server Tests` for portal app-support visibility and diagnostics; `Shellcheck` for `.sh` changes, with PR app-script changes checked at warning level / portal app-support表示とdiagnostics向けに`Result Server Tests`; `.sh`変更は`Shellcheck`、PRのapp script変更はwarning級でも確認 | Run through `GitLab Manual CI` when maintainer starts it / maintainerが`GitLab Manual CI`を起動した場合に実行 | App support and diagnostics read list/build/estimate/run for `/results/usage`; these checks provide visibility, not a readiness gate / app supportとdiagnosticsは`/results/usage`用にlist/build/estimate/runを読む。これはvisibilityでありreadiness gateではない | | Other benchmark app files, legacy Benchpark bridge, or shared scripts / その他のbenchmark appファイル、legacy Benchpark bridge、または共通script | `Shellcheck` for `.sh` changes when covered; otherwise normal GitHub review checks / 対象`.sh`変更は`Shellcheck`; それ以外は通常のGitHub review check | Run through `GitLab Manual CI` when maintainer starts it / maintainerが`GitLab Manual CI`を起動した場合に実行 | Use `code` and `system` for Benchkit validation. Use `benchpark` or `park_only` only for the legacy bridge path when needed / Benchkit検証には`code`と`system`を使う。legacy bridge pathが必要な場合だけ`benchpark`または`park_only`を使う | | GitHub workflow/action `.github/**/*` / GitHub workflow/action `.github/**/*` | `Repository Policy`; workflow-specific checks when applicable / `Repository Policy`; 必要に応じてworkflow固有check | Skipped by `.gitlab-ci.yml` rules / `.gitlab-ci.yml` rulesでskip | GitHub workflow/action changes affect API-calling or sync control logic. Review workflow semantics carefully; protected-branch sync pushes them to GitLab with `ci.skip` / GitHub workflow/action変更はAPI呼び出しやsync制御に影響する。workflowの意味を慎重にreviewする。protected-branch syncでは`ci.skip`付きでGitLabへpushされる | | `.gitlab-ci.yml` / `.gitlab-ci.yml` | Normal GitHub review checks only / 通常のGitHub review checkのみ | Run through `GitLab Manual CI` when a maintainer needs to validate GitLab pipeline behavior / GitLab pipeline挙動の検証が必要な場合にmaintainerが`GitLab Manual CI`で実行 | This file defines GitLab benchmark pipeline behavior / このファイルはGitLab benchmark pipeline挙動を定義する | @@ -318,7 +318,7 @@ Use these examples when deciding whether to split a pull request or start GitLab | `config/system_info.csv` only / `config/system_info.csv`のみ | `Result Server Tests` should verify public site config consistency / 公開site config整合性を`Result Server Tests`で確認 | No benchmark run because this file is portal display metadata / portal表示metadataなのでbenchmark不要 | | `config/system.csv` or `config/queue.csv` for a public system / 公開system向けの`config/system.csv`または`config/queue.csv` | `Result Server Tests` should run the site config preflight / `Result Server Tests`でsite config preflightを実行 | Start `GitLab Manual CI` too when benchmark execution behavior needs validation / benchmark実行挙動の検証が必要なら`GitLab Manual CI`も起動 | | `scripts/bk_functions.sh`, `scripts/result.sh`, or `scripts/result_server/**` only / `scripts/bk_functions.sh`、`scripts/result.sh`、または`scripts/result_server/**`のみ | `Result Server Tests` should run; `Shellcheck` should run for `.sh` changes / `Result Server Tests`が動く; `.sh`変更では`Shellcheck`が動く | Protected-branch sync uses `ci.skip`; direct/manual GitLab pipelines run because `.gitlab-ci.yml` treats `scripts/**/*` as benchmark-affecting / protected branch syncは`ci.skip`を使う。直接/手動GitLab pipelineでは`.gitlab-ci.yml`が`scripts/**/*`をbenchmark影響ありとして扱うため実行される | -| `programs/**/build.sh`, `programs/**/estimate.sh`, `programs/**/run.sh`, or `programs/**/list.csv` / `programs/**/build.sh`、`programs/**/estimate.sh`、`programs/**/run.sh`、または`programs/**/list.csv` | `Result Server Tests` should run for app-support/diagnostics visibility; `Shellcheck` should run for `.sh` changes / app-support/diagnostics visibility向けに`Result Server Tests`が動く; `.sh`変更では`Shellcheck`が動く | Start `GitLab Manual CI` when benchmark validation is needed, preferably with explicit `code` and `system` filters / benchmark検証が必要なら`code`と`system`を明示して`GitLab Manual CI`を起動 | +| `programs/**/build.sh`, `programs/**/estimate.sh`, `programs/**/run.sh`, or `programs/**/list.csv` / `programs/**/build.sh`、`programs/**/estimate.sh`、`programs/**/run.sh`、または`programs/**/list.csv` | `Result Server Tests` should run for app-support/diagnostics visibility; `Shellcheck` should run for `.sh` changes, including warning-level checks for changed app scripts on PRs / app-support/diagnostics visibility向けに`Result Server Tests`が動く; `.sh`変更では`Shellcheck`が動き、PRのapp script変更はwarning級でも確認する | Start `GitLab Manual CI` when benchmark validation is needed, preferably with explicit `code` and `system` filters / benchmark検証が必要なら`code`と`system`を明示して`GitLab Manual CI`を起動 | | `scripts/job_functions.sh` or `scripts/test_submit.sh` / `scripts/job_functions.sh`または`scripts/test_submit.sh` | `Shellcheck` should run for `.sh` changes / `.sh`変更では`Shellcheck`が動く | Start `GitLab Manual CI` when scheduler generation or submission behavior needs validation / scheduler生成や投入挙動の検証が必要なら`GitLab Manual CI`を起動 | | `benchpark-bridge/scripts/**/*.sh` / `benchpark-bridge/scripts/**/*.sh` | `Shellcheck` should run / `Shellcheck`が動く | Start `GitLab Manual CI` with `benchpark=true` or `park_only=true` only when the legacy bridge behavior needs validation / legacy bridge挙動の検証が必要な場合だけ`benchpark=true`または`park_only=true`で`GitLab Manual CI`を起動 | | `.github/workflows/sync-to-gitlab.yml` or `.github/actions/prepare-gitlab-repo/action.yml` / `.github/workflows/sync-to-gitlab.yml`または`.github/actions/prepare-gitlab-repo/action.yml` | `Repository Policy` and GitHub Actions-side validation / `Repository Policy`とGitHub Actions側の確認 | Skipped by `.gitlab-ci.yml` rules when changed alone; protected-branch sync pushes it with `ci.skip` / 単独変更なら`.gitlab-ci.yml` rulesでskip。protected-branch syncでは`ci.skip`付きでpushされる | @@ -342,3 +342,11 @@ To avoid unnecessary heavy CI runs: pull request、issue、commit message、CI設定には、secretやprivate credentialを書かないでください。 Do not include secrets or private credentials in pull requests, issues, commit messages, or CI configuration. + +外部AI toolやprivate toolingのsession URLを、commit message、pull request、issue、CI logに含めないでください。commit attributionが必要な場合は、公開してよい一般的な署名や `Co-authored-by` trailer に留めます。 + +Do not include external AI tool or private tooling session URLs in commit messages, pull requests, issues, or CI logs. If attribution is needed, use public-safe signing or a `Co-authored-by` trailer instead. + +このcheckは新規に入るcommitを対象とし、既存履歴を遡及的に書き換えるためのpolicyではありません。 + +This check applies to newly introduced commits and is not a retroactive history-rewrite policy. diff --git a/scripts/tests/check_commit_messages.py b/scripts/tests/check_commit_messages.py new file mode 100644 index 0000000..694ad5f --- /dev/null +++ b/scripts/tests/check_commit_messages.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""Reject commit messages that expose external AI tool session metadata.""" + +from __future__ import annotations + +import json +import os +import re +import subprocess +import sys +from pathlib import Path + + +ZERO_SHA = "0" * 40 +DISALLOWED_PATTERNS = ( + (re.compile(r"(?im)^Claude-Session\s*:"), "Claude-Session trailer"), + (re.compile(r"(?i)\b(?:https?://)?claude\.ai/code/session[^\s]*"), "claude.ai session URL"), +) + + +def run_git(*args: str) -> str: + return subprocess.check_output(["git", *args], text=True).strip() + + +def git_fetch(*refspecs: str) -> None: + subprocess.run(["git", "fetch", "--no-tags", "origin", *refspecs], check=True) + + +def event_payload() -> dict: + event_path = os.environ.get("GITHUB_EVENT_PATH") + if not event_path: + return {} + with Path(event_path).open(encoding="utf-8") as handle: + return json.load(handle) + + +def changed_commits() -> list[str]: + event_name = os.environ.get("GITHUB_EVENT_NAME") + event = event_payload() + + if event_name == "pull_request": + pull_request = event.get("pull_request") or {} + number = pull_request.get("number") + base = pull_request.get("base") or {} + base_ref = base.get("ref") + base_sha = base.get("sha") + if not number or not base_ref or not base_sha: + raise RuntimeError("pull_request.number, base.ref, or base.sha is missing") + + head_ref = f"refs/remotes/origin/pr/{number}/head" + git_fetch( + f"+refs/heads/{base_ref}:refs/remotes/origin/{base_ref}", + f"+refs/pull/{number}/head:{head_ref}", + ) + revspec = f"{base_sha}..{head_ref}" + return run_git("rev-list", "--reverse", revspec).splitlines() + + if event_name == "push": + before = event.get("before") + after = event.get("after") + if not after or after == ZERO_SHA: + return [] + if not before or before == ZERO_SHA: + return [after] + return run_git("rev-list", "--reverse", f"{before}..{after}").splitlines() + + return [run_git("rev-parse", "HEAD")] + + +def disallowed_metadata(message: str) -> list[str]: + labels: list[str] = [] + for pattern, label in DISALLOWED_PATTERNS: + if pattern.search(message): + labels.append(label) + return labels + + +def main() -> int: + try: + commits = changed_commits() + except (OSError, subprocess.CalledProcessError, RuntimeError) as exc: + print(f"Could not determine commits to check: {exc}", file=sys.stderr) + return 1 + + violations: list[tuple[str, str]] = [] + for commit in commits: + message = run_git("log", "-1", "--format=%B", commit) + for label in disallowed_metadata(message): + violations.append((commit, label)) + + if violations: + print("Disallowed commit metadata detected:", file=sys.stderr) + for commit, label in violations: + print(f" - {commit[:12]}: {label}", file=sys.stderr) + print( + "Remove external AI tool session URLs or private tooling session " + "identifiers from commit messages before merging.", + file=sys.stderr, + ) + return 1 + + print(f"Checked {len(commits)} commit message(s)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())