From 12640b39b2048af088767e819dbaaa2bb6e53dee Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Wed, 26 Aug 2026 11:28:20 +1200 Subject: [PATCH 01/19] feat: move the ai-failure-notifier script here from operator The script is 1,461 lines and its tests another 1,231, and canonical/operator is only the first repository meant to run it. Copying that into each adopting repo means fixing every bug as many times as there are repos, so it moves here and each repo keeps only the workflow YAML that differs. Both files are verbatim from canonical/operator#2663 at f15bcf1e, with exactly one line changed: the test's `import ai_failure_notifier as afn` becomes `from charm_tech_code import ai_failure_notifier as afn`. Nothing else in either file is touched, so the 70 tests passing here are the same 70 assertions that passed there. Splitting the script into modules is the next commit, kept separate so this one stays a move and that one stays a refactor. The package has no runtime dependencies, which is what makes `uvx --from git+` viable as the distribution mechanism with no release process to run. --- .gitignore | 4 + README.md | 40 + pyproject.toml | 38 + src/charm_tech_code/__init__.py | 20 + src/charm_tech_code/ai_failure_notifier.py | 1461 ++++++++++++++++++++ tests/test_ai_failure_notifier.py | 1231 +++++++++++++++++ 6 files changed, 2794 insertions(+) create mode 100644 README.md create mode 100644 pyproject.toml create mode 100644 src/charm_tech_code/__init__.py create mode 100644 src/charm_tech_code/ai_failure_notifier.py create mode 100644 tests/test_ai_failure_notifier.py diff --git a/.gitignore b/.gitignore index 83972fa..2657f32 100644 --- a/.gitignore +++ b/.gitignore @@ -216,3 +216,7 @@ __marimo__/ # Streamlit .streamlit/secrets.toml + +# uv +.venv/ +uv.lock diff --git a/README.md b/README.md new file mode 100644 index 0000000..8b01672 --- /dev/null +++ b/README.md @@ -0,0 +1,40 @@ +# charm-tech-code + +Shared tooling for the Charm Tech repositories (`operator`, `charmlibs`, +`jubilant`, `pebble`, `concierge`, and the rest of the estate). + +Code here is consumed by workflow YAML in the repository that runs it, pinned +by commit SHA: + +```yaml +run: uvx --from git+https://github.com/canonical/charm-tech-code@<40-char-sha> ai-failure-notifier +``` + +The point is that the code lives in one place. A tool used by eleven +repositories should be fixed once, not eleven times, and the workflow YAML +that differs per repository stays in that repository. + +There is no release process and nothing is published. The SHA in the `uvx` +line is the version, which is the same trust decision every pinned +`uses: actions/checkout@` line in those repositories already makes. + +## Tools + +### `ai-failure-notifier` + +Enriches the placeholder issue that a scheduled workflow's failure notifier +opens. It reads the failing run's job logs, builds a deterministic failure +signature, searches for issues that look like the same failure, and either +comments on one or rewrites the placeholder with a real title, body and +labels. Without an API key it falls back to a plain notification, so the +notifier's "always works" property is not affected by this being unavailable. + +See `canonical/operator`'s `.github/workflows/ai-failure-enrich.yaml` for the +calling side. + +## Developing + +```shell +uv sync +uv run pytest +``` diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..e74050a --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,38 @@ +[project] +name = "charm-tech-code" +version = "0.1.0" +description = "Shared tooling for the Charm Tech repositories." +readme = "README.md" +license = "Apache-2.0" +requires-python = ">=3.10" +dependencies = [] + +[project.scripts] +ai-failure-notifier = "charm_tech_code.ai_failure_notifier:main" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/charm_tech_code"] + +[dependency-groups] +dev = ["pytest"] + +[tool.pytest.ini_options] +testpaths = ["tests"] + +[tool.ruff] +line-length = 99 +src = ["src", "tests"] + +[tool.ruff.format] +quote-style = "single" + +[tool.ruff.lint] +select = ["E", "F", "I", "UP", "B"] + +[tool.pyright] +include = ["src", "tests"] +pythonVersion = "3.10" diff --git a/src/charm_tech_code/__init__.py b/src/charm_tech_code/__init__.py new file mode 100644 index 0000000..9e42e0c --- /dev/null +++ b/src/charm_tech_code/__init__.py @@ -0,0 +1,20 @@ +# Copyright 2026 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shared tooling for the Charm Tech repositories. + +Each tool here is consumed by workflow YAML in the repo that uses it, pinned +by commit SHA, so that the code lives in one place rather than being copied +into every repository that runs it. +""" diff --git a/src/charm_tech_code/ai_failure_notifier.py b/src/charm_tech_code/ai_failure_notifier.py new file mode 100644 index 0000000..18761af --- /dev/null +++ b/src/charm_tech_code/ai_failure_notifier.py @@ -0,0 +1,1461 @@ +#!/usr/bin/env python3 +# +# Copyright 2026 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""ai-failure-notifications enrichment step. + +Invoked by `.github/workflows/ai-failure-enrich.yaml` after +`.github/workflows/notify-scheduled-failure.yaml` (the notifier) has already +created or commented on a placeholder issue for a failed scheduled workflow +run. This script: + +1. Finds the placeholder the notifier just touched (or, on a same-run + re-fire, the issue an earlier run of this script already enriched). +2. Fetches and parses the failing job logs into a deterministic failure + signature. +3. Builds a small candidate-issue pool (coarse title/body search). +4. Asks an LLM (via OpenRouter) to decide comment-vs-new and draft the text, + validates the response against the envelope schema, and applies it via + `gh`. +5. Falls back to a plain, generic issue/comment (still marker-stamped) if + OpenRouter is unreachable, misconfigured, or returns invalid JSON. + +The functions above the `--- I/O ---` marker are pure and unit-tested in +`scripts/test/test_ai_failure_notifier.py`. Everything below it talks to `gh` +or OpenRouter and is exercised only by mocking in tests. +""" + +from __future__ import annotations + +import dataclasses +import datetime +import hashlib +import json +import os +import re +import subprocess +import sys +import urllib.request +from typing import Any, Literal + +MARKER_PREFIX = 'ai-failure-notifications' +DEFAULT_MODEL = 'deepseek/deepseek-chat' # DeepSeek V3 on OpenRouter. +CLOSED_CANDIDATE_WINDOW_DAYS = 14 +MAX_CANDIDATES = 3 +# How many recently-updated issues to scan for the notifier's marker. The +# artefact we are looking for was touched minutes ago, so this only has to +# cover issue churn in that window; 50 is far more than `operator` sees. +RECENT_ISSUE_SCAN = 50 + +# Colour escapes, which Actions logs are full of. Two alternatives, because +# the logs contain both the real thing and a mangled form where the ESC byte +# has already been stripped, leaving a bare "[32m". +ANSI = re.compile( + r""" + \x1b\[ [0-9;]* [A-Za-z] # a full escape: ESC [ params letter + | + \[ \d+ (?:;\d+)* m # ESC already stripped: [32m, [1;33m + """, + re.VERBOSE, +) + +# The timestamp Actions prefixes to every log line, for example +# "2026-07-21T16:17:04.8204062Z ". Stripped before anything else is matched. +TS = re.compile( + r""" + ^\d{4}-\d{2}-\d{2} # date: 2026-07-21 + T\d{2}:\d{2}:\d{2} # time: T16:17:04 + \.\d+Z[ ] # fractional seconds, zone, one trailing space + """, + re.VERBOSE, +) + +# Actions' own annotation for a failing step. +ERROR_MARKER = re.compile(r'##\[error\]') + +# The runner opens every step with "##[group]Run