diff --git a/src/microbots/auto_memory/training/__init__.py b/src/microbots/auto_memory/training/__init__.py new file mode 100644 index 0000000..ad0a6ab --- /dev/null +++ b/src/microbots/auto_memory/training/__init__.py @@ -0,0 +1,42 @@ +"""Training subpackage for auto_memory. + +Houses the learning phase that is fully independent of the eval loop in +:mod:`microbots.auto_memory`. The only shared surface between the two +phases is (eventually) :class:`~microbots.auto_memory.repo_memory.RepoMemory`. + +The framework is domain-agnostic: an ``AGENTS.md`` file defines *what* the +agent should learn, a source directory is mounted into the sandbox as the +material to learn *from*, and a ``memory_dir`` collects the resulting +notes. The default ``AGENTS.md`` shipped in this package targets +repository learning, but the framework itself makes no such assumption \u2014 +the source can be a source-code repo, a docs tree, a dataset, an example +gallery, or anything else that fits in a directory. + +Public API + (an existing local path or a git repo to clone). + notes in a shared ``/memories/`` tree. + points. +""" + +from microbots.auto_memory.training.config import TrainingConfig +from microbots.auto_memory.training.training_source import TrainingSource +from microbots.auto_memory.training.orchestrator import ( + TrainingIterationRecord, + TrainingOrchestrator, + TrainingSummary, +) +from microbots.auto_memory.training.cli import ( + run_training, + run_training_from_yaml, +) + +__all__ = [ + "TrainingConfig", + "TrainingSource", + "TrainingOrchestrator", + "TrainingSummary", + "TrainingIterationRecord", + "run_training", + "run_training_from_yaml", +] + diff --git a/src/microbots/auto_memory/training/__main__.py b/src/microbots/auto_memory/training/__main__.py new file mode 100644 index 0000000..599a898 --- /dev/null +++ b/src/microbots/auto_memory/training/__main__.py @@ -0,0 +1,11 @@ +"""Enable ``python -m microbots.auto_memory.training``.""" + +from __future__ import annotations + +import sys + +from microbots.auto_memory.training.cli import main + + +if __name__ == "__main__": # pragma: no cover + sys.exit(main()) diff --git a/src/microbots/auto_memory/training/cli.py b/src/microbots/auto_memory/training/cli.py new file mode 100644 index 0000000..4e189d9 --- /dev/null +++ b/src/microbots/auto_memory/training/cli.py @@ -0,0 +1,385 @@ +"""Programmatic and CLI entry points for the training framework. + +Programmatic: + + from microbots.auto_memory.training import run_training + + # Local directory as source (legacy shape still works): + summary = run_training( + source_path="/path/to/source", + memory_dir="/path/to/memory", + model="azure-openai/gpt-4o", + ) + + # Git repo as source: + from microbots.auto_memory.training import TrainingSource + summary = run_training( + source=TrainingSource(type="git", url="https://github.com/foo/bar.git", + ref="main"), + memory_dir="/path/to/memory", + model="azure-openai/gpt-4o", + ) + +CLI: + + python -m microbots.auto_memory.training \ + --source /path/to/source \ + --memory /path/to/memory \ + --model azure-openai/gpt-5 \ + [--source-git-url https://github.com/foo/bar.git] \ + [--source-ref main] \ + [--source-cache-dir /path/to/clone] \ + [--agents-md /path/to/AGENTS.md] \ + [--iterations 3] \ + [--config path/to/training.yaml] \ + [--workdir path/to/workdir] \ + [--reset-memory] + +The framework is domain-agnostic: the source can be any directory the +agent should learn from (a source-code repo, a docs tree, a dataset, an +example gallery, …) or a git repository URL that is cloned before the +run. What the agent actually does with it is defined by the +``AGENTS.md`` file. +""" + +from __future__ import annotations + +import argparse +import logging +import sys +from datetime import datetime, timezone +from pathlib import Path + +from microbots.auto_memory.errors import ConfigError +from microbots.auto_memory.training.config import TrainingConfig +from microbots.auto_memory.training.orchestrator import ( + TrainingOrchestrator, + TrainingSummary, +) +from microbots.auto_memory.training.training_source import TrainingSource + + +# --------------------------------------------------------------------------- +# Programmatic API +# --------------------------------------------------------------------------- + + +def run_training( + *, + source: TrainingSource | dict | None = None, + source_path: str | Path | None = None, + memory_dir: str | Path, + model: str, + agents_md_path: str | Path | None = None, + iterations: int = 3, + per_iteration_timeout: int = 900, + total_timeout_min: int = 0, + max_bot_steps: int = 40, + reset_memory: bool = False, + workdir: str | Path | None = None, +) -> TrainingSummary: + """Build a :class:`TrainingConfig`, wire the orchestrator, and run it. + + Provide exactly one of ``source`` or ``source_path``. ``source`` is the + new nested form (:class:`TrainingSource` or an equivalent mapping) and lets + you point at a git repo; ``source_path`` remains as a shortcut for a + local directory (or a bare git URL, which is auto-detected). + + Other parameters mirror the fields of :class:`TrainingConfig`. + ``workdir`` is where ``training_meta.json`` and ``training_run.jsonl`` + are written; it defaults to + ``/.training-run-/``. + + Parameters + ---------- + source : TrainingSource | dict | None, optional + Structured source specification or equivalent mapping. + source_path : str | Path | None, optional + Legacy local directory or git URL. Mutually exclusive with ``source``. + memory_dir : str | Path + Host directory backing the agent's persistent memory. + model : str + Model identifier in ``/`` form. + agents_md_path : str | Path | None, optional + Custom training instructions file. + iterations : int, optional + Number of training iterations to run. + per_iteration_timeout : int, optional + Wall-clock limit for each iteration, in seconds. + total_timeout_min : int, optional + Wall-clock limit for the full run, in minutes. Zero disables it. + max_bot_steps : int, optional + Maximum internal bot steps per iteration. + reset_memory : bool, optional + Whether to clear existing memory before training. + workdir : str | Path | None, optional + Directory for training metadata and logs. + + Returns + ------- + TrainingSummary + Summary of the completed run. + """ + if source is not None and source_path is not None: + raise ConfigError( + "Pass either 'source' or 'source_path', not both." + ) + if source is None and source_path is None: + raise ConfigError("One of 'source' or 'source_path' is required.") + + if source is not None: + if isinstance(source, TrainingSource): + source_spec = source + elif isinstance(source, dict): + source_spec = TrainingSource.from_mapping(source, base_dir=Path.cwd()) + else: + raise ConfigError( + "'source' must be a TrainingSource or mapping, got " + f"{type(source).__name__}" + ) + else: + source_spec = TrainingSource.from_legacy_source_path( + source_path, base_dir=Path.cwd() # type: ignore[arg-type] + ) + + cfg_kwargs: dict = { + "source": source_spec, + "memory_dir": Path(memory_dir).resolve(), + "model": model, + "iterations": iterations, + "per_iteration_timeout": per_iteration_timeout, + "total_timeout_min": total_timeout_min, + "max_bot_steps": max_bot_steps, + "reset_memory": reset_memory, + } + if agents_md_path is not None: + cfg_kwargs["agents_md_path"] = Path(agents_md_path).resolve() + + config = TrainingConfig(**cfg_kwargs) + config.validate() + + if workdir is None: + stamp = datetime.now(tz=timezone.utc).strftime("%Y%m%d-%H%M%S") + workdir = config.memory_dir.parent / f".training-run-{stamp}" + + orchestrator = TrainingOrchestrator(config=config, workdir=Path(workdir)) + return orchestrator.run() + + +def run_training_from_yaml( + yaml_path: str | Path, + *, + workdir: str | Path | None = None, +) -> TrainingSummary: + """Load a training YAML config and execute the orchestrator. + + Parameters + ---------- + yaml_path : str | Path + Path to the YAML file describing the training run. + workdir : str | Path | None, optional + Where to persist meta/log files. Defaults to + ``/.training-run-/``. + + Returns + ------- + TrainingSummary + Summary of the completed run. + """ + config = TrainingConfig.load_from_yaml(yaml_path) + + if workdir is None: + stamp = datetime.now(tz=timezone.utc).strftime("%Y%m%d-%H%M%S") + workdir = config.memory_dir.parent / f".training-run-{stamp}" + + orchestrator = TrainingOrchestrator(config=config, workdir=Path(workdir)) + return orchestrator.run() + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def _build_parser() -> argparse.ArgumentParser: + """Build the command-line argument parser. + + Returns + ------- + argparse.ArgumentParser + Parser configured for the training command. + """ + p = argparse.ArgumentParser( + prog="microbots.auto_memory.training", + description=( + "Run a training loop: point an agent at a source directory plus " + "an AGENTS.md prompt, and let it populate a persistent " + "/memories/ tree. The source can be any directory (source-code " + "repo, docs tree, dataset, example gallery, …) — what the " + "agent learns is defined by AGENTS.md." + ), + ) + p.add_argument( + "--config", + type=Path, + help=( + "Path to a training YAML config. When provided, most other " + "flags are ignored (only --workdir and --verbose still apply)." + ), + ) + p.add_argument( + "--source", + type=Path, + help=( + "Local directory the agent should learn from. Mutually exclusive " + "with --source-git-url." + ), + ) + p.add_argument( + "--source-git-url", + type=str, + default=None, + help=( + "Git remote URL to clone as the source. When set, --source is " + "used (if given) as the clone destination; otherwise the loop " + "clones into /source/." + ), + ) + p.add_argument( + "--source-ref", + type=str, + default=None, + help="Branch, tag, or commit to check out for --source-git-url.", + ) + p.add_argument( + "--source-cache-dir", + type=Path, + default=None, + help=( + "Explicit clone destination for --source-git-url. Set this to " + "reuse a checkout across runs." + ), + ) + p.add_argument( + "--memory", + type=Path, + help="Host directory backing the agent's /memories/ tree.", + ) + p.add_argument("--model", type=str, help="Model id, e.g. azure-openai/gpt-4o.") + p.add_argument( + "--agents-md", + type=Path, + default=None, + help="Custom AGENTS.md file (defaults to the one shipped in this package).", + ) + p.add_argument("--iterations", type=int, default=3) + p.add_argument("--per-iteration-timeout", type=int, default=900) + p.add_argument("--total-timeout-min", type=int, default=0) + p.add_argument("--max-bot-steps", type=int, default=40) + p.add_argument( + "--reset-memory", + action="store_true", + help="Wipe the memory directory before starting.", + ) + p.add_argument( + "--workdir", + type=Path, + default=None, + help="Where to write training_meta.json and training_run.jsonl.", + ) + p.add_argument("-v", "--verbose", action="store_true") + return p + + +def main(argv: list[str] | None = None) -> int: + """Run the training command-line interface. + + Parameters + ---------- + argv : list[str] | None, optional + Arguments to parse. Uses :data:`sys.argv` when omitted. + + Returns + ------- + int + Process exit code, with zero indicating completion. + """ + args = _build_parser().parse_args(argv) + + logging.basicConfig( + level=logging.DEBUG if args.verbose else logging.INFO, + format="%(asctime)s %(levelname)s %(name)s: %(message)s", + ) + + try: + if args.config is not None: + summary = run_training_from_yaml(args.config, workdir=args.workdir) + else: + if args.memory is None or args.model is None: + missing = [ + name + for name, val in ( + ("--memory", args.memory), + ("--model", args.model), + ) + if val is None + ] + print( + f"error: missing required flag(s): {', '.join(missing)} " + "(or pass --config).", + file=sys.stderr, + ) + return 2 + + # Build the source: either a local path (--source) or a git URL + # (--source-git-url). --source alone → local; --source-git-url + # → git, optionally with --source as the clone destination. + if args.source_git_url: + source_kwarg: dict = { + "source": TrainingSource( + type="git", + url=args.source_git_url, + ref=args.source_ref, + path=args.source.resolve() if args.source else None, + cache_dir=( + args.source_cache_dir.resolve() + if args.source_cache_dir + else None + ), + ) + } + elif args.source is not None: + source_kwarg = {"source_path": args.source} + else: + print( + "error: missing required flag(s): --source or " + "--source-git-url (or pass --config).", + file=sys.stderr, + ) + return 2 + + summary = run_training( + **source_kwarg, + memory_dir=args.memory, + model=args.model, + agents_md_path=args.agents_md, + iterations=args.iterations, + per_iteration_timeout=args.per_iteration_timeout, + total_timeout_min=args.total_timeout_min, + max_bot_steps=args.max_bot_steps, + reset_memory=args.reset_memory, + workdir=args.workdir, + ) + except ConfigError as exc: + print(f"config error: {exc}", file=sys.stderr) + return 2 + + print( + f"training {summary.final_status}: " + f"iterations={summary.iterations_run} " + f"elapsed={summary.elapsed_s:.1f}s " + f"memory_dir={summary.memory_dir}" + ) + if summary.error_message: + print(f"last error: {summary.error_message}", file=sys.stderr) + return 0 diff --git a/src/microbots/auto_memory/training/config.py b/src/microbots/auto_memory/training/config.py new file mode 100644 index 0000000..87ded87 --- /dev/null +++ b/src/microbots/auto_memory/training/config.py @@ -0,0 +1,247 @@ +"""Configuration for a training run. + +Attributes +---------- +agents_md_path + Path to the ``AGENTS.md`` file used as the base prompt for every + iteration. +source + :class:`~microbots.auto_memory.training.training_source.TrainingSource` describing + where the directory the agent should learn from comes from. It can be + an existing local directory (``type: path``) or a git repository that + is cloned before the run starts (``type: git``). The resolved local + directory is mounted into the bot's sandbox as the working directory. + + In YAML the shape is a nested mapping:: + + source: + type: path + path: /some/dir + + # or + + source: + type: git + url: https://github.com/foo/bar.git + ref: main # optional + cache_dir: /some/dir # optional; defaults to /source + + Legacy top-level ``source_path: `` is still accepted and normalised + into ``TrainingSource(type='path', path=...)``. If the legacy value looks + like a URL it is treated as ``type='git'``. +memory_dir + Host-side directory that backs the agent's ``/memories/`` tree. Notes + persist here across iterations and across runs (see ``reset_memory``). +model + Model identifier forwarded to the runner (e.g. ``"azure-openai/gpt-4o"``). +iterations + Number of training iterations to execute back-to-back. Memory persists + across iterations, so each iteration builds on the previous one. +per_iteration_timeout + Wall-clock cap for one iteration, in seconds. +total_timeout_min + Wall-clock cap for the whole run, in minutes. ``0`` disables it. +max_bot_steps + ``max_iterations`` forwarded to the underlying bot (its internal + tool-loop cap, not the training loop's iteration count). +reset_memory + When ``True``, wipe ``memory_dir`` before the first iteration. When + ``False`` (default), existing notes are preserved. +""" + +from __future__ import annotations + +import yaml +from dataclasses import dataclass +from logging import getLogger +from pathlib import Path + +from microbots.auto_memory.errors import ConfigError +from microbots.auto_memory.training.training_source import TrainingSource +from microbots.constants import ModelProvider + +logger = getLogger(__name__) + +_DEFAULT_AGENTS_MD = Path(__file__).parent / "training_instructions.md" + + +@dataclass +class TrainingConfig: + """All configuration for one training run.""" + + source: TrainingSource + memory_dir: Path + model: str + agents_md_path: Path = _DEFAULT_AGENTS_MD + iterations: int = 3 + per_iteration_timeout: int = 900 + total_timeout_min: int = 0 + max_bot_steps: int = 40 + reset_memory: bool = False + + # ------------------------------------------------------------------ + # Back-compat convenience + + @property + def source_path(self) -> Path | None: + """Best-effort local source path. + + For ``type='path'`` this is the configured directory. For + ``type='git'`` this is ``None`` until :meth:`TrainingSource.materialize` + has been called (typically by the training loop at run start). + + Returns + ------- + Path | None + Configured or materialized local source path, when available. + """ + return self.source.path + + # ------------------------------------------------------------------ + + @classmethod + def load_from_yaml(cls, path: str | Path) -> "TrainingConfig": + """Parse a training YAML file into a :class:`TrainingConfig`. + + The YAML file must provide ``memory_dir``, ``model``, and either a + nested ``source:`` mapping or a legacy ``source_path`` string. All + other keys map 1:1 to the dataclass fields. Filesystem paths + (``memory_dir``, ``agents_md_path``, and any paths inside + ``source``) are resolved relative to the YAML file's directory when + written as relative paths. + + Parameters + ---------- + path : str | Path + Filesystem path to the YAML configuration. + + Returns + ------- + TrainingConfig + Fully validated configuration. + + Raises + ------ + ConfigError + If the file is missing, unparseable, or has invalid fields. + """ + yaml_path = Path(path) + if not yaml_path.exists(): + raise ConfigError(f"Training config file not found: {path}") + + try: + with yaml_path.open() as fh: + data = yaml.safe_load(fh) + except yaml.YAMLError as exc: + raise ConfigError(f"Failed to parse YAML from {path}: {exc}") from exc + + if not isinstance(data, dict): + raise ConfigError(f"Expected a YAML mapping at the top of {path}") + + for required in ("memory_dir", "model"): + if required not in data: + raise ConfigError(f"Missing required field '{required}' in {path}") + + if "source" not in data and "source_path" not in data: + raise ConfigError( + f"Missing source: provide either 'source:' (mapping) or " + f"legacy 'source_path:' in {path}" + ) + + base = yaml_path.resolve().parent + + def _resolve(p: str | Path) -> Path: + """Resolve a configured path relative to the YAML directory. + + Parameters + ---------- + p : str | Path + Configured filesystem path. + + Returns + ------- + Path + Absolute path, or the unchanged absolute input path. + """ + p = Path(p) + return p if p.is_absolute() else (base / p).resolve() + + if "source" in data: + source = TrainingSource.from_mapping(data["source"], base_dir=base) + else: + source = TrainingSource.from_legacy_source_path( + data["source_path"], base_dir=base + ) + + agents_md_raw = data.get("agents_md_path") + agents_md = _resolve(agents_md_raw) if agents_md_raw else _DEFAULT_AGENTS_MD + + config = cls( + source=source, + memory_dir=_resolve(data["memory_dir"]), + model=str(data["model"]), + agents_md_path=agents_md, + iterations=int(data.get("iterations", 3)), + per_iteration_timeout=int(data.get("per_iteration_timeout", 900)), + total_timeout_min=int(data.get("total_timeout_min", 0)), + max_bot_steps=int(data.get("max_bot_steps", 40)), + reset_memory=bool(data.get("reset_memory", False)), + ) + config.validate() + return config + + # ------------------------------------------------------------------ + + def validate(self) -> None: + """Validate every field. Raises :class:`ConfigError` on any issue.""" + self.source.validate() + + if not self.agents_md_path.exists() or not self.agents_md_path.is_file(): + raise ConfigError( + f"'agents_md_path' must point to an existing file, got " + f"{self.agents_md_path}" + ) + + if not self.model: + raise ConfigError("'model' must not be empty") + if self.model.count("/") != 1: + raise ConfigError( + f"'model' must be in the form '/', got '{self.model}'" + ) + provider = self.model.split("/", 1)[0] + supported = [e.value for e in ModelProvider] + if provider not in supported: + raise ConfigError( + f"'model' has unsupported provider '{provider}'; " + f"expected one of {supported}" + ) + + if self.iterations < 1: + raise ConfigError(f"'iterations' must be >= 1, got {self.iterations}") + + if self.per_iteration_timeout < 1: + raise ConfigError( + f"'per_iteration_timeout' must be >= 1, got {self.per_iteration_timeout}" + ) + + if self.total_timeout_min < 0: + raise ConfigError( + f"'total_timeout_min' must be >= 0, got {self.total_timeout_min}" + ) + + if self.max_bot_steps < 1: + raise ConfigError( + f"'max_bot_steps' must be >= 1, got {self.max_bot_steps}" + ) + + # ------------------------------------------------------------------ + + def read_agents_md(self) -> str: + """Read the configured ``AGENTS.md`` file. + + Returns + ------- + str + Contents of the training instructions file. + """ + return self.agents_md_path.read_text(encoding="utf-8") diff --git a/src/microbots/auto_memory/training/orchestrator.py b/src/microbots/auto_memory/training/orchestrator.py new file mode 100644 index 0000000..ce27987 --- /dev/null +++ b/src/microbots/auto_memory/training/orchestrator.py @@ -0,0 +1,302 @@ +"""Training orchestrator for learning agents. + +Drives a sequence of :class:`~microbots.auto_memory.training.runner.LearningRunner` +iterations against a target source directory, accumulating notes in the +shared ``memory_dir``. Each iteration receives the same base ``AGENTS.md`` +prompt plus a small header identifying the iteration index, the source +path, and the memory root - the agent uses the ``memory`` tool to read +prior notes and extend them. + +The framework is deliberately domain-agnostic: what the agent is learning +is defined by the ``AGENTS.md`` file, and the source directory can hold +anything (a source-code repo, a docs tree, a dataset, an example gallery, +...). + +There is deliberately **no** callback / feedback loop here (unlike the +eval loop in :mod:`microbots.auto_memory.orchestrator`). The only +persistent state that matters is the ``/memories/`` tree. +""" + +from __future__ import annotations + +import json +import shutil +import time +from dataclasses import asdict, dataclass, field +from datetime import datetime, timezone +from logging import getLogger +from pathlib import Path + +from microbots.auto_memory.errors import ConfigError +from microbots.auto_memory.training.config import TrainingConfig +from microbots.auto_memory.training.runner import ( + LearningRunner, + TrainingIterationResult, +) + +logger = getLogger(__name__) + + +@dataclass +class TrainingIterationRecord: + """Per-iteration record persisted to ``training_run.jsonl``.""" + + idx: int + status: str + elapsed_s: float + error: str | None = None + + +@dataclass +class TrainingSummary: + """Summary of one completed training run.""" + + final_status: str # "completed" | "timeout" | "error" + iterations_run: int + iteration_records: list[TrainingIterationRecord] = field(default_factory=list) + elapsed_s: float = 0.0 + memory_dir: Path | None = None + error_message: str | None = None + + +_ITER_HEADER = ( + "\n\n---\n" + "# Runtime Context\n" + "- Iteration index (zero-based): {idx}\n" + "- Total iterations planned: {total}\n" + "- Source directory (mounted in sandbox): {source}\n" + "- Memory root: /memories/\n" +) + + +class TrainingOrchestrator: + """Repeatedly invoke the runner, accumulating notes in ``memory_dir``. + + Parameters + ---------- + config : TrainingConfig + Fully validated training configuration. + workdir : Path + Directory that receives ``training_meta.json`` and + ``training_run.jsonl``. Created if missing. + """ + + def __init__(self, config: TrainingConfig, workdir: Path) -> None: + """Initialize a training orchestrator. + + Parameters + ---------- + config : TrainingConfig + Fully validated training configuration. + workdir : Path + Directory for training metadata and logs. + """ + self._config = config + self._workdir = workdir + self._meta_path = workdir / "training_meta.json" + self._log_path = workdir / "training_run.jsonl" + + def run(self) -> TrainingSummary: + """Execute all configured training iterations. + + Returns + ------- + TrainingSummary + Summary of the completed, timed out, or failed run. + """ + self._prepare_workdir() + self._prepare_memory_dir() + + resolved_source = self._config.source.materialize( + default_dest=self._workdir / "source" + ) + self._write_meta(resolved_source=resolved_source) + + runner = LearningRunner( + model=self._config.model, + source_path=resolved_source, + memory_dir=self._config.memory_dir, + max_bot_steps=self._config.max_bot_steps, + ) + + agents_md = self._config.read_agents_md() + total = self._config.iterations + total_budget_s = self._config.total_timeout_min * 60 + started = time.monotonic() + records: list[TrainingIterationRecord] = [] + + for idx in range(total): + elapsed = time.monotonic() - started + if total_budget_s and elapsed >= total_budget_s: + logger.info( + "TrainingOrchestrator: total timeout reached after %.1fs (limit %ds)", + elapsed, + total_budget_s, + ) + return self._finish("timeout", records, elapsed, error=None) + + prompt = agents_md + _ITER_HEADER.format( + idx=idx, + total=total, + source=resolved_source, + ) + + logger.info( + "TrainingOrchestrator: iteration %d/%d starting", idx + 1, total + ) + iter_started = time.monotonic() + try: + result: TrainingIterationResult = runner.run( + prompt, timeout_s=self._config.per_iteration_timeout + ) + except Exception as exc: # noqa: BLE001 - surface as ERROR record + iter_elapsed = time.monotonic() - iter_started + record = TrainingIterationRecord( + idx=idx, + status="error", + elapsed_s=iter_elapsed, + error=f"{type(exc).__name__}: {exc}", + ) + records.append(record) + self._append_log(record) + logger.exception( + "TrainingOrchestrator: iteration %d raised %s", + idx, + type(exc).__name__, + ) + return self._finish( + "error", + records, + time.monotonic() - started, + error=record.error, + ) + + iter_elapsed = time.monotonic() - iter_started + record = TrainingIterationRecord( + idx=idx, + status=result.status, + elapsed_s=iter_elapsed, + error=result.error, + ) + records.append(record) + self._append_log(record) + + logger.info( + "TrainingOrchestrator: iteration %d finished status=%s in %.1fs", + idx, + result.status, + iter_elapsed, + ) + + if result.status == "timeout": + return self._finish( + "timeout", + records, + time.monotonic() - started, + error=result.error, + ) + + return self._finish( + "completed", + records, + time.monotonic() - started, + error=None, + ) + + def _prepare_workdir(self) -> None: + """Create the training work directory if it does not exist.""" + try: + self._workdir.mkdir(parents=True, exist_ok=True) + except OSError as exc: + raise ConfigError( + f"Cannot create workdir {self._workdir}: {exc}" + ) from exc + + def _prepare_memory_dir(self) -> None: + """Create, or reset and recreate, the persistent memory directory.""" + mem = self._config.memory_dir + if self._config.reset_memory and mem.exists(): + logger.info("TrainingOrchestrator: reset_memory=True - wiping %s", mem) + shutil.rmtree(mem, ignore_errors=True) + try: + mem.mkdir(parents=True, exist_ok=True) + except OSError as exc: + raise ConfigError( + f"Cannot create memory_dir {mem}: {exc}" + ) from exc + + def _write_meta(self, *, resolved_source: Path) -> None: + """Initialize run metadata and the iteration log. + + Parameters + ---------- + resolved_source : Path + Materialized local source directory used by the runner. + """ + meta = { + "started_at": datetime.now(tz=timezone.utc).isoformat(), + "source": self._config.source.to_meta(), + "source_path": str(resolved_source), + "memory_dir": str(self._config.memory_dir), + "agents_md_path": str(self._config.agents_md_path), + "model": self._config.model, + "iterations": self._config.iterations, + "per_iteration_timeout": self._config.per_iteration_timeout, + "total_timeout_min": self._config.total_timeout_min, + "max_bot_steps": self._config.max_bot_steps, + "reset_memory": self._config.reset_memory, + } + self._meta_path.write_text(json.dumps(meta, indent=2), encoding="utf-8") + self._log_path.write_text("", encoding="utf-8") + + def _append_log(self, record: TrainingIterationRecord) -> None: + """Append an iteration record to the JSON Lines log. + + Parameters + ---------- + record : TrainingIterationRecord + Completed iteration record to persist. + """ + with self._log_path.open("a", encoding="utf-8") as fh: + fh.write(json.dumps(asdict(record)) + "\n") + + def _finish( + self, + final_status: str, + records: list[TrainingIterationRecord], + elapsed_s: float, + error: str | None, + ) -> TrainingSummary: + """Build and log the final training summary. + + Parameters + ---------- + final_status : str + Overall completion status. + records : list[TrainingIterationRecord] + Iteration records accumulated during the run. + elapsed_s : float + Total elapsed wall-clock time in seconds. + error : str | None + Final error description, if any. + + Returns + ------- + TrainingSummary + Final summary for the run. + """ + summary = TrainingSummary( + final_status=final_status, + iterations_run=len(records), + iteration_records=records, + elapsed_s=elapsed_s, + memory_dir=self._config.memory_dir, + error_message=error, + ) + logger.info( + "TrainingOrchestrator: finished status=%s iterations=%d elapsed=%.1fs", + final_status, + len(records), + elapsed_s, + ) + return summary \ No newline at end of file diff --git a/src/microbots/auto_memory/training/runner.py b/src/microbots/auto_memory/training/runner.py new file mode 100644 index 0000000..8696226 --- /dev/null +++ b/src/microbots/auto_memory/training/runner.py @@ -0,0 +1,155 @@ +"""Agent runner for training iterations. + +Wraps :class:`~microbots.bot.ReadingBot.ReadingBot` so that: + +* the source directory the agent should learn from is mounted into the + sandbox as the working directory (``folder_to_mount``); and +* the agent's ``/memories/`` tree is backed by a host-side directory that + survives across iterations and runs. + +The framework does not assume the source is a source-code repository — it +can be any directory. This is the *only* module in +:mod:`microbots.auto_memory.training` that imports the concrete bot +implementation, keeping the training loop itself decoupled. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from logging import getLogger +from pathlib import Path + +from microbots.bot.ReadingBot import ReadingBot +from microbots.MicroBot import BotRunResult +from microbots.tools.tool_definitions.memory_tool import MemoryTool + +logger = getLogger(__name__) + +_TIMEOUT_PREFIX = "Timeout of " + + +@dataclass(frozen=True) +class TrainingIterationResult: + """Normalised outcome of one training iteration. + + Attributes + ---------- + status : str + One of ``"passed"``, ``"timeout"``, ``"error"``. + output : str | None + Bot's final answer on success, else ``None``. + error : str | None + Error description on failure, else ``None``. + """ + + status: str + output: str | None + error: str | None + + +class LearningRunner: + """Runs one :class:`ReadingBot` iteration against a mounted source directory. + + Parameters + ---------- + model : str + Model identifier forwarded to :class:`ReadingBot`. + source_path : Path + Directory mounted into the sandbox as the material the agent should + learn from. May be a source-code repo, a docs tree, a dataset, or + any other directory. + memory_dir : Path + Host-side directory backing the agent's ``/memories/`` tree. + max_bot_steps : int, optional + ``max_iterations`` forwarded to :meth:`ReadingBot.run`. Defaults to + ``40``. + """ + + def __init__( + self, + *, + model: str, + source_path: Path, + memory_dir: Path, + max_bot_steps: int = 40, + ) -> None: + """Initialize a runner for one training source. + + Parameters + ---------- + model : str + Model identifier forwarded to the bot. + source_path : Path + Directory mounted as the bot's working directory. + memory_dir : Path + Host directory backing the bot's persistent memory. + max_bot_steps : int, optional + Maximum internal bot steps per invocation. + """ + self._model = model + self._source_path = source_path + self._memory_dir = memory_dir + self._max_bot_steps = max_bot_steps + + # ------------------------------------------------------------------ + + def run(self, task_prompt: str, timeout_s: int) -> TrainingIterationResult: + """Execute one bot invocation and return a normalised result. + + Parameters + ---------- + task_prompt : str + Full prompt (typically the AGENTS.md contents plus an iteration + header) passed to the bot as its task. + timeout_s : int + Per-iteration wall-clock cap forwarded to :meth:`ReadingBot.run`. + + Returns + ------- + TrainingIterationResult + Normalised outcome. Never raises for a failed iteration; only + configuration or infrastructure errors propagate. + """ + bot = ReadingBot( + model=self._model, + folder_to_mount=str(self._source_path), + additional_tools=[MemoryTool(memory_dir=str(self._memory_dir))], + ) + + bot_result: BotRunResult = bot.run( + task_prompt, + max_iterations=self._max_bot_steps, + timeout_in_seconds=timeout_s, + ) + + return self._map(bot_result) + + # ------------------------------------------------------------------ + + @staticmethod + def _map(bot_result: BotRunResult) -> TrainingIterationResult: + """Map a bot result to the training iteration result model. + + Parameters + ---------- + bot_result : BotRunResult + Raw result returned by the bot. + + Returns + ------- + TrainingIterationResult + Normalized passed, timeout, or error result. + """ + if bot_result.status: + return TrainingIterationResult( + status="passed", output=bot_result.result, error=None + ) + + error = bot_result.error or "Unknown error" + if error.startswith(_TIMEOUT_PREFIX): + return TrainingIterationResult( + status="timeout", output=None, error=error + ) + return TrainingIterationResult( + status="error", output=None, error=error + ) diff --git a/src/microbots/auto_memory/training/training_instructions.md b/src/microbots/auto_memory/training/training_instructions.md new file mode 100644 index 0000000..5554618 --- /dev/null +++ b/src/microbots/auto_memory/training/training_instructions.md @@ -0,0 +1,79 @@ +# Repo-Learning Agent Instructions + +You are a **package-maintainer agent** in a training phase. Your **only** job +is to **learn the repository** and write down what you learn as durable notes +in memory. You are not here to fix bugs, implement features, close tickets, +land patches, or make any change to the repository itself. + +A future evaluation loop will reuse the notes you leave behind. If it isn't in +memory, it doesn't exist. Optimise every action for "what will the next agent, +starting cold, need in order to act as maintainer of this repo?" + +--- + +## Mission + +For the repository under study, build up a **maintainer's mental model** and +persist it to `/memories/` using the `memory` tool. + +--- + +## Memory Protocol (non-negotiable) + +You have a `memory` tool that persists files under `/memories/`. Follow this +protocol every iteration: + +1. **Always start with** `memory view /memories` to see what prior iterations + already learned. Do not re-derive facts that are already recorded. +2. **Read before you write.** If a note already covers the area you're + exploring, extend or correct it instead of creating a parallel file. +3. **Write as you go.** Record each non-trivial finding immediately, in the + iteration you discovered it — do not batch discoveries until "the end". +4. **Cite sources.** Every claim should be traceable to a file path (and, when + useful, a symbol or line range) or an exact command + observed output. +5. **Prefer facts over prose.** Short bullets, tables, and code snippets beat + paragraphs. Notes are read by another agent, not a human reviewer. +6. **Keep memory tidy.** Rename vague files, delete stale ones, and merge + duplicates. A messy `/memories/` is worse than a small one. +7. **Never invent.** If you don't know, say so and (if possible) record the + next investigation step. Speculation poisons the next agent. + +Choose your own file names and structure inside `/memories/`. Organise it in +whatever way best fits the repo you are studying — just keep it discoverable, +non-duplicative, and easy for a cold-start agent to navigate. + +--- + +## Working Loop + +For each iteration: + +1. `memory view /memories` — recover prior state. +2. Pick the **highest-value gap** in the maintainer mental model above. +3. Investigate read-only: browse code, inspect tests, and run read-only + commands (e.g. listing files, viewing history, running an existing test + suite to observe behaviour). Do **not** modify repository files. +4. Record findings into the appropriate memory file(s), creating or + reorganising files as needed. +5. Before ending the iteration, do a final `memory view /memories` sanity + check: is your latest finding actually saved, cited, and discoverable? + +--- + +## What NOT to Record + +- Raw dumps of large files. Summarise and link by path instead. +- Transient reasoning ("I'm going to look at X next") — only keep it if it + survives the iteration as a real open question. +- Anything you are only guessing. Mark uncertainty explicitly or omit it. +- Secrets, tokens, or environment-specific absolute paths that won't + generalise to the next agent's machine. + +--- + +## Definition of Done (per iteration) + +An iteration is "done" when `/memories/` is strictly more useful to a +cold-start maintainer than it was when the iteration began — new facts +added, stale facts corrected or removed. If memory did not improve, the +iteration is not done. Update memory, then stop. diff --git a/src/microbots/auto_memory/training/training_source.py b/src/microbots/auto_memory/training/training_source.py new file mode 100644 index 0000000..e4ac5a7 --- /dev/null +++ b/src/microbots/auto_memory/training/training_source.py @@ -0,0 +1,388 @@ +"""Source specification for a training run. + +A training run needs a local directory to mount into the bot's sandbox. +The source of that directory can be: + +* ``type: "path"`` — an existing directory on disk. Used as-is. +* ``type: "git"`` — a git URL that is cloned (or fetched + checked out) + into a local destination before the run starts. + +The nested YAML shape is:: + + source: + type: path + path: /some/local/dir + +or:: + + source: + type: git + url: https://github.com/foo/bar.git + ref: main # optional branch / tag / commit + cache_dir: /some/dir # optional; default is /source/ + +Legacy top-level ``source_path: `` is still accepted by +:meth:`TrainingConfig.load_from_yaml` and is normalised into a +``TrainingSource(type="path", path=...)``. +""" + +from __future__ import annotations + +import re +import shutil +import subprocess +from dataclasses import dataclass +from logging import getLogger +from pathlib import Path +from typing import Any + +from microbots.auto_memory.errors import ConfigError + +logger = getLogger(__name__) + +VALID_TYPES = ("path", "git") + +# Recognises the URL shapes we accept as ``type: git``. We only auto-detect +# from a bare string in the legacy ``source_path`` field; the nested form +# uses an explicit ``type:`` so no detection is needed. +_GIT_URL_RE = re.compile( + r"""^( + https?:// # http://, https:// + | git:// # git:// + | ssh:// # ssh:// + | git@[^:]+: # git@host:path + )""", + re.VERBOSE, +) + + +def looks_like_git_url(value: str) -> bool: + """Check whether a value looks like a git remote URL. + + Parameters + ---------- + value : str + Candidate source value. + + Returns + ------- + bool + Whether the value has a recognized git URL form. + """ + if not isinstance(value, str): + return False + if _GIT_URL_RE.match(value): + return True + return value.endswith(".git") + + +@dataclass +class TrainingSource: + """Where the training run's source directory comes from. + + Attributes + ---------- + type : str + ``"path"`` for a local directory, ``"git"`` for a remote repo. + path : Path | None + For ``type="path"``: the existing local directory (required). + For ``type="git"``: optional explicit clone destination; if unset, + the loop uses ``/source/`` at materialization time. + url : str | None + Git remote URL. Required when ``type="git"``, ignored otherwise. + ref : str | None + Branch, tag, or commit to check out after cloning. Ignored for + ``type="path"``. + cache_dir : Path | None + Alternative name for ``path`` used only with ``type="git"``. Set + this to reuse a clone across runs; leave unset for a fresh clone + under the training workdir. + """ + + type: str = "path" + path: Path | None = None + url: str | None = None + ref: str | None = None + cache_dir: Path | None = None + + # ------------------------------------------------------------------ + # Constructors + + @classmethod + def from_mapping(cls, data: Any, *, base_dir: Path) -> "TrainingSource": + """Build a :class:`TrainingSource` from a YAML mapping. + + Parameters + ---------- + data : Any + Value parsed from the YAML ``source:`` key. Must be a mapping. + base_dir : Path + Directory used to resolve any relative ``path`` / ``cache_dir`` + entries (typically the YAML file's directory). + + Returns + ------- + TrainingSource + Parsed source specification. + """ + if not isinstance(data, dict): + raise ConfigError( + "'source' must be a mapping with a 'type' key, " + f"got {type(data).__name__}" + ) + + stype = str(data.get("type", "")).strip() + if stype not in VALID_TYPES: + raise ConfigError( + f"'source.type' must be one of {VALID_TYPES}, got '{stype}'" + ) + + def _resolve(p: Any) -> Path | None: + """Resolve an optional source path relative to ``base_dir``. + + Parameters + ---------- + p : Any + Optional configured path value. + + Returns + ------- + Path | None + Resolved path, or ``None`` when no value was provided. + """ + if p is None: + return None + path = Path(p) + return path if path.is_absolute() else (base_dir / path).resolve() + + return cls( + type=stype, + path=_resolve(data.get("path")), + url=(str(data["url"]) if data.get("url") is not None else None), + ref=(str(data["ref"]) if data.get("ref") is not None else None), + cache_dir=_resolve(data.get("cache_dir")), + ) + + @classmethod + def from_legacy_source_path( + cls, value: str | Path, *, base_dir: Path | None = None + ) -> "TrainingSource": + """Wrap the legacy ``source_path`` value into a :class:`TrainingSource`. + + Auto-detects git URLs so an existing config that puts a URL in + ``source_path`` keeps working. + + Parameters + ---------- + value : str | Path + Legacy local path or git URL value. + base_dir : Path | None, optional + Directory used to resolve a relative local path. + + Returns + ------- + TrainingSource + Normalized source specification. + """ + raw = str(value) + if looks_like_git_url(raw): + return cls(type="git", url=raw) + + path = Path(value) + if not path.is_absolute() and base_dir is not None: + path = (base_dir / path).resolve() + return cls(type="path", path=path) + + # ------------------------------------------------------------------ + # Validation + + def validate(self) -> None: + """Validate the spec. Cheap checks only — cloning is deferred.""" + if self.type == "path": + if self.path is None: + raise ConfigError("'source.path' is required when type='path'") + if not self.path.exists() or not self.path.is_dir(): + raise ConfigError( + "'source.path' must be an existing directory, " + f"got {self.path}" + ) + elif self.type == "git": + if not self.url: + raise ConfigError("'source.url' is required when type='git'") + # ``path`` and ``cache_dir`` are optional; either may double as the + # clone destination. If both are given, ``cache_dir`` wins because + # it is the git-specific field. + else: # pragma: no cover — from_mapping / __post_init__ blocks this. + raise ConfigError(f"Unknown source.type '{self.type}'") + + # ------------------------------------------------------------------ + # Materialization + + def materialize(self, default_dest: Path) -> Path: + """Return a local directory ready to be mounted into the sandbox. + + For ``type="path"`` this is a no-op returning :attr:`path`. For + ``type="git"`` this clones (or fetches + resets) into either + :attr:`cache_dir`, :attr:`path`, or ``default_dest``. + + Parameters + ---------- + default_dest : Path + Fallback clone destination when neither ``cache_dir`` nor + ``path`` is set. Typically ``/source``. + + Returns + ------- + Path + The local, on-disk source directory. + """ + if self.type == "path": + assert self.path is not None # guarded by validate() + return self.path + + assert self.type == "git" and self.url # guarded by validate() + dest = self.cache_dir or self.path or default_dest + dest = dest.resolve() + + if _is_existing_git_checkout(dest): + logger.info("TrainingSource: refreshing existing git checkout at %s", dest) + _git_fetch_and_checkout(dest, url=self.url, ref=self.ref) + else: + if dest.exists() and any(dest.iterdir()): + raise ConfigError( + f"Git clone destination {dest} exists and is not empty " + "(and is not an existing git checkout); refusing to clobber it." + ) + dest.parent.mkdir(parents=True, exist_ok=True) + logger.info("TrainingSource: cloning %s into %s", self.url, dest) + _git_clone(url=self.url, dest=dest, ref=self.ref) + + # Cache the resolved local path so downstream code / metadata sees it. + self.path = dest + return dest + + # ------------------------------------------------------------------ + # Serialisation + + def to_meta(self) -> dict: + """Build metadata for ``training_meta.json``. + + Returns + ------- + dict + JSON-serializable source metadata. + """ + return { + "type": self.type, + "path": str(self.path) if self.path else None, + "url": self.url, + "ref": self.ref, + "cache_dir": str(self.cache_dir) if self.cache_dir else None, + } + + +# --------------------------------------------------------------------------- +# git helpers +# --------------------------------------------------------------------------- + + +def _run_git(args: list[str], *, cwd: Path | None = None) -> None: + """Run a git subcommand, raising :class:`ConfigError` on failure. + + Parameters + ---------- + args : list[str] + Git arguments following the executable name. + cwd : Path | None, optional + Working directory for the command. + """ + cmd = ["git", *args] + logger.debug("TrainingSource: running %s (cwd=%s)", " ".join(cmd), cwd) + try: + subprocess.run( + cmd, + cwd=str(cwd) if cwd else None, + check=True, + capture_output=True, + text=True, + ) + except FileNotFoundError as exc: + raise ConfigError( + "'git' executable not found on PATH; required for type='git' sources" + ) from exc + except subprocess.CalledProcessError as exc: + stderr = (exc.stderr or "").strip() + raise ConfigError( + f"git {' '.join(args)} failed (exit {exc.returncode}): {stderr}" + ) from exc + + +def _is_existing_git_checkout(dest: Path) -> bool: + """Check whether a destination contains a git checkout. + + Parameters + ---------- + dest : Path + Candidate checkout directory. + + Returns + ------- + bool + Whether the destination and its ``.git`` entry exist. + """ + return dest.exists() and (dest / ".git").exists() + + +def _git_clone(*, url: str, dest: Path, ref: str | None) -> None: + """Clone a git source and optionally check out a requested ref. + + Parameters + ---------- + url : str + Remote repository URL. + dest : Path + Local clone destination. + ref : str | None + Optional branch, tag, or commit to check out. + """ + args = ["clone", url, str(dest)] + if ref: + # ``--branch`` accepts branches or tags. Commits still need a + # follow-up checkout below. + args[1:1] = ["--branch", ref] + try: + _run_git(args) + except ConfigError: + # A commit SHA isn't a valid --branch target; retry without it and + # check out the SHA after the clone. + if not ref: + raise + if dest.exists(): + shutil.rmtree(dest, ignore_errors=True) + _run_git(["clone", url, str(dest)]) + _run_git(["checkout", ref], cwd=dest) + + +def _git_fetch_and_checkout(dest: Path, *, url: str, ref: str | None) -> None: + """Refresh an existing checkout from its configured remote and ref. + + Parameters + ---------- + dest : Path + Existing local checkout. + url : str + Remote repository URL to configure as ``origin``. + ref : str | None + Optional branch, tag, or commit to check out. + """ + # Point origin at the requested URL in case it changed since the last run. + _run_git(["remote", "set-url", "origin", url], cwd=dest) + _run_git(["fetch", "--prune", "origin"], cwd=dest) + if ref: + _run_git(["checkout", ref], cwd=dest) + # For branches, fast-forward to the fetched tip. Ignored (harmless + # error) for tags / detached HEADs by wrapping in a try. + try: + _run_git(["merge", "--ff-only", f"origin/{ref}"], cwd=dest) + except ConfigError: + pass diff --git a/test/auto_memory/training/__init__.py b/test/auto_memory/training/__init__.py new file mode 100644 index 0000000..13ec8dd --- /dev/null +++ b/test/auto_memory/training/__init__.py @@ -0,0 +1 @@ +"""Tests for the auto-memory training package.""" \ No newline at end of file diff --git a/test/auto_memory/training/test_cli.py b/test/auto_memory/training/test_cli.py new file mode 100644 index 0000000..8bdf50c --- /dev/null +++ b/test/auto_memory/training/test_cli.py @@ -0,0 +1,247 @@ +"""Tests for programmatic and command-line training entry points.""" + +import runpy +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +from microbots.auto_memory.errors import ConfigError +from microbots.auto_memory.training.cli import ( + main, + run_training, + run_training_from_yaml, +) +from microbots.auto_memory.training.orchestrator import TrainingSummary +from microbots.auto_memory.training.training_source import TrainingSource + +pytestmark = pytest.mark.unit + + +def _summary(*, error: str | None = None) -> TrainingSummary: + return TrainingSummary( + final_status="completed", + iterations_run=2, + elapsed_s=1.25, + memory_dir=Path("/memory"), + error_message=error, + ) + + +def test_run_training_accepts_source_forms_and_default_workdir( + tmp_path: Path, +) -> None: + source_path = tmp_path / "source" + source_path.mkdir() + agents = tmp_path / "AGENTS.md" + agents.write_text("learn", encoding="utf-8") + + with patch( + "microbots.auto_memory.training.cli.TrainingOrchestrator" + ) as orchestrator: + orchestrator.return_value.run.return_value = _summary() + result = run_training( + source={"type": "path", "path": source_path}, + memory_dir=tmp_path / "memory", + model="azure-openai/gpt-4o", + agents_md_path=agents, + ) + + assert result.final_status == "completed" + config = orchestrator.call_args.kwargs["config"] + assert config.source.path == source_path + assert orchestrator.call_args.kwargs["workdir"].name.startswith( + ".training-run-" + ) + + source = TrainingSource(type="path", path=source_path) + with patch( + "microbots.auto_memory.training.cli.TrainingOrchestrator" + ) as orchestrator: + orchestrator.return_value.run.return_value = _summary() + run_training( + source=source, + memory_dir=tmp_path / "memory", + model="azure-openai/gpt-4o", + workdir=tmp_path / "work", + ) + assert orchestrator.call_args.kwargs["config"].source is source + assert orchestrator.call_args.kwargs["workdir"] == tmp_path / "work" + + with patch( + "microbots.auto_memory.training.cli.TrainingOrchestrator" + ) as orchestrator: + orchestrator.return_value.run.return_value = _summary() + run_training( + source_path=source_path, + memory_dir=tmp_path / "memory", + model="azure-openai/gpt-4o", + workdir=tmp_path / "work", + ) + assert orchestrator.call_args.kwargs["config"].source.path == source_path + + +@pytest.mark.parametrize( + ("kwargs", "message"), + [ + ({}, "required"), + ( + {"source": TrainingSource(), "source_path": "."}, + "either 'source' or 'source_path'", + ), + ({"source": object()}, "must be a TrainingSource or mapping"), + ], +) +def test_run_training_rejects_invalid_source( + tmp_path: Path, kwargs: dict[str, object], message: str +) -> None: + with pytest.raises(ConfigError, match=message): + run_training( + **kwargs, + memory_dir=tmp_path / "memory", + model="azure-openai/gpt-4o", + ) + + +def test_run_training_from_yaml_uses_explicit_and_default_workdir( + tmp_path: Path, +) -> None: + config = MagicMock() + config.memory_dir = tmp_path / "memory" + with ( + patch( + "microbots.auto_memory.training.cli.TrainingConfig.load_from_yaml", + return_value=config, + ), + patch( + "microbots.auto_memory.training.cli.TrainingOrchestrator" + ) as orchestrator, + ): + orchestrator.return_value.run.return_value = _summary() + run_training_from_yaml("config.yaml") + assert orchestrator.call_args.kwargs["workdir"].name.startswith( + ".training-run-" + ) + run_training_from_yaml("config.yaml", workdir=tmp_path / "work") + assert orchestrator.call_args.kwargs["workdir"] == tmp_path / "work" + + +def test_main_config_success_and_error_output(capsys: pytest.CaptureFixture[str]) -> None: + with patch( + "microbots.auto_memory.training.cli.run_training_from_yaml", + return_value=_summary(error="last failure"), + ) as run: + assert main(["--config", "config.yaml", "--workdir", "work", "-v"]) == 0 + run.assert_called_once_with(Path("config.yaml"), workdir=Path("work")) + captured = capsys.readouterr() + assert "training completed" in captured.out + assert "last error: last failure" in captured.err + + with patch( + "microbots.auto_memory.training.cli.run_training_from_yaml", + side_effect=ConfigError("bad config"), + ): + assert main(["--config", "bad.yaml"]) == 2 + assert "config error: bad config" in capsys.readouterr().err + + +@pytest.mark.parametrize( + ("args", "missing"), + [ + ([], "--memory, --model"), + (["--memory", "memory"], "--model"), + (["--model", "azure-openai/gpt-4o"], "--memory"), + ], +) +def test_main_reports_missing_required_flags( + args: list[str], missing: str, capsys: pytest.CaptureFixture[str] +) -> None: + assert main(args) == 2 + assert missing in capsys.readouterr().err + + +def test_main_reports_missing_source(capsys: pytest.CaptureFixture[str]) -> None: + assert main(["--memory", "memory", "--model", "azure-openai/gpt-4o"]) == 2 + assert "--source or --source-git-url" in capsys.readouterr().err + + +def test_main_runs_local_and_git_sources(tmp_path: Path) -> None: + with patch( + "microbots.auto_memory.training.cli.run_training", return_value=_summary() + ) as run: + assert ( + main( + [ + "--source", + str(tmp_path), + "--memory", + "memory", + "--model", + "azure-openai/gpt-4o", + ] + ) + == 0 + ) + assert run.call_args.kwargs["source_path"] == tmp_path + + with patch( + "microbots.auto_memory.training.cli.run_training", return_value=_summary() + ) as run: + assert ( + main( + [ + "--source-git-url", + "url", + "--source", + str(tmp_path / "checkout"), + "--source-cache-dir", + str(tmp_path / "cache"), + "--source-ref", + "main", + "--memory", + "memory", + "--model", + "azure-openai/gpt-4o", + "--iterations", + "2", + "--per-iteration-timeout", + "10", + "--total-timeout-min", + "3", + "--max-bot-steps", + "4", + "--reset-memory", + "--agents-md", + "AGENTS.md", + ] + ) + == 0 + ) + source = run.call_args.kwargs["source"] + assert source.path == (tmp_path / "checkout").resolve() + assert source.cache_dir == (tmp_path / "cache").resolve() + assert source.ref == "main" + + with patch( + "microbots.auto_memory.training.cli.run_training", return_value=_summary() + ) as run: + main( + [ + "--source-git-url", + "url", + "--memory", + "memory", + "--model", + "azure-openai/gpt-4o", + ] + ) + assert run.call_args.kwargs["source"].path is None + assert run.call_args.kwargs["source"].cache_dir is None + + +def test_module_entry_point_exits_with_main_result() -> None: + with ( + patch("microbots.auto_memory.training.cli.main", return_value=7), + pytest.raises(SystemExit, match="7"), + ): + runpy.run_module("microbots.auto_memory.training.__main__", run_name="__main__") diff --git a/test/auto_memory/training/test_config.py b/test/auto_memory/training/test_config.py new file mode 100644 index 0000000..e27b2f1 --- /dev/null +++ b/test/auto_memory/training/test_config.py @@ -0,0 +1,145 @@ +"""Tests for training configuration loading and validation.""" + +from pathlib import Path +from unittest.mock import patch + +import pytest +import yaml + +from microbots.auto_memory.errors import ConfigError +from microbots.auto_memory.training.config import TrainingConfig +from microbots.auto_memory.training.training_source import TrainingSource + +pytestmark = pytest.mark.unit + + +def _config(tmp_path: Path, **overrides: object) -> TrainingConfig: + source = tmp_path / "source" + source.mkdir(exist_ok=True) + agents = tmp_path / "AGENTS.md" + agents.write_text("instructions", encoding="utf-8") + values = { + "source": TrainingSource(type="path", path=source), + "memory_dir": tmp_path / "memory", + "model": "azure-openai/gpt-4o", + "agents_md_path": agents, + } + values.update(overrides) + return TrainingConfig(**values) + + +def test_load_nested_yaml_resolves_paths_and_options(tmp_path: Path) -> None: + (tmp_path / "source").mkdir() + (tmp_path / "instructions.md").write_text("learn", encoding="utf-8") + config_path = tmp_path / "training.yaml" + config_path.write_text( + "\n".join( + [ + "source:", + " type: path", + " path: source", + "memory_dir: memory", + "model: azure-openai/gpt-4o", + "agents_md_path: instructions.md", + "iterations: 2", + "per_iteration_timeout: 10", + "total_timeout_min: 3", + "max_bot_steps: 5", + "reset_memory: true", + ] + ), + encoding="utf-8", + ) + + config = TrainingConfig.load_from_yaml(config_path) + + assert config.source_path == (tmp_path / "source").resolve() + assert config.memory_dir == (tmp_path / "memory").resolve() + assert config.agents_md_path == (tmp_path / "instructions.md").resolve() + assert config.iterations == 2 + assert config.per_iteration_timeout == 10 + assert config.total_timeout_min == 3 + assert config.max_bot_steps == 5 + assert config.reset_memory is True + assert config.read_agents_md() == "learn" + + +def test_load_legacy_yaml_uses_defaults_and_absolute_memory(tmp_path: Path) -> None: + source = tmp_path / "source" + source.mkdir() + memory = tmp_path / "memory" + config_path = tmp_path / "training.yaml" + config_path.write_text( + f"source_path: source\nmemory_dir: {memory}\n" + "model: azure-openai/gpt-4o\n", + encoding="utf-8", + ) + + config = TrainingConfig.load_from_yaml(config_path) + + assert config.source.path == source.resolve() + assert config.memory_dir == memory + assert config.iterations == 3 + assert config.read_agents_md() + + +@pytest.mark.parametrize( + ("contents", "message"), + [ + ("- item\n", "Expected a YAML mapping"), + ("model: azure-openai/gpt-4o\nsource_path: .\n", "memory_dir"), + ("memory_dir: memory\nsource_path: .\n", "model"), + ("memory_dir: memory\nmodel: azure-openai/gpt-4o\n", "Missing source"), + ], +) +def test_load_yaml_rejects_invalid_shapes( + tmp_path: Path, contents: str, message: str +) -> None: + config_path = tmp_path / "training.yaml" + config_path.write_text(contents, encoding="utf-8") + + with pytest.raises(ConfigError, match=message): + TrainingConfig.load_from_yaml(config_path) + + +def test_load_yaml_reports_missing_and_malformed_files(tmp_path: Path) -> None: + missing = tmp_path / "missing.yaml" + with pytest.raises(ConfigError, match="not found"): + TrainingConfig.load_from_yaml(missing) + + config_path = tmp_path / "training.yaml" + config_path.write_text("ignored", encoding="utf-8") + with ( + patch("microbots.auto_memory.training.config.yaml.safe_load") as load, + pytest.raises(ConfigError, match="Failed to parse YAML"), + ): + load.side_effect = yaml.YAMLError("bad yaml") + TrainingConfig.load_from_yaml(config_path) + + +@pytest.mark.parametrize( + ("overrides", "message"), + [ + ({"agents_md_path": Path("missing")}, "agents_md_path"), + ({"model": ""}, "must not be empty"), + ({"model": "invalid"}, "must be in the form"), + ({"model": "unknown/model"}, "unsupported provider"), + ({"iterations": 0}, "iterations"), + ({"per_iteration_timeout": 0}, "per_iteration_timeout"), + ({"total_timeout_min": -1}, "total_timeout_min"), + ({"max_bot_steps": 0}, "max_bot_steps"), + ], +) +def test_validate_rejects_invalid_fields( + tmp_path: Path, overrides: dict[str, object], message: str +) -> None: + config = _config(tmp_path, **overrides) + + with pytest.raises(ConfigError, match=message): + config.validate() + + +def test_validate_accepts_valid_config(tmp_path: Path) -> None: + config = _config(tmp_path) + + config.validate() diff --git a/test/auto_memory/training/test_orchestrator.py b/test/auto_memory/training/test_orchestrator.py new file mode 100644 index 0000000..030c1fa --- /dev/null +++ b/test/auto_memory/training/test_orchestrator.py @@ -0,0 +1,139 @@ +"""Focused tests for TrainingOrchestrator.""" + +from pathlib import Path +from unittest.mock import patch + +import pytest + +from microbots.auto_memory.errors import ConfigError +from microbots.auto_memory.training import TrainingOrchestrator +from microbots.auto_memory.training.config import TrainingConfig +from microbots.auto_memory.training.orchestrator import ( + TrainingOrchestrator as DirectOrchestrator, +) +from microbots.auto_memory.training.runner import TrainingIterationResult +from microbots.auto_memory.training.training_source import TrainingSource + +pytestmark = pytest.mark.unit + + +def _config(tmp_path: Path, *, iterations: int = 2) -> TrainingConfig: + source = tmp_path / "source" + source.mkdir() + agents_md = tmp_path / "AGENTS.md" + agents_md.write_text("Learn the repository.", encoding="utf-8") + return TrainingConfig( + source=TrainingSource(type="path", path=source), + memory_dir=tmp_path / "memory", + model="azure-openai/gpt-4o", + agents_md_path=agents_md, + iterations=iterations, + per_iteration_timeout=45, + max_bot_steps=9, + ) + + +def test_public_export_is_renamed_orchestrator() -> None: + assert TrainingOrchestrator is DirectOrchestrator + + +def test_completed_run_records_iterations_and_runtime_context(tmp_path: Path) -> None: + config = _config(tmp_path) + passed = TrainingIterationResult("passed", "done", None) + + with patch( + "microbots.auto_memory.training.orchestrator.LearningRunner" + ) as runner_class: + runner_class.return_value.run.return_value = passed + summary = TrainingOrchestrator(config, tmp_path / "work").run() + + runner_class.assert_called_once_with( + model=config.model, + source_path=config.source.path, + memory_dir=config.memory_dir, + max_bot_steps=config.max_bot_steps, + ) + assert summary.final_status == "completed" + assert summary.iterations_run == 2 + assert [record.status for record in summary.iteration_records] == [ + "passed", + "passed", + ] + first_prompt = runner_class.return_value.run.call_args_list[0].args[0] + assert "Learn the repository." in first_prompt + assert "Iteration index (zero-based): 0" in first_prompt + assert f"Source directory (mounted in sandbox): {config.source.path}" in first_prompt + assert runner_class.return_value.run.call_args_list[0].kwargs == { + "timeout_s": 45 + } + + +def test_timeout_result_stops_the_run(tmp_path: Path) -> None: + config = _config(tmp_path, iterations=3) + timeout = TrainingIterationResult("timeout", None, "Timeout of 45 seconds") + + with patch( + "microbots.auto_memory.training.orchestrator.LearningRunner" + ) as runner_class: + runner_class.return_value.run.return_value = timeout + summary = TrainingOrchestrator(config, tmp_path / "work").run() + + assert summary.final_status == "timeout" + assert summary.iterations_run == 1 + assert summary.error_message == "Timeout of 45 seconds" + + +def test_runner_exception_returns_error_summary(tmp_path: Path) -> None: + config = _config(tmp_path) + + with patch( + "microbots.auto_memory.training.orchestrator.LearningRunner" + ) as runner_class: + runner_class.return_value.run.side_effect = RuntimeError("broken") + summary = TrainingOrchestrator(config, tmp_path / "work").run() + + assert summary.final_status == "error" + assert summary.iterations_run == 1 + assert summary.error_message == "RuntimeError: broken" + + +def test_total_timeout_stops_before_an_iteration(tmp_path: Path) -> None: + config = _config(tmp_path) + config.total_timeout_min = 1 + + with patch( + "microbots.auto_memory.training.orchestrator.time.monotonic", + side_effect=[0.0, 61.0], + ): + summary = TrainingOrchestrator(config, tmp_path / "work").run() + + assert summary.final_status == "timeout" + assert summary.iterations_run == 0 + + +def test_prepare_workdir_reports_creation_error(tmp_path: Path) -> None: + orchestrator = TrainingOrchestrator(_config(tmp_path), tmp_path / "work") + + with ( + patch.object(Path, "mkdir", side_effect=OSError("denied")), + pytest.raises(ConfigError, match="Cannot create workdir"), + ): + orchestrator.run() + + +def test_prepare_memory_resets_and_reports_creation_error(tmp_path: Path) -> None: + config = _config(tmp_path) + config.reset_memory = True + config.memory_dir.mkdir() + (config.memory_dir / "old").write_text("data", encoding="utf-8") + orchestrator = TrainingOrchestrator(config, tmp_path / "work") + + orchestrator._prepare_memory_dir() + assert config.memory_dir.exists() + assert not (config.memory_dir / "old").exists() + + with ( + patch.object(Path, "mkdir", side_effect=OSError("denied")), + pytest.raises(ConfigError, match="Cannot create memory_dir"), + ): + orchestrator._prepare_memory_dir() \ No newline at end of file diff --git a/test/auto_memory/training/test_runner.py b/test/auto_memory/training/test_runner.py new file mode 100644 index 0000000..8904089 --- /dev/null +++ b/test/auto_memory/training/test_runner.py @@ -0,0 +1,76 @@ +"""Focused tests for the read-only training runner.""" + +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +from microbots.MicroBot import BotRunResult +from microbots.auto_memory.training.runner import LearningRunner + +pytestmark = pytest.mark.unit + + +def test_learning_runner_constructs_and_invokes_reading_bot() -> None: + bot = MagicMock() + bot.run.return_value = BotRunResult(status=True, result="learned", error=None) + + with ( + patch("microbots.auto_memory.training.runner.MemoryTool") as memory_tool, + patch( + "microbots.auto_memory.training.runner.ReadingBot", + return_value=bot, + ) as reading_bot, + ): + result = LearningRunner( + model="azure-openai/gpt-4o", + source_path=Path("/source"), + memory_dir=Path("/memory"), + max_bot_steps=7, + ).run("study this", timeout_s=30) + + memory_tool.assert_called_once_with(memory_dir="/memory") + reading_bot.assert_called_once_with( + model="azure-openai/gpt-4o", + folder_to_mount="/source", + additional_tools=[memory_tool.return_value], + ) + bot.run.assert_called_once_with( + "study this", max_iterations=7, timeout_in_seconds=30 + ) + assert result.status == "passed" + assert result.output == "learned" + assert result.error is None + + +@pytest.mark.parametrize( + ("bot_result", "expected_status", "expected_error"), + [ + ( + BotRunResult(False, None, "Timeout of 30 seconds"), + "timeout", + "Timeout of 30 seconds", + ), + (BotRunResult(False, None, "failed"), "error", "failed"), + (BotRunResult(False, None, None), "error", "Unknown error"), + ], +) +def test_learning_runner_maps_failures( + bot_result: BotRunResult, + expected_status: str, + expected_error: str, +) -> None: + with ( + patch("microbots.auto_memory.training.runner.MemoryTool"), + patch("microbots.auto_memory.training.runner.ReadingBot") as reading_bot, + ): + reading_bot.return_value.run.return_value = bot_result + result = LearningRunner( + model="azure-openai/gpt-4o", + source_path=Path("/source"), + memory_dir=Path("/memory"), + ).run("study this", timeout_s=30) + + assert result.status == expected_status + assert result.output is None + assert result.error == expected_error \ No newline at end of file diff --git a/test/auto_memory/training/test_training_source.py b/test/auto_memory/training/test_training_source.py new file mode 100644 index 0000000..60b002d --- /dev/null +++ b/test/auto_memory/training/test_training_source.py @@ -0,0 +1,254 @@ +"""Tests for local and git training sources.""" + +import subprocess +from pathlib import Path +from unittest.mock import call, patch + +import pytest + +from microbots.auto_memory.errors import ConfigError +from microbots.auto_memory.training.training_source import ( + TrainingSource, + _git_clone, + _git_fetch_and_checkout, + _is_existing_git_checkout, + _run_git, + looks_like_git_url, +) + +pytestmark = pytest.mark.unit + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + (123, False), + ("https://example.com/repo", True), + ("git@example.com:repo", True), + ("repo.git", True), + ("local/path", False), + ], +) +def test_looks_like_git_url(value: object, expected: bool) -> None: + assert looks_like_git_url(value) is expected # type: ignore[arg-type] + + +def test_from_mapping_resolves_all_fields(tmp_path: Path) -> None: + absolute = tmp_path / "absolute" + source = TrainingSource.from_mapping( + { + "type": "git", + "path": "checkout", + "url": 123, + "ref": 456, + "cache_dir": absolute, + }, + base_dir=tmp_path, + ) + + assert source.path == (tmp_path / "checkout").resolve() + assert source.url == "123" + assert source.ref == "456" + assert source.cache_dir == absolute + + +def test_from_mapping_defaults_optional_fields(tmp_path: Path) -> None: + source = TrainingSource.from_mapping({"type": "path"}, base_dir=tmp_path) + + assert source.path is None + assert source.url is None + assert source.ref is None + assert source.cache_dir is None + + +@pytest.mark.parametrize("data", [None, "path"]) +def test_from_mapping_requires_mapping(data: object, tmp_path: Path) -> None: + with pytest.raises(ConfigError, match="must be a mapping"): + TrainingSource.from_mapping(data, base_dir=tmp_path) + + +def test_from_mapping_rejects_invalid_type(tmp_path: Path) -> None: + with pytest.raises(ConfigError, match="source.type"): + TrainingSource.from_mapping({"type": "other"}, base_dir=tmp_path) + + +def test_legacy_source_detection_and_resolution(tmp_path: Path) -> None: + remote = TrainingSource.from_legacy_source_path("https://example/repo.git") + relative = TrainingSource.from_legacy_source_path("source", base_dir=tmp_path) + absolute = TrainingSource.from_legacy_source_path(tmp_path) + + assert remote == TrainingSource(type="git", url="https://example/repo.git") + assert relative.path == (tmp_path / "source").resolve() + assert absolute.path == tmp_path + + +def test_validate_path_and_git_sources(tmp_path: Path) -> None: + source = tmp_path / "source" + source.mkdir() + TrainingSource(type="path", path=source).validate() + TrainingSource(type="git", url="https://example/repo.git").validate() + + invalid = [ + TrainingSource(type="path"), + TrainingSource(type="path", path=tmp_path / "missing"), + TrainingSource(type="git"), + TrainingSource(type="other"), + ] + for source_spec in invalid: + with pytest.raises(ConfigError): + source_spec.validate() + + +def test_materialize_local_source_is_noop(tmp_path: Path) -> None: + source = TrainingSource(type="path", path=tmp_path) + + assert source.materialize(tmp_path / "unused") == tmp_path + + +def test_materialize_clones_into_selected_destinations(tmp_path: Path) -> None: + path_dest = tmp_path / "path-dest" + cache_dest = tmp_path / "cache-dest" + source = TrainingSource( + type="git", + url="https://example/repo.git", + path=path_dest, + cache_dir=cache_dest, + ) + + with patch( + "microbots.auto_memory.training.training_source._git_clone" + ) as clone: + assert source.materialize(tmp_path / "default") == cache_dest.resolve() + + clone.assert_called_once_with( + url="https://example/repo.git", dest=cache_dest.resolve(), ref=None + ) + assert source.path == cache_dest.resolve() + + default_source = TrainingSource(type="git", url="url") + with patch( + "microbots.auto_memory.training.training_source._git_clone" + ) as clone: + default_source.materialize(tmp_path / "default") + assert clone.call_args.kwargs["dest"] == (tmp_path / "default").resolve() + + +def test_materialize_refreshes_checkout_and_rejects_nonempty_dest( + tmp_path: Path, +) -> None: + checkout = tmp_path / "checkout" + (checkout / ".git").mkdir(parents=True) + source = TrainingSource(type="git", url="url", path=checkout, ref="main") + with patch( + "microbots.auto_memory.training.training_source._git_fetch_and_checkout" + ) as fetch: + assert source.materialize(tmp_path / "default") == checkout.resolve() + fetch.assert_called_once_with(checkout.resolve(), url="url", ref="main") + + nonempty = tmp_path / "nonempty" + nonempty.mkdir() + (nonempty / "file").write_text("data", encoding="utf-8") + with pytest.raises(ConfigError, match="refusing to clobber"): + TrainingSource(type="git", url="url", path=nonempty).materialize( + tmp_path / "default" + ) + + +def test_metadata_and_checkout_detection(tmp_path: Path) -> None: + checkout = tmp_path / "checkout" + (checkout / ".git").mkdir(parents=True) + source = TrainingSource( + type="git", path=checkout, url="url", ref="main", cache_dir=tmp_path + ) + + assert _is_existing_git_checkout(checkout) + assert not _is_existing_git_checkout(tmp_path / "missing") + assert source.to_meta() == { + "type": "git", + "path": str(checkout), + "url": "url", + "ref": "main", + "cache_dir": str(tmp_path), + } + assert TrainingSource().to_meta()["path"] is None + assert TrainingSource().to_meta()["cache_dir"] is None + + +def test_run_git_success_and_errors(tmp_path: Path) -> None: + with patch("subprocess.run") as run: + _run_git(["status"], cwd=tmp_path) + _run_git(["version"]) + assert run.call_args_list[0].kwargs["cwd"] == str(tmp_path) + assert run.call_args_list[1].kwargs["cwd"] is None + + with ( + patch("subprocess.run", side_effect=FileNotFoundError), + pytest.raises(ConfigError, match="executable not found"), + ): + _run_git(["status"]) + + error = subprocess.CalledProcessError(3, "git", stderr="failure") + with ( + patch("subprocess.run", side_effect=error), + pytest.raises(ConfigError, match="failure"), + ): + _run_git(["status"]) + + +def test_git_clone_paths(tmp_path: Path) -> None: + dest = tmp_path / "dest" + with patch( + "microbots.auto_memory.training.training_source._run_git" + ) as run: + _git_clone(url="url", dest=dest, ref=None) + _git_clone(url="url", dest=dest, ref="main") + assert run.call_args_list == [ + call(["clone", "url", str(dest)]), + call(["clone", "--branch", "main", "url", str(dest)]), + ] + + +def test_git_clone_retries_commit_and_propagates_without_ref(tmp_path: Path) -> None: + dest = tmp_path / "dest" + dest.mkdir() + with patch( + "microbots.auto_memory.training.training_source._run_git", + side_effect=[ConfigError("branch"), None, None], + ) as run: + _git_clone(url="url", dest=dest, ref="abc123") + assert run.call_args_list[1:] == [ + call(["clone", "url", str(dest)]), + call(["checkout", "abc123"], cwd=dest), + ] + assert not dest.exists() + + with patch( + "microbots.auto_memory.training.training_source._run_git", + side_effect=[ConfigError("branch"), None, None], + ) as run: + _git_clone(url="url", dest=dest, ref="abc123") + assert run.call_count == 3 + + with ( + patch( + "microbots.auto_memory.training.training_source._run_git", + side_effect=ConfigError("clone"), + ), + pytest.raises(ConfigError, match="clone"), + ): + _git_clone(url="url", dest=dest, ref=None) + + +def test_git_fetch_checkout_with_and_without_ref(tmp_path: Path) -> None: + with patch( + "microbots.auto_memory.training.training_source._run_git" + ) as run: + _git_fetch_and_checkout(tmp_path, url="url", ref=None) + assert run.call_count == 2 + + with patch( + "microbots.auto_memory.training.training_source._run_git", + side_effect=[None, None, None, ConfigError("detached")], + ) as run: + _git_fetch_and_checkout(tmp_path, url="url", ref="main") + assert run.call_count == 4