feat: enhance runner configuration and loading mechanism for auto_memory module

src/microbots/auto_memory/cli.py

@@ -2,20 +2,117 @@
 
 from __future__ import annotations
 
+import importlib
+import importlib.util
 import uuid
 from datetime import datetime, timezone
 from logging import getLogger
 from pathlib import Path
+from typing import Callable
 
 from microbots.auto_memory.callbacks import ShellCallbackRunner
 from microbots.auto_memory.config import TaskConfig
+from microbots.auto_memory.errors import ConfigError
 from microbots.auto_memory.orchestrator import RunSummary, TrainingLoopOrchestrator
-from microbots.auto_memory.runners.writing_bot_runner import WritingBotRunner
+from microbots.auto_memory.runners.base import AgentRunner
 from microbots.auto_memory.workspace import WorkspaceManager
 
 logger = getLogger(__name__)
 
 
+def _load_runner_class(runner_spec: str, base_dir: Path) -> Callable[..., AgentRunner]:
+    """Resolve a runner class from a task config ``runner`` string.
+
+    Two forms are supported:
+
+    * **Dotted import path** — ``"pkg.module.ClassName"``. Imported via the
+      normal import system; the module must be importable (installed package
+      or on ``sys.path``).
+    * **File path plus class** — ``"path/to/file.py:ClassName"``. Loaded
+      directly from disk with :mod:`importlib.util`, so the runner can live
+      outside the microbots package. Relative file paths are resolved against
+      *base_dir* (the task YAML's directory).
+
+    Parameters
+    ----------
+    runner_spec : str
+        The ``runner`` value from the task configuration.
+    base_dir : Path
+        Directory used to resolve relative file paths (the task YAML's dir).
+
+    Returns
+    -------
+    Callable[..., AgentRunner]
+        The resolved runner factory — a class or any callable that accepts
+        ``model=...`` plus ``runner_params`` and returns an
+        :class:`~microbots.auto_memory.runners.base.AgentRunner`.
+
+    Raises
+    ------
+    ConfigError
+        If the spec is malformed, the module/file cannot be imported, the
+        class is not found, or the resolved attribute is not callable.
+    """
+    if ":" in runner_spec:
+        # File-path form: "<path>.py:<ClassName>"
+        file_part, _, cls_name = runner_spec.rpartition(":")
+        if not file_part or not cls_name:
+            raise ConfigError(
+                f"Invalid runner spec '{runner_spec}'; expected 'path/to/file.py:ClassName'"
+            )
+        file_path = Path(file_part)
+        if not file_path.is_absolute():
+            file_path = (base_dir / file_path).resolve()
+        if not file_path.is_file():
+            raise ConfigError(f"Runner file not found: {file_path}")
+
+        module_spec = importlib.util.spec_from_file_location(
+            "_microbots_user_runner", file_path
+        )
+        if module_spec is None or module_spec.loader is None:
+            raise ConfigError(f"Cannot load runner module from {file_path}")
+        module = importlib.util.module_from_spec(module_spec)
+        try:
+            module_spec.loader.exec_module(module)
+        except Exception as exc:  # noqa: BLE001 - surface any import-time error
+            raise ConfigError(
+                f"Failed to import runner file {file_path}: {exc}"
+            ) from exc
+    else:
+        # Dotted import path form: "pkg.module.ClassName"
+        module_path, _, cls_name = runner_spec.rpartition(".")
+        if not module_path or not cls_name:
+            raise ConfigError(
+                f"Invalid runner spec '{runner_spec}'; expected "
+                f"'pkg.module.ClassName' or 'path/to/file.py:ClassName'"
+            )
+        try:
+            module = importlib.import_module(module_path)
+        except ImportError as exc:
+            raise ConfigError(
+                f"Cannot import runner module '{module_path}': {exc}"
+            ) from exc
+        except Exception as exc:  # noqa: BLE001 - surface import-time errors
+            raise ConfigError(
+                f"Failed to import runner module '{module_path}': {exc}"
+            ) from exc
+
+    try:
+        runner_obj = getattr(module, cls_name)
+    except AttributeError as exc:
+        raise ConfigError(
+            f"Runner class '{cls_name}' not found in '{runner_spec}'"
+        ) from exc
+
+    if not callable(runner_obj):
+        raise ConfigError(
+            f"Runner '{cls_name}' in '{runner_spec}' is not callable "
+            f"(got {type(runner_obj).__name__}); expected a class or factory "
+            f"that accepts model=... and returns an AgentRunner"
+        )
+    return runner_obj
+
+
 def run_from_yaml(
     yaml_path: str | Path,
     workdir: str | Path,
@@ -49,14 +146,15 @@ def run_from_yaml(
         random suffix of the form ``run-YYYYMMDD-HHMMSS-ffffff-<rand>`` is
         generated to avoid collisions.
     model : str
-        Model identifier forwarded to :class:`WritingBotRunner` (required,
+        Model identifier forwarded to the configured runner (required,
         keyword-only — e.g. ``"azure-openai/gpt-4o"``).
 
     Returns
     -------
     RunSummary
         Summary of the completed run.
     """
+    yaml_path = Path(yaml_path)
     config = TaskConfig.load_from_yaml(str(yaml_path))
 
     if run_id is None:
@@ -65,8 +163,27 @@ def run_from_yaml(
     run_dir = Path(workdir) / "runs" / run_id
     logger.info("auto_memory: starting run %s at %s", run_id, run_dir)
 
+    runner_cls = _load_runner_class(config.runner, base_dir=yaml_path.resolve().parent)
+    try:
+        agent_runner: AgentRunner = runner_cls(model=model, **config.runner_params)
+    except Exception as exc:  # noqa: BLE001 - surface construction errors as config errors
+        raise ConfigError(
+            f"Failed to construct runner '{config.runner}' with "
+            f"runner_params={config.runner_params!r}: {exc}"
+        ) from exc
+
+    # Structural check: the constructed object must satisfy the AgentRunner
+    # protocol (i.e. expose a run() method). This only verifies method
+    # presence, not its signature, but catches gross misconfigurations early
+    # with a clear error instead of failing deep inside the orchestrator.
+    if not isinstance(agent_runner, AgentRunner):
+        raise ConfigError(
+            f"Runner '{config.runner}' does not satisfy the AgentRunner "
+            f"protocol; it must define run(ctx, timeout_s)"
+        )
+    logger.info("auto_memory: using runner %s", config.runner)
+
     workspace = WorkspaceManager(run_dir=run_dir)
-    agent_runner = WritingBotRunner(model=model)
     callback_runner = ShellCallbackRunner()
 
     orchestrator = TrainingLoopOrchestrator(

src/microbots/auto_memory/config.py

@@ -36,6 +36,15 @@ class TaskConfig:
     analyzer_max_iterations: int = 20
     analyzer_timeout_s: int = 300
 
+    # --- runner selection ---
+    # Either a dotted import path ("pkg.module.ClassName") or a file-path form
+    # ("path/to/file.py:ClassName") resolved relative to the task YAML's dir.
+    # Defaults to the built-in WritingBotRunner so existing configs are
+    # unaffected. Extra keyword arguments for the runner's constructor go in
+    # runner_params (model is always injected separately by the CLI).
+    runner: str = "microbots.auto_memory.runners.writing_bot_runner.WritingBotRunner"
+    runner_params: dict = field(default_factory=dict)
+
     # -----------------------------------------------------------------------
 
     @classmethod
@@ -112,6 +121,16 @@ def load_from_yaml(cls, path: str) -> "TaskConfig":
             analyzer_model=str(data.get("analyzer_model", "azure-openai/gpt-4o")),
             analyzer_max_iterations=int(data.get("analyzer_max_iterations", 20)),
             analyzer_timeout_s=int(data.get("analyzer_timeout_s", 300)),
+            runner=str(
+                data.get(
+                    "runner",
+                    "microbots.auto_memory.runners.writing_bot_runner.WritingBotRunner",
+                )
+            ),
+            # Pass through as-is (no dict() coercion) so a non-mapping value
+            # reaches validate() and produces a clear ConfigError instead of a
+            # cryptic ValueError/TypeError from dict().
+            runner_params=data.get("runner_params", {}) or {},
         )
         config.validate()
         return config
@@ -201,3 +220,17 @@ def validate(self) -> None:
                 raise ConfigError(
                     f"reference_input '{ri.name}' must have a non-empty 'value'"
                 )
+
+        if not self.runner or not self.runner.strip():
+            raise ConfigError("'runner' must not be empty")
+
+        if not isinstance(self.runner_params, dict):
+            raise ConfigError(
+                f"'runner_params' must be a mapping, got {type(self.runner_params).__name__}"
+            )
+
+        if "model" in self.runner_params:
+            raise ConfigError(
+                "'runner_params' must not contain 'model'; it is injected "
+                "automatically by the CLI"
+            )

src/microbots/auto_memory/orchestrator.py

@@ -302,7 +302,11 @@ def run_iteration(
             feedback=feedback,
         )
 
-        ctx = IterationContext(task=task_prompt, memory_dir=memory_dir)
+        ctx = IterationContext(
+            task=task_prompt,
+            memory_dir=memory_dir,
+            output_dir=str(candidate_path),
+        )
 
         # Run agent
         agent_result: AgentResult = self._agent_runner.run(

src/microbots/auto_memory/runners/base.py

@@ -19,10 +19,16 @@ class IterationContext:
     memory_dir : str
         Host-side directory that is both mounted into the agent container
         (``folder_to_mount``) and surfaced to the agent via :class:`~microbots.tools.tool_definitions.memory_tool.MemoryTool`.
+    output_dir : str
+        Host-side directory (the iteration's ``candidate`` path) where the
+        runner should write the artefact that callbacks validate. Callbacks
+        receive this same path as ``$CANDIDATE``, so a runner that produces a
+        file/tree here closes the produce → validate loop.
     """
 
     task: str
     memory_dir: str
+    output_dir: str
 
 
 @dataclass

src/microbots/auto_memory/workspace.py

@@ -26,7 +26,9 @@ class WorkspaceManager:
 
         <run_dir>/
         ├── memory/
-        │   └── feedback.jsonl          ← managed by MemoryStore
+        │   ├── feedback.jsonl          ← managed by MemoryStore (framework)
+        │   └── <agent notes>           ← written by MemoryTool; the agent's
+        │                                 /memories/… tree maps 1:1 here
         ├── iterations/
         │   ├── iter_00/
         │   │   ├── candidate/          ← agent writes output here

test/auto_memory/runners/test_writing_bot_runner.py

@@ -18,7 +18,9 @@
 
 
 def _make_ctx(memory_dir: str, task: str = _TASK) -> IterationContext:
-    return IterationContext(task=task, memory_dir=memory_dir)
+    return IterationContext(
+        task=task, memory_dir=memory_dir, output_dir=memory_dir
+    )
 
 
 # ---------------------------------------------------------------------------