From b6d19891480d289e9e95f2f37ccfe0090c52f1a0 Mon Sep 17 00:00:00 2001 From: jkbennitt Date: Fri, 7 Aug 2026 18:00:38 -0400 Subject: [PATCH] feat(efficiency): TeamSizeConfig and context caps for token savings Make team-size thresholds config-driven, skip length-based growth at high confidence, and cap collaborative context entry size. Adds tests and Yield-framework pass notes. --- notes/efficiency-pass-2026-08-07.md | 111 ++++++++++++++++++ src/felix_agent_sdk/spawning/__init__.py | 3 +- src/felix_agent_sdk/spawning/optimizer.py | 104 ++++++++++------ src/felix_agent_sdk/spawning/spawner.py | 3 +- .../workflows/context_builder.py | 66 +++++++++-- tests/unit/test_context_builder.py | 73 ++++++++++++ tests/unit/test_team_size_optimizer.py | 16 ++- 7 files changed, 331 insertions(+), 45 deletions(-) create mode 100644 notes/efficiency-pass-2026-08-07.md diff --git a/notes/efficiency-pass-2026-08-07.md b/notes/efficiency-pass-2026-08-07.md new file mode 100644 index 0000000..21f7546 --- /dev/null +++ b/notes/efficiency-pass-2026-08-07.md @@ -0,0 +1,111 @@ +# Felix efficiency pass — 2026-08-07 + +Hard task: research the helix multi-agent concept and iteratively improve efficiency. Yield Framework applied throughout. + +**Actors:** human-directed session + Prime Agent (WSL, `openrouter/deepseek/deepseek-v4-flash`) for exploration; concrete patches landed in-repo after agent stalled on Windows/WSL venv paths. + +--- + +## Concept (research summary) + +Felix models multi-agent work as **motion on a helix**: + +| Idea | Mechanism | +|------|-----------| +| Explore → synthesize | Helix time `t` drives temperature / prompt stance | +| O(N) comms | Hub-spoke via `CentralPost` (not full mesh) | +| Dynamic capacity | `DynamicSpawner` + `ConfidenceMonitor` + `ContentAnalyzer` | +| Context growth | `CollaborativeContextBuilder` scores recency + confidence | +| Token awareness | `TokenBudget.from_helix_position` shifts input/output share with `t` | + +Efficiency bottlenecks (Yield lens): + +1. **Cleverness debt in team sizing** — magic length/confidence thresholds. +2. **Unnecessary agents** — length growth even when confidence is already high. +3. **Context bloat** — unlimited contribution text re-injected each round. +4. **Spawn path work** — coverage analysis after confidence already says HOLD (code already short-circuits; comment clarified). + +--- + +## Yield audit + +### AMPLIFY +- Hub-spoke O(N) design and helix-position token budgets. +- Config-driven `TeamSizeConfig` + high-confidence short-circuit. +- Per-entry context char caps + empty contribution skip. +- Keyword cache on dedupe. + +### DELETE +- None of product surface; removed *reliance* on hardcoded optimizer thresholds as the only API. + +### REPLACE +- Magic numbers in `TeamSizeOptimizer` → `TeamSizeConfig` dataclass. +- Unbounded context payload → truncated `build_context` bodies. + +### DEFER +- Learned team-size policy (RL / bandit) instead of any hand threshold. + ✅ 2026-08-07: `_score_contributions` magic numbers made configurable. +- Streaming context compression tied to live token meters. +- Cross-project Continual Harness isolation for multi-CEM agents. + +--- + +## Changes made + +### 1. `TeamSizeOptimizer` (`spawning/optimizer.py`) +- Added frozen `TeamSizeConfig` with prior defaults. +- **High-confidence path:** if average confidence ≥ `high_confidence_skip_length` (default 0.85), **skip length-based headcount growth** so long prompts do not spawn extra agents when quality is already high. +- Constructor accepts optional `config=`. + +### 2. `CollaborativeContextBuilder` (`workflows/context_builder.py`) +- `max_chars_per_entry` (default 4000) truncates bodies in `build_context`. +- `skip_empty=True` drops blank contributions. +- Keyword cache for faster `deduplicate`. + +### 3. `DynamicSpawner` +- Comment-only clarification of the existing early return before coverage analysis when confidence says HOLD. + +### 4. `CollaborativeContextBuilder` — configurable scoring parameters +- Added `recency_decay_rate`, `recency_max_weight`, `confidence_weight` constructor + parameters to replace hardcoded magic numbers in `_score_contributions`. +- Backward-compatible: defaults (`0.01`, `0.5`, `0.5`) preserve original behaviour + exactly (confirmed by `test_defaults_match_original_behavior`). +- Removes three hardcoded values flagged by Yield audit without breaking API. + +### Tests +- `test_high_confidence_skips_length_growth` +- `test_config_overrides_thresholds` +- `test_skips_empty_content`, `test_truncates_long_content_in_build`, `test_no_truncate_when_disabled` +- `test_custom_recency_decay`, `test_custom_confidence_weight`, `test_defaults_match_original_behavior` + +--- + +## Efficiency impact (reasoned) + +| Change | Expected effect | +|--------|-----------------| +| High-conf skip length growth | Fewer LLM agents on long, already-solved tasks → lower multi-agent token multiply | +| Context char cap | Bounds prompt growth from verbose agents | +| Skip empty | Fewer no-op history rows | +| Keyword cache | Cheaper dedupe on large contribution sets | +| Configurable scoring params | Lets callers tune recency half-life and confidence weight for their task duration, avoiding stale-context or noise problems without forking the builder | + +Not measured end-to-end with live providers in this pass (cost/latency depends on model). + +--- + +## Prime Agent trial notes + +- Install + WSL path works with DeepSeek V4 Flash via OpenRouter. +- Agent successfully: loaded yield skill, scanned repo, read core modules. +- Agent stalled: Windows `.venv` on `/mnt/c` vs WSL (pytest path hell). +- Mitigation: Linux venv at `~/felix-venv` (897 unit tests green baseline); restart prompt forces that interpreter. + +--- + +## Remaining work + +- Wire `TeamSizeConfig` into `WorkflowConfig` / CLI defaults. +- Auto-dedupe each round before `build_context` in runner. +- Optional soft token budget hard-stop in runner when over allocation. +- Prime Agent: document WSL+Windows monorepo venv recipe in Felix README. diff --git a/src/felix_agent_sdk/spawning/__init__.py b/src/felix_agent_sdk/spawning/__init__.py index b6b5307..e088cbe 100644 --- a/src/felix_agent_sdk/spawning/__init__.py +++ b/src/felix_agent_sdk/spawning/__init__.py @@ -9,7 +9,7 @@ SpawnRecommendation, ) from felix_agent_sdk.spawning.content_analyzer import ContentAnalyzer, CoverageReport -from felix_agent_sdk.spawning.optimizer import TeamSizeOptimizer +from felix_agent_sdk.spawning.optimizer import TeamSizeConfig, TeamSizeOptimizer from felix_agent_sdk.spawning.spawner import DynamicSpawner __all__ = [ @@ -18,6 +18,7 @@ "SpawnRecommendation", "ContentAnalyzer", "CoverageReport", + "TeamSizeConfig", "TeamSizeOptimizer", "DynamicSpawner", ] diff --git a/src/felix_agent_sdk/spawning/optimizer.py b/src/felix_agent_sdk/spawning/optimizer.py index d3d6c03..0ca2de7 100644 --- a/src/felix_agent_sdk/spawning/optimizer.py +++ b/src/felix_agent_sdk/spawning/optimizer.py @@ -1,37 +1,63 @@ -"""Team size optimisation heuristic. +"""Team size optimisation. Recommends optimal team size based on task complexity signals and -current result quality. +current result quality. Thresholds are config-driven (not buried magic). """ from __future__ import annotations +from dataclasses import dataclass from typing import Any, Dict, List -# Base team size for a simple task -_BASE_TEAM_SIZE = 3 -# Each complexity signal adds this many agents -_SIGNAL_INCREMENT = 1 +@dataclass(frozen=True) +class TeamSizeConfig: + """Configurable thresholds for :class:`TeamSizeOptimizer`. -# Hard cap to prevent runaway growth -_MAX_TEAM_SIZE = 15 + Defaults preserve historical heuristic behaviour while allowing + callers to tune without forking the class. + """ + + base_size: int = 3 + signal_increment: int = 1 + max_size: int = 15 + length_medium: int = 200 + length_long: int = 500 + conf_low: float = 0.5 + conf_mid: float = 0.7 + conf_default: float = 0.5 + spread_threshold: float = 0.3 + # When average confidence is already this high, skip length-based + # growth — extra agents mostly burn tokens without quality lift. + high_confidence_skip_length: float = 0.85 class TeamSizeOptimizer: - """Heuristic recommender for team size. + """Recommender for team size from complexity + quality signals. - Considers task description length, topic breadth (keyword count), - and current confidence spread to suggest an appropriate team size. + Considers task description length and current confidence spread to + suggest an appropriate team size. High-confidence rounds avoid + inflating headcount for long prompts (token efficiency). Args: min_size: Minimum team size to recommend. max_size: Maximum team size to recommend. + config: Optional threshold config (defaults match prior magic numbers). """ - def __init__(self, min_size: int = 3, max_size: int = _MAX_TEAM_SIZE) -> None: + def __init__( + self, + min_size: int = 3, + max_size: int | None = None, + config: TeamSizeConfig | None = None, + ) -> None: + self._config = config or TeamSizeConfig() self._min_size = min_size - self._max_size = max_size + self._max_size = max_size if max_size is not None else self._config.max_size + + @property + def config(self) -> TeamSizeConfig: + return self._config def recommend_team_size( self, @@ -48,28 +74,38 @@ def recommend_team_size( Returns: Recommended team size clamped to [min_size, max_size]. """ - size = _BASE_TEAM_SIZE - - # Signal 1: long task descriptions suggest complexity - if len(task_description) > 200: - size += _SIGNAL_INCREMENT - if len(task_description) > 500: - size += _SIGNAL_INCREMENT + cfg = self._config + size = cfg.base_size - # Signal 2: low average confidence from existing results + confidences: list[float] = [] if current_results: - confidences = [r.get("confidence", 0.5) for r in current_results] - avg_conf = sum(confidences) / len(confidences) - if avg_conf < 0.5: - size += _SIGNAL_INCREMENT * 2 - elif avg_conf < 0.7: - size += _SIGNAL_INCREMENT - - # Signal 3: wide confidence spread suggests disagreement - if current_results and len(current_results) >= 2: - confidences = [r.get("confidence", 0.5) for r in current_results] - spread = max(confidences) - min(confidences) - if spread > 0.3: - size += _SIGNAL_INCREMENT + confidences = [ + float(r.get("confidence", cfg.conf_default)) for r in current_results + ] + + avg_conf = ( + sum(confidences) / len(confidences) if confidences else None + ) + high_quality = ( + avg_conf is not None and avg_conf >= cfg.high_confidence_skip_length + ) + + # Length signals — skipped when quality is already high (efficiency). + if not high_quality: + if len(task_description) > cfg.length_medium: + size += cfg.signal_increment + if len(task_description) > cfg.length_long: + size += cfg.signal_increment + + if confidences: + if avg_conf is not None and avg_conf < cfg.conf_low: + size += cfg.signal_increment * 2 + elif avg_conf is not None and avg_conf < cfg.conf_mid: + size += cfg.signal_increment + + if len(confidences) >= 2: + spread = max(confidences) - min(confidences) + if spread > cfg.spread_threshold: + size += cfg.signal_increment return max(self._min_size, min(self._max_size, size)) diff --git a/src/felix_agent_sdk/spawning/spawner.py b/src/felix_agent_sdk/spawning/spawner.py index ba7066f..5a58622 100644 --- a/src/felix_agent_sdk/spawning/spawner.py +++ b/src/felix_agent_sdk/spawning/spawner.py @@ -90,7 +90,8 @@ def check_and_spawn( } self._monitor.record_round(confidences) - # Check if spawning is recommended + # Fast path: high confidence → no spawn and no coverage analysis + # (avoids an extra LLM-shaped pass over round contents). if not self._monitor.should_spawn(): return [] diff --git a/src/felix_agent_sdk/workflows/context_builder.py b/src/felix_agent_sdk/workflows/context_builder.py index 5bf4465..9c89125 100644 --- a/src/felix_agent_sdk/workflows/context_builder.py +++ b/src/felix_agent_sdk/workflows/context_builder.py @@ -38,11 +38,36 @@ class CollaborativeContextBuilder: :meth:`add_contribution`. The builder then provides merged context for the next round via :meth:`build_context` and :meth:`get_context_history`. + + Args: + max_chars_per_entry: Hard cap on each contribution's text when + building context strings (token efficiency). ``None`` disables. + skip_empty: Ignore blank/whitespace-only contributions. + recency_decay_rate: Per-second decay applied to contribution age + in relevance scoring (default 0.01 → 50 s half-life). + recency_max_weight: Maximum score contribution from recency + (default 0.5). + confidence_weight: Multiplier for contribution confidence in score + (default 0.5). """ - def __init__(self) -> None: + def __init__( + self, + max_chars_per_entry: int | None = 4000, + skip_empty: bool = True, + recency_decay_rate: float = 0.01, + recency_max_weight: float = 0.5, + confidence_weight: float = 0.5, + ) -> None: self._contributions: list[Contribution] = [] self._version: int = 0 + self._max_chars_per_entry = max_chars_per_entry + self._skip_empty = skip_empty + self._recency_decay_rate = recency_decay_rate + self._recency_max_weight = recency_max_weight + self._confidence_weight = confidence_weight + # Keyword cache for dedupe — avoids re-tokenizing the same text. + self._keyword_cache: dict[int, set[str]] = {} # ------------------------------------------------------------------ # Add / query contributions @@ -66,6 +91,8 @@ def add_contribution( phase: str = "exploration", ) -> None: """Record an agent's output as a contribution.""" + if self._skip_empty and not (content or "").strip(): + return self._contributions.append( Contribution( agent_id=agent_id, @@ -106,10 +133,11 @@ def build_context(self, max_entries: int = 10) -> str: parts: list[str] = [] for contrib, _score in selected: + body = self._truncate(contrib.content) parts.append( f"[{contrib.agent_id} ({contrib.phase}), " f"confidence={contrib.confidence:.2f}]: " - f"{contrib.content}" + f"{body}" ) return "\n\n".join(parts) @@ -150,9 +178,9 @@ def deduplicate(self, similarity_threshold: float = 0.6) -> int: for contrib in self._contributions[1:]: is_dup = False - kw_new = self._extract_keywords(contrib.content) + kw_new = self._keywords_for(contrib) for existing in to_keep: - kw_existing = self._extract_keywords(existing.content) + kw_existing = self._keywords_for(existing) sim = self._jaccard(kw_new, kw_existing) if sim >= similarity_threshold: is_dup = True @@ -163,6 +191,11 @@ def deduplicate(self, similarity_threshold: float = 0.6) -> int: to_keep.append(contrib) self._contributions = to_keep + # Drop keyword entries for removed contributions + kept_ids = {id(c) for c in to_keep} + self._keyword_cache = { + k: v for k, v in self._keyword_cache.items() if k in kept_ids + } return removed # ------------------------------------------------------------------ @@ -177,14 +210,31 @@ def _score_contributions(self) -> list[tuple[Contribution, float]]: now = time.time() scored: list[tuple[Contribution, float]] = [] for contrib in self._contributions: - # Recency: more recent = higher (0.0 – 0.5) + # Recency: more recent = higher (0.0 – recency_max_weight) age = now - contrib.timestamp - recency = max(0.0, 0.5 - age * 0.01) - # Confidence weight (0.0 – 0.5) - conf = contrib.confidence * 0.5 + recency = max(0.0, self._recency_max_weight - age * self._recency_decay_rate) + # Confidence weight (0.0 – confidence_weight) + conf = contrib.confidence * self._confidence_weight scored.append((contrib, recency + conf)) return scored + def _truncate(self, text: str) -> str: + limit = self._max_chars_per_entry + if limit is None or len(text) <= limit: + return text + if limit <= 1: + return text[:limit] + return text[: limit - 1] + "…" + + def _keywords_for(self, contrib: Contribution) -> set[str]: + key = id(contrib) + cached = self._keyword_cache.get(key) + if cached is not None: + return cached + kw = self._extract_keywords(contrib.content) + self._keyword_cache[key] = kw + return kw + @staticmethod def _extract_keywords(text: str) -> set[str]: words = re.findall(r"\b\w{4,}\b", text.lower()) diff --git a/tests/unit/test_context_builder.py b/tests/unit/test_context_builder.py index 8975c4b..b005b02 100644 --- a/tests/unit/test_context_builder.py +++ b/tests/unit/test_context_builder.py @@ -3,6 +3,8 @@ from __future__ import annotations +import time + from felix_agent_sdk.workflows.context_builder import ( CollaborativeContextBuilder, Contribution, @@ -119,3 +121,74 @@ def test_no_duplicates(self): def test_empty_dedup(self): builder = CollaborativeContextBuilder() assert builder.deduplicate() == 0 + +class TestContextEfficiency: + def test_skips_empty_content(self): + builder = CollaborativeContextBuilder() + builder.add_contribution("a1", "research", " ", 0.7, "exploration") + builder.add_contribution("a2", "research", "real findings", 0.8, "exploration") + assert builder.contribution_count == 1 + + def test_truncates_long_content_in_build(self): + builder = CollaborativeContextBuilder(max_chars_per_entry=50) + builder.add_contribution("a1", "research", "x" * 200, 0.9, "exploration") + ctx = builder.build_context() + # header + truncated body; body should not retain all 200 chars + assert "x" * 200 not in ctx + assert "…" in ctx or len(ctx) < 200 + + def test_no_truncate_when_disabled(self): + builder = CollaborativeContextBuilder(max_chars_per_entry=None) + body = "y" * 120 + builder.add_contribution("a1", "research", body, 0.9, "exploration") + ctx = builder.build_context() + assert body in ctx + + +class TestScoringConfig: + """Configurable recency / confidence scoring parameters.""" + + def test_custom_recency_decay(self): + """Faster decay means older contributions score lower.""" + slow = CollaborativeContextBuilder(recency_decay_rate=0.01) + fast = CollaborativeContextBuilder(recency_decay_rate=1.0) + + now = time.time() + old = Contribution("a1", "research", "old", 0.5, "exploration", timestamp=now - 30) + new = Contribution("a2", "research", "new", 0.5, "exploration", timestamp=now) + + def recency_score(builder, c): + age = time.time() - c.timestamp + return max(0.0, builder._recency_max_weight - age * builder._recency_decay_rate) + + assert recency_score(slow, old) > 0.0 # slow decay keeps old score + assert recency_score(fast, old) == 0.0 # fast decay zeros old + assert recency_score(fast, new) > 0.0 # even fast decay keeps new + + def test_custom_confidence_weight(self): + """Higher weight amplifies confidence in combined score.""" + low = CollaborativeContextBuilder(confidence_weight=0.3) + high = CollaborativeContextBuilder(confidence_weight=0.9) + low.add_contribution("a1", "research", "data", 0.8, "exploration") + high.add_contribution("a1", "research", "data", 0.8, "exploration") + + # Both produce context (score > 0) + assert len(low.build_context()) > 0 + assert len(high.build_context()) > 0 + + # The higher-weight builder gives a strictly higher score for same input + low_score = low._score_contributions()[0][1] + high_score = high._score_contributions()[0][1] + assert high_score > low_score + + def test_defaults_match_original_behavior(self): + """Default parameter values preserve original magic-number behaviour.""" + default = CollaborativeContextBuilder() + now = time.time() + c = Contribution("a1", "r", "content", 0.7, "exploration", timestamp=now) + age = time.time() - c.timestamp + expected_recency = max(0.0, 0.5 - age * 0.01) + expected = expected_recency + 0.7 * 0.5 + default._contributions.append(c) + score = default._score_contributions()[0][1] + assert abs(score - expected) < 0.001 diff --git a/tests/unit/test_team_size_optimizer.py b/tests/unit/test_team_size_optimizer.py index f6d5eb2..c7ad739 100644 --- a/tests/unit/test_team_size_optimizer.py +++ b/tests/unit/test_team_size_optimizer.py @@ -2,7 +2,7 @@ from __future__ import annotations -from felix_agent_sdk.spawning.optimizer import TeamSizeOptimizer +from felix_agent_sdk.spawning.optimizer import TeamSizeConfig, TeamSizeOptimizer class TestTeamSizeOptimizer: @@ -35,6 +35,20 @@ def test_high_confidence_keeps_base(self): size = opt.recommend_team_size("Short", results) assert size == 3 + def test_high_confidence_skips_length_growth(self): + """Token efficiency: high-quality rounds should not grow for long prompts.""" + opt = TeamSizeOptimizer() + results = [{"confidence": 0.95, "content": "solid"} for _ in range(3)] + size = opt.recommend_team_size("x" * 600, results) + assert size == 3 + + def test_config_overrides_thresholds(self): + cfg = TeamSizeConfig(base_size=2, length_medium=10, length_long=20) + opt = TeamSizeOptimizer(min_size=1, config=cfg) + assert opt.recommend_team_size("short") == 2 + assert opt.recommend_team_size("x" * 15) == 3 + assert opt.recommend_team_size("x" * 25) == 4 + def test_wide_spread_increases_size(self): opt = TeamSizeOptimizer() results = [