fix(eval): keep leftover processes through framework timeout

.github/workflows/ci.yml

@@ -192,6 +192,19 @@ jobs:
           fi
           node scripts/run-workspace-tests-parallel.mjs --concurrency=3 --workspaces="$WORKSPACES"
 
+      - name: Install Harbor lifecycle test dependencies
+        if: contains(steps.plan.outputs.standard_workspaces, 'packages/eval')
+        run: |
+          python3 -m venv "$RUNNER_TEMP/maka-harbor-lifecycle"
+          "$RUNNER_TEMP/maka-harbor-lifecycle/bin/python" -m pip install --disable-pip-version-check 'harbor==0.20.0'
+
+      - name: Run real Harbor lifecycle tests
+        if: contains(steps.plan.outputs.standard_workspaces, 'packages/eval')
+        env:
+          MAKA_EVAL_HARBOR_LIFECYCLE_TEST: '1'
+        run: |
+          "$RUNNER_TEMP/maka-harbor-lifecycle/bin/python" packages/eval/harbor/test_harbor_trial_lifecycle.py
+
       - name: Run Runtime Host tests
         if: steps.plan.outputs.runtime_host == 'true'
         run: npm --workspace @maka/runtime-host run test:dist

packages/cli/src/__tests__/runtime-host-operator-command.test.ts

@@ -209,6 +209,7 @@ describe('Runtime Host operator commands', () => {
         'access.credential.rotation.prepare',
         'access.credential.rotation.revoke',
         'host.upgrade.prepare',
+        'hosted.execution.admit',
         'hosted.execution.cancel',
         'hosted.execution.start',
       ],

packages/eval/harbor/run_trial.py

@@ -24,6 +24,7 @@
 import importlib
 import importlib.metadata
 import json
+import math
 import os
 import signal
 import sys
@@ -162,10 +163,43 @@ def apply_subject_egress_policy(task: object) -> None:
     agent.allowed_hosts = [allowed_host]
 
 
+def apply_framework_timeout_budget(config: object, task: object) -> None:
+    task_config = task.config
+    defaults = (
+        [
+            step.agent.timeout_sec
+            if step.agent.timeout_sec is not None
+            else task_config.agent.timeout_sec
+            for step in task_config.steps
+        ]
+        if task_config.steps
+        else [task_config.agent.timeout_sec]
+    )
+    multiplier = (
+        config.agent_timeout_multiplier
+        if config.agent_timeout_multiplier is not None
+        else config.timeout_multiplier
+    )
+    timeouts = []
+    for default_timeout in defaults:
+        base_timeout = config.agent.override_timeout_sec or default_timeout
+        if base_timeout is None:
+            continue
+        maximum = config.agent.max_timeout_sec or float("inf")
+        timeouts.append(min(base_timeout, maximum) * multiplier)
+    if not timeouts:
+        return
+    timeout_ms = math.floor(min(timeouts) * 1000)
+    if timeout_ms <= 0:
+        raise RuntimeError("Harbor agent timeout is shorter than one millisecond")
+    config.agent.kwargs["framework_timeout_ms"] = timeout_ms
+
+
 async def create_harbor_trial(trial_type: type, config: object) -> object:
     trial_type._resolve_agent_skills(config)
     task, task_download_result = await trial_type._load_task(config)
     apply_subject_egress_policy(task)
+    apply_framework_timeout_budget(config, task)
     if task.has_steps:
         from harbor.trial.multi_step import MultiStepTrial