Fix #614: verify daemon lifecycle and stale server state

tests/integration/test_cli.py

@@ -83,6 +83,12 @@ def _isolate_real_world_side_effects(tmp_path_factory, monkeypatch):
     monkeypatch.setattr(
         "tokenjam.cli.cmd_onboard._stop_serve_for_db_write", lambda: False
     )
+    # Doctor's duplicate-instance check is intentionally machine-wide. Keep
+    # this integration file hermetic instead of letting a developer's real
+    # foreground/managed serve processes change its exit-code assertions.
+    monkeypatch.setattr(
+        "tokenjam.core.server_state.list_serve_processes", lambda: []
+    )
     # Storage paths resolve via os.path.expanduser($HOME); redirect them at the
     # default ``~/.tj/telemetry.duckdb`` so any stray open_db lands in tmp.
     monkeypatch.setenv("HOME", str(iso))

tests/unit/test_doctor_daemon_lifecycle.py

@@ -0,0 +1,135 @@
+"""Doctor coverage for the daemon lifecycle gaps reported in #614."""
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from tokenjam.cli.cmd_doctor import _check_daemon_lifecycle
+from tokenjam.core.config import ApiConfig, TjConfig
+from tokenjam.core.server_state import DaemonUnitState, ServeProcess
+
+
+def _config() -> TjConfig:
+    return TjConfig(version="1", api=ApiConfig(port=7391))
+
+
+def _patch_state_path(monkeypatch, path) -> None:
+    monkeypatch.setattr("tokenjam.core.server_state.server_state_path", lambda: path)
+
+
+def test_doctor_warns_when_launchd_plist_exists_but_is_not_loaded(
+    tmp_path, monkeypatch,
+):
+    plist = tmp_path / "Library/LaunchAgents/com.tokenjam.serve.plist"
+    plist.parent.mkdir(parents=True)
+    plist.write_text("<plist/>")
+    monkeypatch.setattr(
+        "tokenjam.core.server_state.inspect_daemon_unit",
+        lambda: DaemonUnitState("launchd", plist, True, False, None),
+    )
+    monkeypatch.setattr("tokenjam.core.server_state.list_serve_processes", lambda: [])
+    _patch_state_path(monkeypatch, tmp_path / "missing.state")
+
+    checks = _check_daemon_lifecycle(_config())
+    service = next(check for check in checks if check["name"] == "Daemon service")
+
+    assert service["level"] == "warning"
+    assert "not loaded" in service["message"]
+    assert "tj onboard" in service["message"]
+
+
+def test_doctor_reports_multiple_live_serve_instances(tmp_path, monkeypatch):
+    monkeypatch.setattr(
+        "tokenjam.core.server_state.inspect_daemon_unit",
+        lambda: DaemonUnitState(None, None, False, None, None),
+    )
+    monkeypatch.setattr(
+        "tokenjam.core.server_state.list_serve_processes",
+        lambda: [
+            ServeProcess(111, "/usr/local/bin/tj serve"),
+            ServeProcess(222, "/opt/venv/bin/tj serve --port 9341"),
+        ],
+    )
+    _patch_state_path(monkeypatch, tmp_path / "missing.state")
+
+    checks = _check_daemon_lifecycle(_config())
+    instances = next(check for check in checks if check["name"] == "Daemon instances")
+
+    assert instances["level"] == "warning"
+    assert "111" in instances["message"]
+    assert "222" in instances["message"]
+
+
+def test_doctor_reports_server_state_pointing_at_a_dead_pid(tmp_path, monkeypatch):
+    state_path = tmp_path / "server.state"
+    state_path.write_text(json.dumps({"pid": 333, "port": 8123, "config_path": None}))
+    monkeypatch.setattr(
+        "tokenjam.core.server_state.inspect_daemon_unit",
+        lambda: DaemonUnitState(None, None, False, None, None),
+    )
+    monkeypatch.setattr("tokenjam.core.server_state.list_serve_processes", lambda: [])
+    monkeypatch.setattr("tokenjam.core.server_state.is_pid_alive", lambda pid: False)
+    _patch_state_path(monkeypatch, state_path)
+
+    checks = _check_daemon_lifecycle(_config())
+    state = next(check for check in checks if check["name"] == "Server state")
+
+    assert state["level"] == "warning"
+    assert "dead PID 333" in state["message"]
+    assert "port 8123" in state["message"]
+
+
+def test_doctor_reports_live_state_on_the_wrong_port(tmp_path, monkeypatch):
+    state_path = tmp_path / "server.state"
+    state_path.write_text(json.dumps({"pid": 444, "port": 9341, "config_path": None}))
+    monkeypatch.setattr(
+        "tokenjam.core.server_state.inspect_daemon_unit",
+        lambda: DaemonUnitState(None, None, False, None, None),
+    )
+    monkeypatch.setattr(
+        "tokenjam.core.server_state.list_serve_processes",
+        lambda: [ServeProcess(444, "/opt/venv/bin/tj serve --port 9341")],
+    )
+    monkeypatch.setattr("tokenjam.core.server_state.is_pid_alive", lambda pid: True)
+    monkeypatch.setattr("tokenjam.core.server_state.is_serve_process", lambda pid: True)
+    _patch_state_path(monkeypatch, state_path)
+
+    checks = _check_daemon_lifecycle(_config())
+    state = next(check for check in checks if check["name"] == "Server state")
+
+    assert state["level"] == "warning"
+    assert "port 9341" in state["message"]
+    assert "expects 7391" in state["message"]
+
+
+def test_doctor_reports_live_state_under_a_different_config(tmp_path, monkeypatch):
+    """A same-port daemon running under a DIFFERENT config file is not
+    healthy — it validates the wrong database and ingest secret, and port
+    matching alone can't see this (two configs can legitimately share a port
+    across separate installs/worktrees)."""
+    state_path = tmp_path / "server.state"
+    state_path.write_text(json.dumps({
+        "pid": 555, "port": 7391, "config_path": "/other/project/.tj/config.toml",
+    }))
+    monkeypatch.setattr(
+        "tokenjam.core.server_state.inspect_daemon_unit",
+        lambda: DaemonUnitState(None, None, False, None, None),
+    )
+    monkeypatch.setattr(
+        "tokenjam.core.server_state.list_serve_processes",
+        lambda: [ServeProcess(555, "/opt/venv/bin/tj serve")],
+    )
+    monkeypatch.setattr("tokenjam.core.server_state.is_pid_alive", lambda pid: True)
+    monkeypatch.setattr("tokenjam.core.server_state.is_serve_process", lambda pid: True)
+    _patch_state_path(monkeypatch, state_path)
+
+    config = TjConfig(
+        version="1", api=ApiConfig(port=7391),
+        config_path=Path("/this/project/.tj/config.toml"),
+    )
+    checks = _check_daemon_lifecycle(config)
+    state = next(check for check in checks if check["name"] == "Server state")
+
+    assert state["level"] == "warning"
+    assert "/other/project/.tj/config.toml" in state["message"]
+    assert "/this/project/.tj/config.toml" in state["message"]

tests/unit/test_onboard_daemon.py

@@ -332,6 +332,24 @@ def _run(cmd, **kwargs):
 
         assert result is None
 
+class TestSystemdInstallVerification:
+    def test_written_but_disabled_unit_is_detected(self, tmp_path, monkeypatch, capsys):
+        from tokenjam.cli.cmd_onboard import _install_systemd
+
+        monkeypatch.setattr("tokenjam.cli.cmd_onboard.Path.home", lambda: tmp_path)
+        monkeypatch.setattr("tokenjam.cli.cmd_onboard.shutil.which", lambda _: "/usr/bin/tj")
+        results = iter([
+            MagicMock(returncode=0),  # enable --now accepted
+            MagicMock(returncode=1, stdout="disabled\n"),
+        ])
+
+        with patch("tokenjam.cli.cmd_onboard.subprocess.run", side_effect=results):
+            installed = _install_systemd("/tmp/cfg.toml")
+
+        assert installed is None
+        assert (tmp_path / ".config/systemd/user/tokenjam.service").exists()
+        assert "did not enable tokenjam" in capsys.readouterr().out
+
 
 class TestTjBinaryResolution:
     """The daemon installers must point launchd/systemd at a real `tj` binary.

tests/unit/test_server_state.py

@@ -11,7 +11,13 @@
 """
 from __future__ import annotations
 
-from tokenjam.core.server_state import _looks_like_serve
+import subprocess
+
+from tokenjam.core.server_state import (
+    _looks_like_serve,
+    inspect_daemon_unit,
+    list_serve_processes,
+)
 
 
 class TestMatchesRealDaemonInvocations:
@@ -54,3 +60,85 @@ def test_does_not_match_tj_as_substring_of_longer_token(self):
         # "tj" must be a bare token (or a path basename), not a substring
         # of some unrelated word.
         assert _looks_like_serve("/usr/bin/notjserve --serve") is False
+
+
+class TestDaemonUnitInspection:
+    def test_written_launchd_plist_is_not_mistaken_for_a_loaded_job(
+        self, tmp_path, monkeypatch,
+    ):
+        plist = tmp_path / "Library/LaunchAgents/com.tokenjam.serve.plist"
+        plist.parent.mkdir(parents=True)
+        plist.write_text("<plist/>")
+        monkeypatch.setattr(
+            "tokenjam.core.server_state.subprocess.run",
+            lambda *args, **kwargs: subprocess.CompletedProcess(args[0], 3, "", "not found"),
+        )
+
+        state = inspect_daemon_unit(system="Darwin", home=tmp_path)
+
+        assert state.installed is True
+        assert state.loaded is False
+        assert state.manager == "launchd"
+
+    def test_systemd_reports_enabled_and_active_separately(self, tmp_path, monkeypatch):
+        unit = tmp_path / ".config/systemd/user/tokenjam.service"
+        unit.parent.mkdir(parents=True)
+        unit.write_text("[Service]\n")
+
+        def _run(argv, **kwargs):
+            if "is-enabled" in argv:
+                return subprocess.CompletedProcess(argv, 0, "enabled\n", "")
+            return subprocess.CompletedProcess(argv, 0, "active\n", "")
+
+        monkeypatch.setattr("tokenjam.core.server_state.subprocess.run", _run)
+        state = inspect_daemon_unit(system="Linux", home=tmp_path)
+
+        assert state.loaded is True
+        assert state.active is True
+
+    def test_systemd_linked_or_aliased_unit_does_not_count_as_enabled(
+        self, tmp_path, monkeypatch,
+    ):
+        """`linked`/`linked-runtime`/`alias` are unit states `is-enabled`
+        reports successfully (returncode 0) for a unit that is registered but
+        NOT attached to the login target — it will not autostart at the next
+        login. Reporting these as enabled would tell the user something
+        untrue."""
+        unit = tmp_path / ".config/systemd/user/tokenjam.service"
+        unit.parent.mkdir(parents=True)
+        unit.write_text("[Service]\n")
+
+        def _run(argv, **kwargs):
+            if "is-enabled" in argv:
+                return subprocess.CompletedProcess(argv, 0, "linked\n", "")
+            return subprocess.CompletedProcess(argv, 0, "active\n", "")
+
+        monkeypatch.setattr("tokenjam.core.server_state.subprocess.run", _run)
+        state = inspect_daemon_unit(system="Linux", home=tmp_path)
+
+        assert state.loaded is False
+
+
+class TestServeProcessInventory:
+    def test_lists_every_tj_serve_instance_and_ignores_unrelated_processes(self, monkeypatch):
+        output = (
+            " 101 /usr/local/bin/tj --config /tmp/a.toml serve\n"
+            " 202 /opt/venv/bin/tj serve --port 9341\n"
+            " 303 python manage.py serve\n"
+        )
+        monkeypatch.setattr(
+            "tokenjam.core.server_state.subprocess.run",
+            lambda *args, **kwargs: subprocess.CompletedProcess(args[0], 0, output, ""),
+        )
+
+        processes = list_serve_processes()
+
+        assert processes is not None
+        assert [process.pid for process in processes] == [101, 202]
+
+    def test_process_inventory_is_unknown_when_ps_cannot_run(self, monkeypatch):
+        def _denied(*args, **kwargs):
+            raise PermissionError("ps denied")
+
+        monkeypatch.setattr("tokenjam.core.server_state.subprocess.run", _denied)
+        assert list_serve_processes() is None