diff --git a/.github/workflows/build-app.yml b/.github/workflows/build-app.yml new file mode 100644 index 0000000..eac64e7 --- /dev/null +++ b/.github/workflows/build-app.yml @@ -0,0 +1,73 @@ +name: Build app + +# Builds the native ATLAS desktop app (Tauri shell + frozen FastAPI sidecar) for +# macOS and Windows. Runs on a version tag (release) or manual dispatch. Signing + +# notarization activate only when the corresponding secrets are present, so forks +# and unsigned test builds still succeed. +# +# Required secrets for signed macOS builds (Settings → Secrets → Actions): +# APPLE_CERTIFICATE (base64 .p12), APPLE_CERTIFICATE_PASSWORD, APPLE_SIGNING_IDENTITY, +# APPLE_ID, APPLE_PASSWORD (app-specific), APPLE_TEAM_ID +# Windows code-signing (beat SmartScreen) is added later via Azure Key Vault. + +on: + push: + tags: ["v*"] + workflow_dispatch: + +jobs: + build: + strategy: + fail-fast: false + matrix: + include: + - os: macos-14 # Apple Silicon + triple: aarch64-apple-darwin + - os: windows-latest + triple: x86_64-pc-windows-msvc + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.triple }} + + - name: Python deps + PyInstaller + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt pyinstaller + + - name: Freeze the FastAPI core (sidecar) + run: pyinstaller --clean -y deploy/atlas-core.spec + + - name: Name the sidecar with its target triple + shell: bash + run: | + mkdir -p src-tauri/binaries + if [ "${{ runner.os }}" = "Windows" ]; then + cp dist/atlas-core.exe "src-tauri/binaries/atlas-core-${{ matrix.triple }}.exe" + else + cp dist/atlas-core "src-tauri/binaries/atlas-core-${{ matrix.triple }}" + chmod +x "src-tauri/binaries/atlas-core-${{ matrix.triple }}" + fi + + - name: Build the app (tauri-action; signs when secrets present) + uses: tauri-apps/tauri-action@v0 + env: + APPLE_CERTIFICATE: ${{ secrets.APPLE_CERTIFICATE }} + APPLE_CERTIFICATE_PASSWORD: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }} + APPLE_SIGNING_IDENTITY: ${{ secrets.APPLE_SIGNING_IDENTITY }} + APPLE_ID: ${{ secrets.APPLE_ID }} + APPLE_PASSWORD: ${{ secrets.APPLE_PASSWORD }} + APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }} + with: + projectPath: src-tauri + tagName: ${{ github.ref_type == 'tag' && github.ref_name || '' }} + releaseName: "ATLAS ${{ github.ref_name }}" + releaseDraft: true diff --git a/.gitignore b/.gitignore index 14cf090..0658c5d 100644 --- a/.gitignore +++ b/.gitignore @@ -48,3 +48,9 @@ atlas/memory/vault/people/*.md # L8 Scheduler runtime state (last_run / next_run per job) atlas/scheduler/state.json + +# Tauri app build artifacts (PyInstaller freeze + Tauri bundle) +/build/ +/dist/ +src-tauri/target/ +src-tauri/binaries/ diff --git a/atlas/VERSION.json b/atlas/VERSION.json index d77e63c..62b0190 100644 --- a/atlas/VERSION.json +++ b/atlas/VERSION.json @@ -1,6 +1,6 @@ { - "version": "0.2.0", - "codename": "ATLAS — Automated Task & Logic Assistant System", + "version": "0.6.0", + "codename": "ATLAS \u2014 Automated Task & Logic Assistant System", "cycle": 1, "phase": 1, "layers": { @@ -14,7 +14,7 @@ "L8_scheduler": "1.0.0" }, "model_backend": "claude-sonnet-4-6", - "model_backend_notes": "Phase 1 — Claude API is the brain. Sonnet for routine turns; escalates to Opus for heavy reasoning per atlas/orchestration/config.json. Local wake word, STT, TTS, memory, and UI run on-device.", + "model_backend_notes": "Phase 1 \u2014 Claude API is the brain. Sonnet for routine turns; escalates to Opus for heavy reasoning per atlas/orchestration/config.json. Local wake word, STT, TTS, memory, and UI run on-device.", "last_upgrade": "2026-06-13T08:55:00Z", "next_scheduled_cycle": "2026-06-14T03:00:00Z", "rollback_point": "git tag v0.1.0 (commit aed71b6)", @@ -22,7 +22,7 @@ "status": "L8_scheduler_added", "owner_approval_pending": [ "L1 identity personality sign-off", - "Identity canaries — replace placeholder answers with real Owner facts", - "L7 first 3 connectors — confirm calendar/email/web and authorize install" + "Identity canaries \u2014 replace placeholder answers with real Owner facts", + "L7 first 3 connectors \u2014 confirm calendar/email/web and authorize install" ] } diff --git a/atlas/config/settings.example.json b/atlas/config/settings.example.json index 5510fde..7ee7bb8 100644 --- a/atlas/config/settings.example.json +++ b/atlas/config/settings.example.json @@ -20,7 +20,9 @@ "tts_voice": "kokoro:bm_george", "speech_rate": 1.0, "streaming_tts": true, - "mic_mute": false + "mic_mute": false, + "native_audio": false, + "aec_backend": "os" }, "model": { "mode": "api", diff --git a/atlas/connectors/web_search.py b/atlas/connectors/web_search.py index 3f9f2f0..1f38858 100644 --- a/atlas/connectors/web_search.py +++ b/atlas/connectors/web_search.py @@ -133,9 +133,11 @@ def rich(query: str, *, max_results: int | None = None) -> dict[str, Any]: try: rows = _rows(query, n, web) text = _format(rows, query) + # Only fetch images when the text search produced results — otherwise the + # HUD would show orphaned photos with no article to open. + images = _images(query, max(n + 1, 6)) if rows else [] except Exception as exc: - rows, text = [], f"Web search failed ({type(exc).__name__})." - images = _images(query, max(n + 1, 6)) + rows, text, images = [], f"Web search failed ({type(exc).__name__}).", [] # Numbered article results (the Owner can say "open article 2"): each real # web result gets a thumbnail (best-effort, zipped by index) for the HUD. results = [] diff --git a/atlas/interface/voice/aec.py b/atlas/interface/voice/aec.py new file mode 100644 index 0000000..029854f --- /dev/null +++ b/atlas/interface/voice/aec.py @@ -0,0 +1,110 @@ +"""Acoustic echo cancellation (L3, Phase 3) — remove the far-end (Kokoro TTS) +signal from the mic so ATLAS can listen WHILE it speaks without hearing itself. + +Backends (voice.aec_backend): + "os" — macOS VoiceProcessingIO (best; the OS owns mic+speaker and cancels + echo for free). Needs a native audio-unit bridge; when unavailable we + fall back to the software filter and say so. + "webrtc"— WebRTC AEC3 via webrtc-audio-processing (needs a C++ build). + "nlms" — a pure-numpy frequency-domain adaptive filter (FDAF). No build deps, + deterministic, unit-testable. Handles linear echo; not as strong as + the OS/WebRTC cancellers on nonlinear echo, but always available. + "none" — passthrough (Phase 2 half-duplex). + +The reference (far-end) frame is the exact Kokoro PCM being played, handed in +time-aligned by the audio service — that's what makes real cancellation possible. +""" +from __future__ import annotations + +import numpy as np + +FRAME = 1280 # 80 ms @ 16 kHz + + +class EchoCanceller: + """Constrained frequency-domain adaptive filter (overlap-save FDAF), single + partition of `block` samples. process(mic, ref) → near-end (echo removed).""" + + def __init__(self, block: int = FRAME, mu: float = 0.3, eps: float = 1e-3) -> None: + self.N = block + self.mu = float(mu) + self.eps = float(eps) + # Constrained overlap-save FDAF, single partition. Full complex FFT of size + # 2N; the filter W and smoothed power P live in that 2N frequency domain. + self._W = np.zeros(2 * block, dtype=np.complex128) + self._x_prev = np.zeros(block, dtype=np.float64) # previous ref block + self._P = np.ones(2 * block, dtype=np.float64) # smoothed ref power + + def process(self, mic_i16, ref_i16): + N = self.N + d = np.asarray(mic_i16, dtype=np.float64)[:N] / 32768.0 + x_cur = np.asarray(ref_i16, dtype=np.float64)[:N] / 32768.0 + if d.size < N: + d = np.pad(d, (0, N - d.size)) + if x_cur.size < N: + x_cur = np.pad(x_cur, (0, N - x_cur.size)) + + x = np.concatenate([self._x_prev, x_cur]) # 2N overlap-save block + X = np.fft.fft(x) # 2N complex + y = np.real(np.fft.ifft(self._W * X))[N:] # linear-conv echo estimate + e = d - y # near-end estimate + + E = np.fft.fft(np.concatenate([np.zeros(N), e])) # 2N + self._P = 0.9 * self._P + 0.1 * (np.abs(X) ** 2) + G = np.conj(X) * E / (self._P + self.eps) # per-bin normalized grad + g = np.real(np.fft.ifft(G)) + g[N:] = 0 # gradient constraint (first N taps) + self._W += self.mu * np.fft.fft(g) + + self._x_prev = x_cur + return np.clip(e * 32768.0, -32768, 32767).astype(np.int16) + + +class _Passthrough: + backend = "none" + + def process(self, mic_i16, ref_i16): + return np.asarray(mic_i16, dtype=np.int16) + + +class _Software: + backend = "nlms" + + def __init__(self): + self._ec = EchoCanceller() + + def process(self, mic_i16, ref_i16): + return self._ec.process(mic_i16, ref_i16) + + +def _os_backend(): + """macOS VoiceProcessingIO. Requires a native audio-unit bridge (PyObjC/ctypes + or a small Rust helper) that we don't build here yet — returns None so the + caller falls back to the software filter. The hook is intentional: wiring the + VPIO unit is the on-device upgrade for best-in-class AEC.""" + return None + + +def make_aec(backend: str = "os"): + """Build an echo canceller for the requested backend, falling back gracefully. + Returns an object with .process(mic_i16, ref_i16) and a .backend label.""" + backend = (backend or "os").lower() + if backend in ("none", "off"): + return _Passthrough() + if backend == "os": + vpio = _os_backend() + if vpio is not None: + return vpio + sw = _Software() # VPIO bridge not built → software fallback + sw.backend = "nlms(os-unavailable)" + return sw + if backend == "webrtc": + try: + import webrtc_audio_processing # noqa: F401 + # (wiring left for the C++-built environment; software fallback here) + except Exception: + pass + sw = _Software() + sw.backend = "nlms(webrtc-unavailable)" + return sw + return _Software() diff --git a/atlas/interface/voice/audio_service.py b/atlas/interface/voice/audio_service.py new file mode 100644 index 0000000..1aa6ae5 --- /dev/null +++ b/atlas/interface/voice/audio_service.py @@ -0,0 +1,712 @@ +"""L3 native audio service — mic → wake → VAD → STT → Router → Kokoro, streaming +state to the HUD over /voice/ws. Replaces the browser Web Speech loop with a +fully on-device, offline pipeline (openWakeWord + whisper.cpp + Kokoro). + +Phase 2 is HALF-DUPLEX: the mic is ignored while ATLAS speaks (barge-in falls back +to the UI Stop). Phase 3 swaps in echo-cancelled full-duplex barge-in. + +Every heavy dep (sounddevice, openwakeword, onnxruntime, pywhispercpp) is OPTIONAL: +if any is missing, available() is False, the service never starts, and the HUD +keeps its browser Web Speech fallback. The pipeline stages are INJECTABLE +(wake_fn / stt_fn / synth_fn / play_fn / publish), so the whole state machine is +unit-tested with fakes — no microphone or models required. +""" +from __future__ import annotations + +import queue +import re +import threading +from typing import Any, Callable + +SAMPLE_RATE = 16000 # openWakeWord + whisper both want 16 kHz mono +FRAME = 1280 # 80 ms @ 16 kHz — the openWakeWord frame size +_MAX_UTTERANCE_S = 15 # hard cap so a stuck VAD can't record forever + + +class _Playback: + """One persistent output stream reused across sentences. + + Per-sentence ``sd.play()/sd.wait()`` opened and tore down a fresh OutputStream + for every sentence; that churn (plus the synthesis gap between sentences) made + ATLAS's speech choppy and glitchy. A single long-lived stream written + sentence-by-sentence is smooth. ``stop()`` aborts immediately so barge-in / the + UI Stop cut playback without waiting for the buffer to drain. + """ + + def __init__(self) -> None: + self._stream = None + self._sr = None + self._abort = threading.Event() + self._lock = threading.Lock() + + def _ensure(self, sr: int): + import sounddevice as sd + if self._stream is not None and self._sr == sr and not self._stream.active: + try: + self._stream.start() # re-arm after a prior abort() + except Exception: + try: self._stream.close() + except Exception: pass + self._stream = None + if self._stream is None or self._sr != sr: + if self._stream is not None: + try: self._stream.stop(); self._stream.close() + except Exception: pass + self._stream = sd.OutputStream(samplerate=int(sr), channels=1, dtype="float32") + self._stream.start() + self._sr = int(sr) + return self._stream + + def play(self, samples, sr: int, gain: float = 1.0) -> None: + import numpy as np + self._abort.clear() + data = np.asarray(samples, dtype=np.float32).reshape(-1, 1) + if gain != 1.0: + data = data * float(gain) + with self._lock: + try: + stream = self._ensure(sr) + except Exception: + return + block = 2048 # ~85 ms @ 24 kHz → snappy abort checks + for i in range(0, len(data), block): + if self._abort.is_set(): + break + try: + stream.write(data[i:i + block]) + except Exception: + break + + def stop(self) -> None: + """Interrupt playback now — safe to call from another thread (barge/Stop).""" + self._abort.set() + try: + if self._stream is not None: + self._stream.abort() # drop buffered frames immediately + except Exception: + pass + + def close(self) -> None: + self._abort.set() + with self._lock: + try: + if self._stream is not None: + self._stream.stop(); self._stream.close() + except Exception: + pass + self._stream = None + self._sr = None + + +_PLAYBACK = _Playback() + + +def _stop_playback() -> None: + """Interrupt any in-progress native playback (barge-in / user Stop).""" + _PLAYBACK.stop() + + +# --------------------------------------------------------------------------- # +# Dependency / model availability +# --------------------------------------------------------------------------- # +def deps_present() -> bool: + try: + import numpy, sounddevice, openwakeword, pywhispercpp # noqa: F401 + return True + except Exception: + return False + + +def wake_model_path(name: str = "hey_jarvis") -> str | None: + try: + import glob + import os + # Prefer a repo-local custom model (e.g. our "hey_atlas" model, trained + # and committed under voice/models/) so it survives openWakeWord reinstalls + # and travels with the source tree. Fall back to openWakeWord's bundled set. + local_dir = os.path.join(os.path.dirname(__file__), "models") + hits = glob.glob(os.path.join(local_dir, f"{name}*.onnx")) + if hits: + return hits[0] + import openwakeword + d = os.path.join(os.path.dirname(openwakeword.__file__), "resources", "models") + hits = glob.glob(os.path.join(d, f"{name}*.onnx")) + return hits[0] if hits else None + except Exception: + return None + + +def available() -> bool: + """True if the native pipeline can run (deps + a wake model present).""" + return deps_present() and wake_model_path() is not None + + +# --------------------------------------------------------------------------- # +# Pub-sub bridge: audio thread → async WebSocket clients +# --------------------------------------------------------------------------- # +class VoiceHub: + """Thread-safe fan-out from the audio worker thread to /voice/ws clients. + The FastAPI lifespan binds the running loop; publish() is called off-thread.""" + + def __init__(self) -> None: + self._loop = None + self._queues: set = set() + + def bind_loop(self, loop) -> None: + self._loop = loop + + def subscribe(self): + import asyncio + q: asyncio.Queue = asyncio.Queue() + self._queues.add(q) + return q + + def unsubscribe(self, q) -> None: + self._queues.discard(q) + + def publish(self, event: dict) -> None: + loop = self._loop + if loop is None: + return + for q in list(self._queues): + try: + loop.call_soon_threadsafe(q.put_nowait, event) + except Exception: + pass + + +# --------------------------------------------------------------------------- # +# Speech text cleanup (mirror of the HUD's speechClean, so TTS never reads markup) +# --------------------------------------------------------------------------- # +def speech_clean(s: str) -> str: + if not s: + return "" + s = re.sub(r"```[\s\S]*?```", " code shown on screen ", s) + s = re.sub(r"【[^】]*】", "", s) + s = re.sub(r"\[\^?\d+\]", "", s) + s = re.sub(r"!?\[([^\]]+)\]\([^)]+\)", r"\1", s) + s = re.sub(r"\bhttps?://\S+", "", s) + s = re.sub(r"^\s{0,3}#{1,6}\s+", "", s, flags=re.M) + s = re.sub(r"[*_#`>~|]", " ", s) + s = re.sub(r"\s*&\s*", " and ", s) + s = re.sub(r"\n{2,}", ". ", s).replace("\n", ", ") + s = re.sub(r"\s+([.,!?;:])", r"\1", s) + return re.sub(r"\s{2,}", " ", s).strip() + + +def split_sentences(text: str) -> list[str]: + parts = re.split(r"(?<=[.!?])\s+", text.strip()) + return [p.strip() for p in parts if p.strip()] + + +_NUMWORD = {"first": 1, "second": 2, "third": 3, "fourth": 4, "fifth": 5, "sixth": 6, + "seventh": 7, "eighth": 8, "ninth": 9, "tenth": 10, + "one": 1, "two": 2, "three": 3, "four": 4, "five": 5, "six": 6, + "seven": 7, "eight": 8, "nine": 9, "ten": 10} +_NUMRE = "|".join(_NUMWORD) +# open/show/read/pull up + any filler words (the, article, number, up…) + a number. +_OPEN_RE = re.compile( + r"\b(?:open|show|read|pull\s*up|bring\s*up|go\s*to|display)\b\s+" + r"(?:(?:the|up|to|me|us|article|articles|result|results|number|link|story|item|photo|image|one|no)\.?\s+){0,4}" + r"(\d{1,2}|" + _NUMRE + r")\b", re.I) +_STOP_RE = re.compile(r"^\s*(stop|stop it|stop talking|be quiet|quiet|shut up|shush|silence|" + r"enough|that'?s enough|cancel|never\s*mind|hold on|halt)\s*[.!?]*\s*$", re.I) +_CLOSE_RE = re.compile(r"\b(close|dismiss|hide|exit)\b", re.I) + + +def local_command(text: str): + """Recognize an in-HUD command spoken by voice → (action, n) or (None, 0). + Forgiving of how whisper phrases things ('open article one', 'open the second + one', 'open number 3'…). Handled by the frontend (reader / stop), not the model.""" + t = (text or "").lower().strip() + if _STOP_RE.match(t): + return ("stop", 0) + if _CLOSE_RE.search(t) and re.search(r"\b(article|reader|it|that|this|page|window|frame)\b", t): + return ("close", 0) + if t.rstrip(".!?") in ("close", "go back", "dismiss", "exit"): + return ("close", 0) + m = _OPEN_RE.search(t) + if m: + g = m.group(1) + n = int(g) if g.isdigit() else _NUMWORD.get(g, 0) + if n: + return ("open", n) + return (None, 0) + + +# --------------------------------------------------------------------------- # +# The service (state machine is injectable → unit-testable without hardware) +# --------------------------------------------------------------------------- # +class AudioService: + """States: idle (await wake) → listening (record) → transcribing → thinking → + speaking → idle. Frames are int16 [FRAME]. Inject the pipeline stages for + tests; build_default() wires the real engines.""" + + def __init__( + self, + *, + router: Any, + publish: Callable[[dict], None], + wake_fn: Callable[[Any], bool], + stt_fn: Callable[[Any], str], + synth_fn: Callable[[str], Any], # text -> (float32 samples, sr) | None + play_fn: Callable[[Any, int], None], # (samples, sr) -> plays, honors abort + vad_silence_ms: int = 700, + session: str = "s_voice", + aec: Any = None, # EchoCanceller (Phase 3) or None + ref_provider: Callable[[], Any] = None, # -> current far-end (TTS) frame, or None + full_duplex: bool = False, # listen while speaking (wake-word barge) + async_turn: bool = True, # run the LLM+TTS turn off the mic thread + barge_wake_fn: Callable[[Any], bool] = None, # dedicated wake model for barge + barge_gap_ms: int = 600, # inter-sentence listen window for barge-in + barge_energy: float = 0.05, # talk-over RMS that counts as an interrupt + ) -> None: + self.router = router + self.publish = publish + self.wake_fn = wake_fn + self.stt_fn = stt_fn + self.synth_fn = synth_fn + self.play_fn = play_fn + self.vad_silence_ms = vad_silence_ms + self.session = session + self.aec = aec + self.ref_provider = ref_provider + # Barge-in only needs a wake detector, NOT working echo cancellation: the + # software FDAF cannot cancel the split USB-mic/speaker echo, and running + # the wake model on the raw mic (its phrase != ATLAS's words) is more robust. + self.barge_wake_fn = barge_wake_fn + self.full_duplex = bool(full_duplex) + # Many mics (e.g. the MAONO DGM20 with ENC) are ducked to near-silence by + # macOS whenever ANY audio is playing, so the mic is deaf mid-sentence. We + # therefore stop the output for a short gap between sentences — the mic + # recovers and can hear "Hey Atlas" or plain talk-over during that window. + self.barge_gap_ms = int(barge_gap_ms) + self.barge_energy = float(barge_energy) + self.async_turn = async_turn + + self.state = "idle" + self._utter: list = [] + self._silence_frames = 0 + self._heard = False # has the owner actually started speaking yet? + self._abort = threading.Event() + self._barge = False + self._barge_frames = 0 # sustained talk-over frames (energy barge) + self._reader_open = False # an in-HUD article reader is open (extends listen) + self._turn = 0 + + # ----- helpers -------------------------------------------------------- # + def _set_state(self, value: str) -> None: + self.state = value + self.publish({"type": "state", "value": value}) + + def _rms(self, frame) -> float: + import numpy as np + f = np.asarray(frame, dtype=np.float32) / 32768.0 + return float(np.sqrt(np.mean(f * f)) if f.size else 0.0) + + @property + def _silence_limit_frames(self) -> int: + return max(1, int(self.vad_silence_ms / 80)) # 80 ms per frame + + @property + def _onset_limit_frames(self) -> int: + # While an article reader is open we keep listening much longer (no wake word + # needed) so "close" / "next" / a follow-up question still work after you've + # been reading for a while — otherwise the 5 s window drops the mic to idle + # and the reader commands go dead. + secs = 120.0 if self._reader_open else 5.0 + return int(secs * SAMPLE_RATE / FRAME) + + # ----- per-frame state machine (the testable core) -------------------- # + def feed(self, frame) -> None: + """Process one 80 ms int16 frame. During 'speaking', full-duplex listens for + the wake word on the raw mic so you can interrupt by saying 'Hey Atlas'.""" + if self.state == "speaking": + if self.full_duplex: + self._maybe_barge(frame) + return + if self.state in ("transcribing", "thinking"): + return + if self.state == "idle": + try: + if self.wake_fn(frame): + self.publish({"type": "wake"}) + self._utter = [] + self._silence_frames = 0 + self._heard = False + self._set_state("listening") + except Exception: + pass + return + if self.state == "listening": + self._utter.append(frame) + speaking = self._rms(frame) >= 0.02 # is the owner talking now? + if speaking: + self._heard = True + self._silence_frames = 0 + elif self._heard: + self._silence_frames += 1 # only count end-pause AFTER speech starts + n = len(self._utter) + if not self._heard and n >= self._onset_limit_frames: + self._utter = [] # nobody spoke → drop it, back to idle + self._set_state("idle") + return + too_long = n > int(_MAX_UTTERANCE_S * SAMPLE_RATE / FRAME) + if (self._heard and self._silence_frames >= self._silence_limit_frames) or too_long: + self._start_turn() + + def _maybe_barge(self, frame) -> bool: + """Interrupt ATLAS while it speaks, on EITHER the wake word ('Hey Atlas') OR + plain talk-over energy. This is called every frame during 'speaking', but the + mic is ducked to near-silence while audio actually plays — so neither trigger + can fire on ATLAS itself. It's the inter-sentence listen gap (_listen_gap + stops the output) that lets the owner's voice through, where this fires.""" + barge = False + try: + detect = self.barge_wake_fn or self.wake_fn + if detect(frame): + barge = True # "Hey Atlas" over the top + elif self._rms(frame) >= self.barge_energy: + self._barge_frames += 1 + if self._barge_frames >= 3: # ~240 ms of talk-over in a gap + barge = True + else: + self._barge_frames = 0 + except Exception: + pass + if barge: + self._barge_frames = 0 + self._barge = True + self._abort.set() + _stop_playback() + self.publish({"type": "wake"}) + return True + return False + + def _start_turn(self) -> None: + import numpy as np + self._set_state("transcribing") + frames = self._utter + self._utter = [] + audio = (np.concatenate([np.asarray(f, dtype=np.int16) for f in frames]) + if frames else np.zeros(0, dtype=np.int16)) + if self.async_turn: + threading.Thread(target=self._run_turn, args=(audio,), daemon=True).start() + else: + self._run_turn(audio) + + def _run_turn(self, audio) -> None: + """STT → Router → speak. Runs off the mic thread (async) so the mic loop + stays live for barge-in while ATLAS thinks and speaks.""" + try: + text = (self.stt_fn(audio.astype("float32") / 32768.0) or "").strip() + except Exception: + text = "" + if not text: + self._set_state("idle") + return + self.publish({"type": "final", "text": text}) + # In-HUD voice commands (open article N / close / stop) are handled by the + # frontend, not the model — so "open article 1" actually opens the reader. + action, n = local_command(text) + try: + from ...evaluation.logger import log_event + log_event("voice_turn", {"transcript": text, "command": action, "n": n}) + except Exception: + pass + if action == "stop": + self._reader_open = False + self._set_state("idle") # explicit stop → go quiet + return + if action: # open / close → drive the HUD… + self.publish({"type": "command", "action": action, "n": n}) + if action == "open": + self._reader_open = True # extend the listen window while reading + elif action == "close": + self._reader_open = False + self._utter = [] # …then keep listening (no wake word) so + self._silence_frames = 0 # "close" / "next" work while reading + self._heard = False + self._set_state("listening") + return + self._abort.clear() + self._barge = False + self._turn += 1 + self._set_state("thinking") + try: + out = self.router.chat(text, channel="voice", session=self.session, turn=self._turn) + except Exception as exc: + out = {"reply": f"(voice error: {type(exc).__name__})", "media": None} + reply = out.get("reply") or "(no reply)" + self.publish({"type": "reply", "text": reply, "media": out.get("media")}) + self._speak(reply) + + def _listen_gap(self) -> None: + """Close playback for barge_gap_ms so the mic (ducked while audio plays) + recovers, then let the mic thread's _maybe_barge listen for an interrupt. + Runs on the speak/worker thread; the mic _loop runs feed()->_maybe_barge in + parallel and sets _abort if it hears the owner, ending the gap early.""" + import time + _PLAYBACK.close() # release output device → mic un-ducks + self._barge_frames = 0 + deadline = time.monotonic() + self.barge_gap_ms / 1000.0 + while time.monotonic() < deadline and not self._abort.is_set(): + time.sleep(0.02) + + def _speak(self, reply: str) -> None: + import concurrent.futures as _f + spoken = speech_clean(reply) + if not spoken: + self._set_state("idle") + return + # Give the barge detector a clean buffer for this utterance so residual echo + # from the previous turn can't bias 'Hey Atlas' detection. + if self.barge_wake_fn is not None and hasattr(self.barge_wake_fn, "reset"): + try: self.barge_wake_fn.reset() + except Exception: pass + self._set_state("speaking") + self.publish({"type": "tts", "state": "start"}) + sentences = split_sentences(spoken) + # Pipeline: synthesize sentence k+1 while sentence k is playing, so there is + # no audible gap between sentences (a major cause of choppy playback). + with _f.ThreadPoolExecutor(max_workers=1) as ex: + nxt = ex.submit(self.synth_fn, sentences[0]) if sentences else None + for i, sentence in enumerate(sentences): + if self._abort.is_set(): + break + cur, nxt = nxt, (ex.submit(self.synth_fn, sentences[i + 1]) + if i + 1 < len(sentences) else None) + self.publish({"type": "tts", "state": "sentence_start", "text": sentence}) + try: + pcm = cur.result() if cur else None + if pcm and not self._abort.is_set(): + self.play_fn(pcm[0], pcm[1]) + except Exception: + pass + # Between sentences, stop the output briefly so the (ducked) mic can + # hear an interrupt — "Hey Atlas" or just talking over ATLAS. + if (self.full_duplex and self.barge_gap_ms > 0 + and i < len(sentences) - 1 and not self._abort.is_set()): + self._listen_gap() + if self._abort.is_set(): + break + # Release the output device now that we're done speaking; leaving it open + # keeps the mic ducked, which silently kills the next "Hey Atlas". + _PLAYBACK.close() + self.publish({"type": "tts", "state": "stop"}) + if self._abort.is_set() and not self._barge: + self._set_state("idle") # explicit "stop" → go quiet + else: + # Conversational: after answering (or a barge-in), keep listening for a + # follow-up with NO wake word needed. If you don't reply, the onset + # timeout drops back to idle (wake-word listening). + self._barge = False + self._utter = [] + self._silence_frames = 0 + self._heard = False + self._set_state("listening") + + # ----- external commands (from /voice/ws) ----------------------------- # + def command(self, cmd: str) -> None: + if cmd == "abort": + self._barge = False # explicit stop → go quiet, not into listening + self._abort.set() + _stop_playback() + if self.state in ("speaking", "listening"): + self._utter = [] + self._set_state("idle") + elif cmd == "start" and self.state == "idle": + self._utter = [] + self._silence_frames = 0 + self._heard = False + self._set_state("listening") + + +# --------------------------------------------------------------------------- # +# Real-engine wiring + capture thread +# --------------------------------------------------------------------------- # +class NativeAudioRunner: + """Owns the mic stream + worker thread and drives an AudioService with the real + openWakeWord / whisper.cpp / Kokoro engines. start()/stop() from the lifespan.""" + + def __init__(self, router: Any, hub: VoiceHub) -> None: + self.router = router + self.hub = hub + self._q: queue.Queue = queue.Queue(maxsize=64) + self._stream = None + self._thread = None + self._running = False + self._ready = False + self.service: AudioService | None = None + # Rolling mic-health snapshot (max since last read) so a "is the mic actually + # hearing me?" check is possible without a browser — surfaced at /api/voice/native. + self._mic_dbg = {"frames": 0, "max_rms": 0.0, "max_wake": 0.0, + "max_barge": 0.0, "speak_rms": 0.0, "state": "idle"} + + def mic_health(self, reset: bool = True) -> dict: + d = dict(self._mic_dbg) + if reset: + for k in ("frames", "max_rms", "max_wake", "max_barge", "speak_rms"): + self._mic_dbg[k] = 0.0 + return d + + @property + def ready(self) -> bool: + return self._ready + + def start(self) -> bool: + """Return immediately; load the models + open the mic on a background thread + so the server (and the app window) come up instantly instead of blocking + ~15s while whisper/openWakeWord load. Announces readiness over the hub.""" + if self._running or not available(): + return False + self._running = True + self._thread = threading.Thread(target=self._build_and_run, daemon=True) + self._thread.start() + return True + + def _build_and_run(self) -> None: + from ... import settings as cfg + import numpy as np + import sounddevice as sd + + vcfg = (cfg.settings().get("voice") or {}) + wake_name = vcfg.get("wake_word_model", "hey_jarvis") + wake_sens = float(vcfg.get("wake_word_sensitivity", 0.6)) + wake = _build_wake(wake_name, wake_sens) + # A SECOND, independent wake model for barge-in so speaking-time frames never + # pollute the idle detector's streaming buffer (and vice-versa). + barge_wake = _build_wake(wake_name, wake_sens) + stt = _build_stt(vcfg.get("stt_model", "base.en")) + voice_id = str(vcfg.get("tts_voice", "kokoro:bm_george")).split(":")[-1] + speed = float(vcfg.get("speech_rate", 1.0)) + + from . import tts as tts_engine + + def synth_fn(text: str): + return tts_engine.synth_pcm(text, voice=voice_id, speed=speed) + + def play_fn(samples, sr): + # One persistent output stream, written sentence-by-sentence — no + # per-sentence stream open/close churn, no dual-clock re-latching. + _PLAYBACK.play(samples, sr) + + self.service = AudioService( + router=self.router, publish=self.hub.publish, + wake_fn=wake, stt_fn=stt, synth_fn=synth_fn, play_fn=play_fn, + vad_silence_ms=int(vcfg.get("vad_silence_ms", 700)), + barge_wake_fn=barge_wake, full_duplex=True, + barge_gap_ms=int(vcfg.get("barge_gap_ms", 600)), + barge_energy=float(vcfg.get("barge_energy", 0.05)), + ) + + def _cb(indata, frames, time_info, status): + try: + # clip before int16 cast: +1.0 * 32768 overflows to -32768 and + # garbles loud audio (e.g. a close-mic "Hey Atlas") for the detector. + pcm = np.clip(indata[:, 0] * 32768.0, -32768, 32767).astype(np.int16) + self._q.put_nowait(pcm) + except queue.Full: + pass + + try: + self._stream = sd.InputStream(samplerate=SAMPLE_RATE, channels=1, + blocksize=FRAME, dtype="float32", callback=_cb) + self._stream.start() + except Exception: + self._stream = None + self._running = False + return + # Native voice is live — tell any connected HUD to hand over the mic. + self._ready = True + # aec=False is honest: there is no working echo cancellation; barge-in is + # wake-word based (say "Hey Atlas" over the top), which the HUD needn't gate on. + self.hub.publish({"type": "capabilities", "v": 1, "audio": True, "wake": True, + "stt": True, "aec": False, "barge": True, + "tts_native": True, "reason": "native audio service running"}) + self.hub.publish({"type": "ready", "audio": True}) + self._loop() + + def _loop(self) -> None: + import numpy as np + while self._running: + try: + frame = self._q.get(timeout=0.5) + except queue.Empty: + continue + if self.service: + st = self.service.state + self.service.feed(frame) + d = self._mic_dbg + d["frames"] += 1 + try: + rms = float(np.sqrt(np.mean((frame.astype(np.float32) / 32768.0) ** 2))) + d["max_rms"] = max(d["max_rms"], rms) + d["max_wake"] = max(d["max_wake"], getattr(self.service.wake_fn, "last_score", 0.0)) + if st == "speaking": # measure barge detection + echo + d["max_barge"] = max(d.get("max_barge", 0.0), + getattr(self.service.barge_wake_fn, "last_score", 0.0)) + d["speak_rms"] = max(d.get("speak_rms", 0.0), rms) + except Exception: + pass + d["state"] = self.service.state + + def command(self, cmd: str) -> None: + if self.service: + self.service.command(cmd) + + def stop(self) -> None: + self._running = False + try: + if self._stream: + self._stream.stop(); self._stream.close() + except Exception: + pass + self._stream = None + _PLAYBACK.close() + + +class _WakeDetector: + """A callable openWakeWord detector with a resettable streaming buffer. reset() + clears the model's rolling feature window — used when entering 'speaking' so the + barge detector isn't primed on the previous turn's audio.""" + + def __init__(self, name: str, threshold: float) -> None: + from openwakeword.model import Model + path = wake_model_path(name) or wake_model_path("hey_jarvis") + self._model = Model(wakeword_models=[path], inference_framework="onnx") + self._threshold = float(threshold) + self._key = None + self.last_score = 0.0 + + def __call__(self, frame) -> bool: + scores = self._model.predict(frame) + if self._key is None: + self._key = next(iter(scores), None) + self.last_score = float(scores.get(self._key, 0.0)) if self._key else 0.0 + return self.last_score >= self._threshold + + def reset(self) -> None: + try: + self._model.reset() + except Exception: + pass + + +def _build_wake(name: str, threshold: float) -> "_WakeDetector": + return _WakeDetector(name, threshold) + + +def _build_stt(model_name: str) -> Callable[[Any], str]: + from pywhispercpp.model import Model + model = Model(model_name, print_realtime=False, print_progress=False, print_timestamps=False) + + def stt_fn(samples) -> str: + segs = model.transcribe(samples) + text = " ".join(s.text for s in segs).strip() + # whisper marks non-speech as [BLANK_AUDIO], (music), [ Silence ], etc. — + # strip bracketed/parenthetical markers so silence isn't treated as a turn. + text = re.sub(r"[\[(][^\])]*[\])]", "", text) + return re.sub(r"\s{2,}", " ", text).strip() + + return stt_fn diff --git a/atlas/interface/voice/tts.py b/atlas/interface/voice/tts.py index d44d91d..ef3cb14 100644 --- a/atlas/interface/voice/tts.py +++ b/atlas/interface/voice/tts.py @@ -115,3 +115,22 @@ def synth(text: str, voice: str = "bm_george", speed: float = 1.0) -> bytes | No return buf.getvalue() except Exception: return None + + +def synth_pcm(text: str, voice: str = "bm_george", speed: float = 1.0): + """Return (float32 mono samples, sample_rate) for NATIVE playback by the audio + service, or None. Same engine as synth(), no WAV wrapping.""" + text = (text or "").strip() + if not text: + return None + eng = _engine() + if eng is None: + return None + lang = "en-gb" if voice[:1] == "b" else "en-us" + try: + import numpy as np + with _LOCK: + samples, sr = eng.create(text, voice=voice, speed=speed, lang=lang) + return np.asarray(samples, dtype=np.float32), int(sr) + except Exception: + return None diff --git a/atlas/interface/web/app.js b/atlas/interface/web/app.js index c019708..b53d433 100644 --- a/atlas/interface/web/app.js +++ b/atlas/interface/web/app.js @@ -255,7 +255,7 @@ async function ask(text, { speak = false } = {}) { addMsg("atlas", reply); setTranscript(`ATLAS: ${reply}`); renderSearchMedia(data.media); - if (speak) speakReply(reply); else setState("idle"); + if (speak && voiceMode !== "native") speakReply(reply); else setState("idle"); } catch { thinking.remove(); const msg = "Core isn’t running — start it with `python -m atlas.server` to chat for real."; @@ -278,6 +278,11 @@ $("#composer").addEventListener("submit", (e) => { }); /* ---------- voice: Web Speech API (STT + TTS) ---------- */ +/* Provider seam: "web" = browser Web Speech (below); "native" = the Python audio + service over /voice/ws (echo-cancelled wake/STT — Phase 2). Only ONE owns the + mic. Phase 1 stays "web"; the guards below make the switch clean when native + audio arrives. */ +let voiceMode = "web"; const SR = window.SpeechRecognition || window.webkitSpeechRecognition; let recog = null, listening = false, voiceTurn = false, followTimer = null; function initVoice() { @@ -295,6 +300,7 @@ function initVoice() { recog.onend = () => { if (listening) { listening = false; if (orbState === "listening") setState("idle"); } resumeWake(); }; } function startListening() { + if (voiceMode === "native") { NativeVoice.send({ type: "start" }); return; } if (!recog) { openChat(); return; } if (wakeRunning) { try { wakeSR.stop(); } catch {} wakeRunning = false; } // free the mic for command capture stopSpeaking(); @@ -352,6 +358,7 @@ function initBarge() { bargeSR.onerror = () => { bargeRunning = false; }; } function startBarge() { + if (voiceMode === "native") return; // native service handles barge-in with real AEC if (!bargeSR || !wakeEnabled || bargeRunning || orbState !== "speaking") return; stopFrameListen(); // free the mic for the barge listener while speaking try { bargeSR.start(); bargeRunning = true; } catch {} @@ -382,6 +389,7 @@ function initFrameListen() { frameSR.onerror = () => { frameListening = false; }; } function startFrameListen() { + if (voiceMode === "native") return; // native service owns the mic if (!frameSR || !wakeEnabled) return; if ($("#articleFrame").hidden || frameListening || orbState === "speaking") return; try { wakeSR && wakeSR.stop(); } catch {} wakeRunning = false; @@ -467,6 +475,7 @@ function speechClean(s) { /* Primary voice: clean on-device neural TTS (Kokoro) served by /api/tts. Falls back to the OS voice if the local model isn't installed. */ async function speakReply(text) { + if (voiceMode === "native") { setState("idle"); return; } // native service owns the voice const spoken = speechClean(text); if (!spoken) { setState("idle"); resumeWake(); return; } stopSpeaking(); @@ -489,6 +498,10 @@ async function speakReply(text) { } } $("#orb").addEventListener("click", () => { + if (voiceMode === "native") { // native service owns capture/playback + NativeVoice.send({ type: orbState === "speaking" || orbState === "listening" ? "abort" : "start" }); + return; + } if (orbState === "speaking") { stopSpeaking(); setState("idle"); resumeWake(); return; } listening ? stopListening() : startListening(); }); @@ -527,6 +540,7 @@ function initWake() { }; } function startWake() { + if (voiceMode === "native") return; // native service listens for the wake word if (!wakeSR || wakeRunning || listening || orbState === "speaking" || orbState === "thinking") return; try { wakeSR.start(); wakeRunning = true; } catch {} } @@ -1134,15 +1148,20 @@ function renderSearchMedia(media) { const shown = results.slice(0, 6); const vw = window.innerWidth, vh = window.innerHeight; const compact = vw < 900; - const iw = compact ? 176 : 248, ih = compact ? 124 : 176; // photo size (~2×) - const capH = 40, ch = ih + capH; - const gapX = 22, gapY = 20; + // Target ~2× the previous size, then scale the whole grid down to fit the window. + let iw = compact ? 360 : 496, ih = compact ? 256 : 352; + let capH = 44, gapX = 22, gapY = 20; const cols = Math.min(shown.length, 3); const rows = Math.ceil(shown.length / cols); + let gW = cols * iw + (cols - 1) * gapX, gH = rows * (ih + capH) + (rows - 1) * gapY; + const scale = Math.min(1, (vw * 0.94) / gW, (vh * 0.9) / gH); + iw = Math.round(iw * scale); ih = Math.round(ih * scale); capH = Math.round(capH * scale); + gapX = Math.round(gapX * scale); gapY = Math.round(gapY * scale); + const ch = ih + capH; const gridW = cols * iw + (cols - 1) * gapX; const gridH = rows * ch + (rows - 1) * gapY; - const startX = Math.max(12, (vw - gridW) / 2); // centred horizontally - const startY = Math.max(64, (vh - gridH) / 2); // …and vertically + const startX = Math.max(8, (vw - gridW) / 2); // centred horizontally + const startY = Math.max(48, (vh - gridH) / 2); // …and vertically const cx = vw / 2 - iw / 2, cy = vh / 2 - ch / 2; // screen centre (pop origin) shown.forEach((r, i) => { const col = i % cols, row = Math.floor(i / cols); @@ -1255,19 +1274,28 @@ function initArticleFrame() { /* "open article 2" / "close article" — resolved in-HUD (voice or chat) */ const _NUMWORD = { first: 1, second: 2, third: 3, fourth: 4, fifth: 5, sixth: 6, one: 1, two: 2, three: 3, four: 4, five: 5, six: 6 }; -function handleArticleCommand(text, speak) { +/* Parse an "open article N" / "close" command → {action:"open",n} | {action:"close"} | null. + Shared by the chat/voice entry (handleArticleCommand) and the article-frame + voice handler (handleFrameVoice). */ +function parseArticleCommand(text) { const t = (text || "").toLowerCase().trim(); - if (t === "close" || /\b(close|dismiss|hide)\b.*\b(article|reader|frame|page|window)\b/.test(t)) { + if (t === "close" || /\b(close|dismiss|hide)\b.*\b(article|reader|frame|page|window)\b/.test(t)) + return { action: "close" }; + const m = t.match(/\b(?:open|show|read|pull up|bring up|display|go to)\b\s+(?:the\s+)?(?:(?:article|result|number|link|photo|image|story|item|#|no\.?)\s+)?(?:the\s+)?(\d{1,2}|first|second|third|fourth|fifth|sixth|one|two|three|four|five|six)\b/); + if (!m) return null; + const n = /^\d+$/.test(m[1]) ? +m[1] : _NUMWORD[m[1]]; + return n ? { action: "open", n } : null; +} +function handleArticleCommand(text, speak) { + const cmd = parseArticleCommand(text); + if (!cmd) return false; + if (cmd.action === "close") { if (!$("#articleFrame").hidden) { closeFrame(); return true; } return false; } - const m = t.match(/\b(?:open|show|read|pull up|bring up|display|go to)\b\s+(?:the\s+)?(?:(?:article|result|number|link|photo|image|story|item|#|no\.?)\s+)?(?:the\s+)?(\d{1,2}|first|second|third|fourth|fifth|sixth|one|two|three|four|five|six)\b/); - if (!m) return false; - const n = /^\d+$/.test(m[1]) ? +m[1] : _NUMWORD[m[1]]; - if (!n) return false; - const r = searchResults[n - 1]; - if (!r) { toast(`No article ${n} yet — search first`, "err"); return true; } - addMsg("owner", `open article ${n}`); + const r = searchResults[cmd.n - 1]; + if (!r) { toast(`No article ${cmd.n} yet — search first`, "err"); return true; } + addMsg("owner", `open article ${cmd.n}`); addMsg("atlas", `Opening “${r.title || r.url}”.`); openArticleInFrame(r); if (speak) resumeWake(); @@ -1285,6 +1313,67 @@ function sessionId() { return s; } +/* ---------- native voice channel (Phase 1 seam; audio pipeline lands Phase 2) ---------- */ +/* Connects to the additive /voice/ws. If the backend ever reports audio=true (the + native mic/wake/STT/AEC service is installed), the HUD hands mic ownership to it; + otherwise it stays on the browser Web Speech provider. */ +const NativeVoice = { + ws: null, ready: false, caps: null, retry: 0, + connect() { + try { + const ws = new WebSocket((API || location.origin).replace(/^http/, "ws") + "/voice/ws"); + this.ws = ws; + ws.onmessage = (ev) => { let m; try { m = JSON.parse(ev.data); } catch { return; } this.onMsg(m); }; + ws.onclose = () => { this.ready = false; if (voiceMode === "native") applyVoiceProvider(); this.reconnect(); }; + ws.onerror = () => {}; + } catch {} + }, + reconnect() { this.retry = Math.min(this.retry + 1, 6); setTimeout(() => this.connect(), 1500 * this.retry); }, + onMsg(m) { + if (m.type === "capabilities") { this.caps = m; return; } + if (m.type === "ready") { this.ready = true; this.retry = 0; applyVoiceProvider(); return; } + if (voiceMode !== "native") return; // ignore events unless native owns the mic + switch (m.type) { + case "state": { // drive the orb (map transcribing→thinking) + const s = m.value === "transcribing" ? "thinking" : m.value; + setState(s === "listening" || s === "thinking" || s === "speaking" ? s : "idle"); + break; + } + case "pong": break; + case "final": // what ATLAS heard + if (m.text) { addMsg("owner", m.text); setTranscript(`You: ${m.text}`); } + break; + case "reply": // ATLAS's answer (spoken natively by the service) + if (m.text) { addMsg("atlas", m.text); setTranscript(`ATLAS: ${m.text}`); } + renderSearchMedia(m.media); // always (renders on search, clears otherwise) + break; + case "command": // in-HUD voice command (open article N / close) + if (m.action === "open") { + const r = searchResults[m.n - 1]; + if (r) { addMsg("owner", `open article ${m.n}`); openArticleInFrame(r); } + else toast(`No article ${m.n} — search first`, "err"); + } else if (m.action === "close") { closeFrame(); } + break; + // "wake" and "tts" states are reflected via the "state" events above. + } + }, + available() { return this.ready && !!this.caps && this.caps.audio === true; }, + send(o) { try { if (this.ws && this.ws.readyState === 1) this.ws.send(JSON.stringify(o)); } catch {} }, +}; +function applyVoiceProvider() { + const native = NativeVoice.available(); + const next = native ? "native" : "web"; + if (next === voiceMode) return; + voiceMode = next; + if (native) { // hand the mic to the native service + try { wakeSR && wakeSR.stop(); recog && recog.stop(); bargeSR && bargeSR.stop(); frameSR && frameSR.stop(); } catch {} + wakeRunning = bargeRunning = frameListening = false; + } else if (wakeEnabled) { // native dropped → resume Web Speech + resumeWake(); + } + console.log("[voice] provider:", voiceMode, this && this.caps ? "" : (NativeVoice.caps || "")); +} + /* ---------- live state via WebSocket (optional) ---------- */ function connectWS() { try { @@ -1319,6 +1408,7 @@ initWake(); initBarge(); initFrameListen(); initArticleFrame(); +NativeVoice.connect(); // Phase 1: connect to /voice/ws; stays on Web Speech until native audio is ready if (localStorage.getItem("atlas_wake") === "on") setWake(true); else updateWakeUI(); renderStatus(); renderMetrics(); renderMemory(); renderUpgrade(); renderScheduler(); renderCredits(); connectWS(); diff --git a/atlas/orchestration/router.py b/atlas/orchestration/router.py index 403e9fd..ecfab9b 100644 --- a/atlas/orchestration/router.py +++ b/atlas/orchestration/router.py @@ -29,10 +29,10 @@ def _identity_prompt() -> str: def keychain_secret(ref: str | None) -> str | None: - """Read a secret from the macOS Keychain given a 'keychain:' ref. - Lets the Owner store the API key securely (never in settings.json or git) - and have ATLAS pick it up without an env var. Returns None off-macOS or - if the item is absent.""" + """Read a secret given a 'keychain:' ref. macOS Keychain via the + `security` CLI (primary); on Windows/Linux it falls back to the OS keyring + (Credential Manager / Secret Service) via the optional `keyring` package. So + the key stays out of settings.json/git on every platform. None if absent.""" if not ref or not ref.startswith("keychain:"): return None name = ref.split(":", 1)[1] @@ -41,8 +41,15 @@ def keychain_secret(ref: str | None) -> str | None: ["security", "find-generic-password", "-s", name, "-w"], capture_output=True, text=True, timeout=5, ) - return out.stdout.strip() or None + val = out.stdout.strip() + if val: + return val except (OSError, subprocess.SubprocessError): + pass + try: # cross-platform fallback (Windows/Linux) + import keyring + return keyring.get_password("atlas", name) or None + except Exception: return None diff --git a/atlas/run.sh b/atlas/run.sh old mode 100644 new mode 100755 diff --git a/atlas/server/__main__.py b/atlas/server/__main__.py index c0cb4ae..9d4597b 100644 --- a/atlas/server/__main__.py +++ b/atlas/server/__main__.py @@ -1,20 +1,62 @@ -"""Entry point: `python -m atlas.server`.""" +"""Entry point: `python -m atlas.server` (and the frozen Tauri sidecar). + +Sidecar contract (Phase 0): accepts `--port N` (or `--port 0`/none → pick a free +port), binds 127.0.0.1 only, and prints `ATLAS_READY ` on stdout once uvicorn +is serving. The Tauri shell reads that line, then points its webview at the port. +""" from __future__ import annotations +import argparse import os +import socket +import threading import uvicorn from .. import settings as cfg +def _free_port() -> int: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.bind(("127.0.0.1", 0)) + port = s.getsockname()[1] + s.close() + return port + + +def _announce_when_ready(server: "uvicorn.Server", port: int) -> None: + """Print the readiness line once the server is actually accepting requests, so + a supervisor (Tauri) can gate the webview on it instead of racing startup.""" + import time + for _ in range(600): # up to ~60s + if getattr(server, "started", False): + print(f"ATLAS_READY {port}", flush=True) + return + time.sleep(0.1) + + def main() -> None: - port = int(cfg.settings().get("ui", {}).get("port", 8765)) - # Cheap mode check — the actual Router is built once inside the app module. + ap = argparse.ArgumentParser(prog="atlas.server", add_help=True) + ap.add_argument("--port", type=int, default=None, + help="port to bind (0 or omitted → a free port, printed on stdout)") + ap.add_argument("--host", default="127.0.0.1") + args, _ = ap.parse_known_args() + + if args.port is None: # omitted → settings port (or a free one) + port = int(cfg.settings().get("ui", {}).get("port", 8765) or 0) or _free_port() + elif args.port == 0: # explicit 0 → pick a free port + port = _free_port() + else: + port = args.port + key_env = cfg.settings()["model"].get("api_key_env", "ANTHROPIC_API_KEY") mode = "claude" if os.environ.get(key_env) else "offline/degraded" - print(f"ATLAS core -> http://localhost:{port} (mode: {mode})") - uvicorn.run("atlas.server.app:app", host="127.0.0.1", port=port, reload=False) + print(f"ATLAS core -> http://{args.host}:{port} (mode: {mode})", flush=True) + + config = uvicorn.Config("atlas.server.app:app", host=args.host, port=port, reload=False) + server = uvicorn.Server(config) + threading.Thread(target=_announce_when_ready, args=(server, port), daemon=True).start() + server.run() if __name__ == "__main__": diff --git a/atlas/server/app.py b/atlas/server/app.py index 57b9b1a..068aa06 100644 --- a/atlas/server/app.py +++ b/atlas/server/app.py @@ -52,16 +52,38 @@ def _scheduler_enabled() -> bool: return bool(cfg.settings().get("scheduler", {}).get("enabled")) and not os.environ.get("ATLAS_NO_SCHEDULER") +def _native_audio_enabled() -> bool: + # Opt-in (voice.native_audio) + deps/models present + not disabled for tests. + if os.environ.get("ATLAS_NO_AUDIO") or os.environ.get("ATLAS_FORCE_OFFLINE"): + return False + return bool((cfg.settings().get("voice") or {}).get("native_audio")) + + @asynccontextmanager async def lifespan(_app: FastAPI): if _scheduler_enabled(): scheduler.start() + # Native voice pipeline (Phase 2): mic → wake → STT → Router → Kokoro over /voice/ws. + global audio_runner + if _native_audio_enabled(): + from ..interface.voice.audio_service import NativeAudioRunner, available + if available(): + import asyncio + voice_hub.bind_loop(asyncio.get_running_loop()) + audio_runner = NativeAudioRunner(router, voice_hub) + started = audio_runner.start() + log_event("voice_native", {"started": started}) + if not started: + audio_runner = None log_event("startup", {"mode": cfg.settings()["model"].get("mode", "api"), "backend": router.backend, - "scheduler": _scheduler_enabled()}) + "scheduler": _scheduler_enabled(), + "native_audio": audio_runner is not None}) yield log_event("shutdown", {}) scheduler.stop() + if audio_runner is not None: + audio_runner.stop() app = FastAPI(title="ATLAS", version=__version__, lifespan=lifespan) @@ -72,6 +94,12 @@ async def lifespan(_app: FastAPI): registry = ConnectorRegistry() scheduler = Scheduler() +# Native voice (Phase 2): the hub bridges the audio thread to /voice/ws clients; +# audio_runner is set in lifespan when voice.native_audio is on and deps are present. +from ..interface.voice.audio_service import VoiceHub # noqa: E402 +voice_hub = VoiceHub() +audio_runner = None + class ChatIn(BaseModel): text: str @@ -408,6 +436,50 @@ def tts(body: TTSIn) -> Response: return Response(content=wav, media_type="audio/wav") +@app.get("/api/voice/native") +def voice_native() -> dict[str, Any]: + """Native-voice pipeline status for the HUD/settings: whether it's enabled, + whether the deps+models are installed, and whether it's currently running.""" + from ..interface.voice.audio_service import available, deps_present, wake_model_path + return { + "enabled": _native_audio_enabled(), + "deps_present": deps_present(), + "wake_model": bool(wake_model_path()), + "available": available(), + "running": audio_runner is not None, + "ready": bool(audio_runner is not None and audio_runner.ready), + "mic": (audio_runner.mic_health() if (audio_runner is not None + and hasattr(audio_runner, "mic_health")) else None), + "capabilities": _voice_capabilities(), + } + + +@app.post("/api/voice/echotest") +def voice_echotest(secs: float = 8.0, gain: float = 1.0) -> dict[str, Any]: + """DEBUG: play TTS through the native output while the mic loop keeps running in + idle, so we can measure whether output playback attenuates mic input (the barge-in + 'can't hear me while speaking' symptom). Poll /api/voice/native mic.max_rms during. + gain lets us test whether quieter playback un-ducks the mic (level-based ducking).""" + if audio_runner is None or not getattr(audio_runner, "ready", False): + return {"ok": False, "reason": "native audio not running"} + import threading, time + from ..interface.voice import tts as tts_engine + from ..interface.voice.audio_service import _PLAYBACK + + def _play() -> None: + pcm = tts_engine.synth_pcm( + "Testing one two three four five six seven eight nine ten eleven twelve.", + voice="bm_lewis", speed=1.0) + if not pcm: + return + t0 = time.time() + while time.time() - t0 < secs: + _PLAYBACK.play(pcm[0], pcm[1], gain=float(gain)) + + threading.Thread(target=_play, daemon=True).start() + return {"ok": True, "playing_secs": secs, "gain": gain} + + @app.get("/api/read") def read_page(url: str) -> dict[str, Any]: """Extract an article's readable text so the HUD reader panel can show it @@ -609,6 +681,64 @@ async def ws(sock: WebSocket) -> None: return +def _voice_capabilities() -> dict[str, Any]: + """What the native audio pipeline can do right now. audio=True only when the + NativeAudioRunner actually started (mic + wake + STT live); then the HUD hands + it mic ownership. Otherwise the browser Web Speech fallback stays active.""" + on = audio_runner is not None and audio_runner.ready + if on: + # No working echo cancellation (the software AEC can't cancel the real + # speaker→mic echo); barge-in is wake-word based instead. Report honestly. + return {"v": 1, "audio": True, "wake": True, "stt": True, "aec": False, + "barge": True, "tts_native": True, "reason": "native audio service running"} + from ..interface.voice.audio_service import available as _av + reason = ("native audio off (enable voice.native_audio)" + if not _native_audio_enabled() else + ("native audio deps/models not installed" if not _av() else + ("native audio loading…" if audio_runner is not None else "native audio not started"))) + return {"v": 1, "audio": False, "wake": False, "stt": False, "aec": False, + "tts_native": False, "reason": reason} + + +@app.websocket("/voice/ws") +async def voice_ws(sock: WebSocket) -> None: + """Native-voice channel (additive; /ws is untouched). The HUD's + NativeVoiceProvider connects here, handshakes, and — when capabilities report + audio=True — takes over the mic. Broadcasts the audio service's events + (state/wake/final/reply/tts) and accepts start/stop/abort. Loopback-only.""" + import asyncio + await sock.accept() + caps = _voice_capabilities() + await sock.send_json({"type": "hello", "v": 1, "version": __version__}) + await sock.send_json({"type": "capabilities", **caps}) + await sock.send_json({"type": "ready", "audio": caps["audio"]}) + await sock.send_json({"type": "state", "value": "idle"}) + + # ALWAYS subscribe — the native pipeline loads models in the background and + # publishes an updated {capabilities}/{ready} over the hub when it comes live, + # so a client that connected before it was ready switches to native then. + q = voice_hub.subscribe() + + async def _pump() -> None: + while True: + await sock.send_json(await q.get()) + + pump = asyncio.create_task(_pump()) + try: + while True: + msg = await sock.receive_json() + t = msg.get("type") + if t == "ping": + await sock.send_json({"type": "pong"}) + elif t in ("start", "stop", "abort") and audio_runner is not None: + audio_runner.command(t) + except WebSocketDisconnect: + return + finally: + pump.cancel() + voice_hub.unsubscribe(q) + + # --------------------------------------------------------------------------- # # Static HUD (mounted last so /api/* wins) # --------------------------------------------------------------------------- # diff --git a/atlas/tests/test_aec.py b/atlas/tests/test_aec.py new file mode 100644 index 0000000..dd8608d --- /dev/null +++ b/atlas/tests/test_aec.py @@ -0,0 +1,81 @@ +"""L3 Phase 3: echo cancellation (FDAF) reduces far-end echo, and the audio +service's full-duplex barge-in decision fires on a wake during speech.""" +from __future__ import annotations + +import numpy as np + +from atlas.interface.voice import aec as AEC +from atlas.interface.voice import audio_service as A + + +def _i16(x): + return np.clip(x * 32768.0, -32768, 32767).astype(np.int16) + + +def test_fdaf_reduces_echo_energy(): + rng = np.random.RandomState(0) + ec = AEC.EchoCanceller(block=A.FRAME, mu=0.3) + room = np.array([0.0, 0.6, 0.0, -0.3, 0.15], dtype=np.float64) # short echo path + tail = np.zeros(len(room) - 1) + + def step(measure): + nonlocal tail, mic_energy, out_energy + ref = rng.randn(A.FRAME) * 0.3 # far-end (TTS) signal + full = np.concatenate([tail, ref]) + echo = np.convolve(full, room, mode="valid") # echo the mic picks up + tail = ref[-(len(room) - 1):] + out = ec.process(_i16(echo), _i16(ref)).astype(np.float64) / 32768.0 + if measure: + mic_energy += float(np.sum(echo ** 2)) + out_energy += float(np.sum(out ** 2)) + + mic_energy = out_energy = 0.0 + for _ in range(80): # warm up the adaptive filter + step(measure=False) + for _ in range(40): # measure steady-state residual + step(measure=True) + # once adapted, residual echo should be << the raw mic echo (~>6 dB reduction) + assert out_energy < 0.25 * mic_energy, (out_energy, mic_energy) + + +def test_make_aec_backends_fall_back_gracefully(): + assert AEC.make_aec("none").process(_i16(np.zeros(A.FRAME)), _i16(np.zeros(A.FRAME))) is not None + # os / webrtc have no native build here → software fallback, still functional + assert "nlms" in AEC.make_aec("os").backend + assert "nlms" in AEC.make_aec("webrtc").backend + + +def test_full_duplex_barge_in_fires_on_wake_during_speech(): + events = [] + wake = {"v": False} + + class _R: + def chat(self, text, **kw): + return {"reply": "ok", "media": None} + + svc = A.AudioService( + router=_R(), publish=events.append, + wake_fn=lambda f: wake["v"], stt_fn=lambda s: "hi", + synth_fn=lambda t: (np.zeros(2, dtype=np.float32), 16000), + play_fn=lambda s, sr: None, + aec=AEC.make_aec("none"), ref_provider=lambda: np.zeros(A.FRAME, dtype=np.int16), + full_duplex=True, + ) + svc.state = "speaking" # pretend ATLAS is mid-sentence + wake["v"] = True + barged = svc._maybe_barge(np.zeros(A.FRAME, dtype=np.int16)) + assert barged is True + assert svc._abort.is_set() and svc._barge is True # abort + queued to listen + assert {"type": "wake"} in events + + +def test_full_duplex_no_barge_without_wake(): + svc = A.AudioService( + router=object(), publish=lambda e: None, + wake_fn=lambda f: False, stt_fn=lambda s: "", + synth_fn=lambda t: None, play_fn=lambda s, sr: None, + aec=AEC.make_aec("none"), ref_provider=lambda: None, full_duplex=True, + ) + svc.state = "speaking" + assert svc._maybe_barge(np.zeros(A.FRAME, dtype=np.int16)) is False + assert not svc._abort.is_set() diff --git a/atlas/tests/test_audio_service.py b/atlas/tests/test_audio_service.py new file mode 100644 index 0000000..4ad8327 --- /dev/null +++ b/atlas/tests/test_audio_service.py @@ -0,0 +1,170 @@ +"""L3 native audio service: the full state machine (idle→listening→transcribing→ +thinking→speaking→idle) driven by injected fakes — no microphone or models.""" +from __future__ import annotations + +import numpy as np + +from atlas.interface.voice import audio_service as A + + +class _Router: + def __init__(self): + self.calls = [] + + def chat(self, text, **kw): + self.calls.append((text, kw)) + return {"reply": "Hello there. Nice to meet you.", "media": None} + + +def _svc(**over): + events = [] + played = [] + wake_on = {"v": False} + + defaults = dict( + router=_Router(), + publish=events.append, + wake_fn=lambda frame: wake_on["v"], + stt_fn=lambda samples: "what time is it", + synth_fn=lambda text: (np.zeros(4, dtype=np.float32), 16000), + play_fn=lambda samples, sr: played.append((len(samples), sr)), + vad_silence_ms=160, # 2 frames of silence ends the utterance + async_turn=False, # run the turn inline so tests are deterministic + ) + defaults.update(over) + svc = A.AudioService(**defaults) + return svc, events, played, wake_on + + +def _silence(): + return np.zeros(A.FRAME, dtype=np.int16) + + +def _loud(): + return (np.ones(A.FRAME, dtype=np.int16) * 8000) + + +def test_speech_clean_strips_markup_and_urls(): + out = A.speech_clean("**Bold** see [link](https://x.com) and `code`") + assert "*" not in out and "https" not in out and "`" not in out + assert "Bold" in out and "link" in out + + +def test_split_sentences(): + assert A.split_sentences("One. Two! Three?") == ["One.", "Two!", "Three?"] + + +def test_wake_transitions_idle_to_listening(): + svc, events, _, wake = _svc() + svc.feed(_silence()) + assert svc.state == "idle" # no wake yet + wake["v"] = True + svc.feed(_loud()) + assert svc.state == "listening" + assert {"type": "wake"} in events + + +def test_full_turn_records_transcribes_responds_and_speaks(): + svc, events, played, wake = _svc() + wake["v"] = True + svc.feed(_loud()) # wake → listening + wake["v"] = False + svc.feed(_loud()) # speech frame (owner starts talking) + svc.feed(_silence()); svc.feed(_silence()) # 2 silent frames → end utterance + assert svc.state == "listening" # conversational: awaits a follow-up + types = [e.get("type") for e in events] + assert "final" in types and "reply" in types + # tts start → stop bracket the speech, and Kokoro played at least one sentence + tts_states = [e["state"] for e in events if e.get("type") == "tts"] + assert tts_states[0] == "start" and tts_states[-1] == "stop" + assert played, "expected native playback of at least one sentence" + # states passed through transcribing → thinking → speaking + seen = [e["value"] for e in events if e.get("type") == "state"] + assert "transcribing" in seen and "thinking" in seen and "speaking" in seen + + +def test_empty_transcript_returns_to_idle_without_responding(): + router = _Router() + svc, events, _, wake = _svc(router=router, stt_fn=lambda s: " ") + wake["v"] = True + svc.feed(_loud()); wake["v"] = False + svc.feed(_loud()) # owner spoke, but STT returns blank + svc.feed(_silence()); svc.feed(_silence()) + assert svc.state == "idle" # blank transcript → idle, no follow-up + assert router.calls == [] # nothing heard → no Router call + assert "reply" not in [e.get("type") for e in events] + + +def test_waits_for_speech_after_wake_then_times_out_to_idle(): + # THE bug fix: after wake, pure silence must NOT trigger a turn — it waits for + # the owner to actually speak, then times out to idle if they never do. + router = _Router() + svc, _, _, wake = _svc(router=router) + wake["v"] = True + svc.feed(_loud()); wake["v"] = False # wake → listening + for _ in range(3): # a few silent frames… + svc.feed(_silence()) + assert svc.state == "listening" # still waiting for you to speak + assert router.calls == [] + for _ in range(svc._onset_limit_frames + 1): # …no speech for the whole window + svc.feed(_silence()) + assert svc.state == "idle" # gives up gracefully, no response + assert router.calls == [] + + +def test_follow_up_after_answer_needs_no_wake_word(): + # Conversational: after ATLAS answers, it listens again with no wake word. + svc, events, _, wake = _svc() + wake["v"] = True + svc.feed(_loud()); wake["v"] = False + svc.feed(_loud()); svc.feed(_silence()); svc.feed(_silence()) # first turn + assert svc.state == "listening" + # a follow-up utterance — WITHOUT saying the wake word again + svc.feed(_loud()); svc.feed(_silence()); svc.feed(_silence()) + assert len(svc.router.calls) == 2 # second turn handled directly + + +def test_abort_stops_speaking(): + # A synth that aborts mid-way: after the first sentence, flip the abort flag. + svc, events, played, wake = _svc() + + def synth(text): + svc.command("abort") # user says stop during playback + return (np.zeros(2, dtype=np.float32), 16000) + + svc.synth_fn = synth + wake["v"] = True + svc.feed(_loud()); wake["v"] = False + svc.feed(_loud()) # speech → utterance + svc.feed(_silence()); svc.feed(_silence()) + assert svc.state == "idle" # explicit stop → quiet, not follow-up + assert {"type": "tts", "state": "stop"} in events # always brackets to stop + + +def test_local_command_open_routes_to_hud_not_model(): + # "open article 2" by voice must drive the HUD, not the LLM. + router = _Router() + svc, events, _, wake = _svc(router=router, stt_fn=lambda s: "open article 2") + wake["v"] = True + svc.feed(_loud()); wake["v"] = False + svc.feed(_loud()); svc.feed(_silence()); svc.feed(_silence()) + assert router.calls == [] # NOT sent to the model + cmds = [e for e in events if e.get("type") == "command"] + assert cmds and cmds[0]["action"] == "open" and cmds[0]["n"] == 2 + assert svc.state == "listening" # keeps listening for "close"/"next" + + +def test_local_command_stop_goes_quiet(): + router = _Router() + svc, events, _, wake = _svc(router=router, stt_fn=lambda s: "stop") + wake["v"] = True + svc.feed(_loud()); wake["v"] = False + svc.feed(_loud()); svc.feed(_silence()); svc.feed(_silence()) + assert router.calls == [] + assert not [e for e in events if e.get("type") == "command"] # stop → no HUD command + assert svc.state == "idle" + + +def test_hub_publish_without_loop_is_safe(): + hub = A.VoiceHub() + hub.publish({"type": "state", "value": "idle"}) # no loop bound → no crash diff --git a/atlas/tests/test_server.py b/atlas/tests/test_server.py index f121d2b..ef58cd4 100644 --- a/atlas/tests/test_server.py +++ b/atlas/tests/test_server.py @@ -92,3 +92,50 @@ def test_scheduler_endpoint_lists_jobs(): def test_scheduler_run_unknown_job_404(): r = client.post("/api/scheduler/run/ghost") assert r.status_code == 404 + + +def test_voice_ws_handshake_reports_no_native_audio(): + # Phase 1 seam: /voice/ws exists and handshakes, but advertises audio=False so + # the HUD keeps its browser Web Speech provider until the native service lands. + with client.websocket_connect("/voice/ws") as ws: + assert ws.receive_json()["type"] == "hello" + caps = ws.receive_json() + assert caps["type"] == "capabilities" and caps["audio"] is False + assert ws.receive_json() == {"type": "ready", "audio": False} + assert ws.receive_json() == {"type": "state", "value": "idle"} + ws.send_json({"type": "ping"}) + assert ws.receive_json() == {"type": "pong"} + + +def test_voice_ws_native_path_forwards_commands(monkeypatch): + # When the native runner is active, /voice/ws reports audio=True and routes + # start/stop/abort to it (verified with a fake runner — no mic needed). + from atlas.server import app as appmod + + class FakeRunner: + ready = True + + def __init__(self): + self.cmds = [] + + def command(self, c): + self.cmds.append(c) + + fake = FakeRunner() + monkeypatch.setattr(appmod, "audio_runner", fake) + with client.websocket_connect("/voice/ws") as ws: + assert ws.receive_json()["type"] == "hello" + assert ws.receive_json()["audio"] is True # capabilities + assert ws.receive_json() == {"type": "ready", "audio": True} + assert ws.receive_json() == {"type": "state", "value": "idle"} + ws.send_json({"type": "abort"}) + ws.send_json({"type": "ping"}) + assert ws.receive_json() == {"type": "pong"} + assert "abort" in fake.cmds + + +def test_voice_native_status_endpoint(): + r = client.get("/api/voice/native") + assert r.status_code == 200 + b = r.json() + assert "available" in b and "running" in b and "capabilities" in b diff --git a/deploy/NATIVE_APP.md b/deploy/NATIVE_APP.md new file mode 100644 index 0000000..8552340 --- /dev/null +++ b/deploy/NATIVE_APP.md @@ -0,0 +1,41 @@ +# ATLAS native app (Tauri) — build & status + +ATLAS ships as a native desktop app that wraps the existing FastAPI HUD. The Python +core is unchanged: it runs as a **sidecar** (a PyInstaller-frozen binary) and the +Tauri shell points a webview at it. The localhost web HUD still works exactly as +before — Tauri is just a better front door. + +## Build (macOS) +Prereqs (once): Rust (`rustup`), Tauri CLI (`cargo install tauri-cli --version ^2`), +PyInstaller (`pip install pyinstaller`), and app icons +(`cd src-tauri && cargo tauri icon ../path/to/atlas-logo.png`). + +```bash +./deploy/build-app.sh # freeze sidecar → build .app + .dmg +``` + +Output: `src-tauri/target/release/bundle/`. + +## Signing / release +`.github/workflows/build-app.yml` builds macOS + Windows on a `v*` tag. macOS +signing + notarization activate when these repo secrets are set (never commit them): +`APPLE_CERTIFICATE`, `APPLE_CERTIFICATE_PASSWORD`, `APPLE_SIGNING_IDENTITY`, +`APPLE_ID`, `APPLE_PASSWORD`, `APPLE_TEAM_ID`. Entitlements (mic + hardened runtime) +are in `src-tauri/Entitlements.plist`; the mic prompt string is in `src-tauri/Info.plist`. + +## What's verified vs. needs your machine +- ✅ **Verified here:** the sidecar contract (`--port 0` → free port + `ATLAS_READY`), + and the **frozen binary serving the API + HUD with no Python installed**. +- 🖥️ **Your on-device step:** `cargo tauri build` (long Rust compile), launching the + GUI, granting the mic permission, and the interactive voice test. + +## Windows follow-ons (Phase 5) +The shell, `/voice/ws` contract, HUD, settings, Router, and Kokoro are all +platform-agnostic and reused verbatim. Windows-specific work, staged: +- **Signing:** Azure Key Vault (HSM cert) to beat SmartScreen — add to the CI job. +- **AEC reference:** feed the WebRTC canceller a **WASAPI loopback** capture of the + render endpoint (never RAW capture — it strips AEC). Hook in `aec.py` / + `audio_service.py` (`aec_backend="webrtc"`). +- **Secrets:** `keychain_secret` already falls back to the OS keyring (Windows + Credential Manager) when the macOS `security` CLI isn't present. +- **STT:** whisper.cpp on CUDA (NVIDIA) or CPU instead of Metal. diff --git a/deploy/atlas-core.spec b/deploy/atlas-core.spec new file mode 100644 index 0000000..3554d6a --- /dev/null +++ b/deploy/atlas-core.spec @@ -0,0 +1,50 @@ +# PyInstaller spec — freeze the ATLAS FastAPI core into a single binary that the +# Tauri shell runs as a sidecar (so end users need no Python). Build: +# pyinstaller --clean -y deploy/atlas-core.spec +# The heavy optional native-audio deps (onnxruntime/whisper/sounddevice) are NOT +# bundled here — the shell ships with chat + browser voice; native audio is an +# opt-in local install, mirroring how Kokoro is optional. +import os +from PyInstaller.utils.hooks import collect_submodules, collect_data_files + +ROOT = os.path.dirname(SPECPATH) # repo root (spec lives in deploy/) + + +def _p(rel): + return os.path.join(ROOT, rel) + + +datas = [] +# Bundled, non-gitignored assets the server serves / reads on a fresh run: +datas += [(_p("atlas/interface/web"), "atlas/interface/web")] +datas += [(_p("atlas/config/settings.example.json"), "atlas/config")] +datas += [(_p("atlas/connectors/registry.example.json"), "atlas/connectors")] +datas += [(_p("atlas/VERSION.json"), "atlas")] +datas += [(_p("atlas/core"), "atlas/core")] +datas += [(_p("atlas/memory/vault"), "atlas/memory/vault")] +datas += [(_p("atlas/orchestration/config.json"), "atlas/orchestration")] +datas += collect_data_files("anthropic") + +hiddenimports = [] +hiddenimports += collect_submodules("uvicorn") +hiddenimports += collect_submodules("anthropic") +hiddenimports += ["atlas.server.app", "atlas.server.__main__"] + +a = Analysis( + [_p("deploy/atlas_core_entry.py")], # absolute-import launcher (keeps package context) + pathex=[ROOT], + binaries=[], + datas=datas, + hiddenimports=hiddenimports, + excludes=["tkinter", "matplotlib", "onnxruntime", "pywhispercpp", "openwakeword"], + noarchive=False, +) +pyz = PYZ(a.pure) +exe = EXE( + pyz, a.scripts, a.binaries, a.datas, [], + name="atlas-core", + console=True, # stdout carries the ATLAS_READY line the shell reads + strip=False, + upx=False, + target_arch=None, +) diff --git a/deploy/atlas_core_entry.py b/deploy/atlas_core_entry.py new file mode 100644 index 0000000..7b865f9 --- /dev/null +++ b/deploy/atlas_core_entry.py @@ -0,0 +1,7 @@ +"""PyInstaller entry point for the frozen sidecar. Uses an ABSOLUTE import so the +`atlas` package context is intact (running atlas/server/__main__.py directly would +break its `from .. import ...` relative imports).""" +from atlas.server.__main__ import main + +if __name__ == "__main__": + main() diff --git a/deploy/build-app.sh b/deploy/build-app.sh new file mode 100755 index 0000000..8168f26 --- /dev/null +++ b/deploy/build-app.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Build the native ATLAS desktop app (Tauri shell + frozen FastAPI sidecar). +# +# ./deploy/build-app.sh +# +# Produces src-tauri/target/release/bundle/ (.app + .dmg on macOS). +# Prereqs (installed once): Rust (rustup), Tauri CLI (`cargo install tauri-cli --version ^2`), +# PyInstaller (`pip install pyinstaller`). Signing/notarization is CI's job (see +# .github/workflows/build-app.yml) and needs the owner's Apple Developer cert. +set -euo pipefail +cd "$(dirname "$0")/.." + +if [ ! -f src-tauri/icons/icon.icns ]; then + echo "✗ App icons missing. Generate them once from a square PNG logo:" + echo " ( cd src-tauri && cargo tauri icon ../path/to/atlas-logo.png )" + exit 1 +fi + +echo "==> 1/3 Freezing the FastAPI core with PyInstaller" +pyinstaller --clean -y deploy/atlas-core.spec + +echo "==> 2/3 Placing the sidecar with its target-triple suffix" +TRIPLE="$(rustc -vV | awk -F': ' '/host/{print $2}')" +mkdir -p src-tauri/binaries +cp "dist/atlas-core" "src-tauri/binaries/atlas-core-${TRIPLE}" +chmod +x "src-tauri/binaries/atlas-core-${TRIPLE}" +echo " → src-tauri/binaries/atlas-core-${TRIPLE}" + +echo "==> 3/3 Building the Tauri app" +( cd src-tauri && cargo tauri build ) + +echo "✓ Done. Bundle in src-tauri/target/release/bundle/" diff --git a/docs/DEV_CAPABILITY_DESIGN.md b/docs/DEV_CAPABILITY_DESIGN.md new file mode 100644 index 0000000..7e78673 --- /dev/null +++ b/docs/DEV_CAPABILITY_DESIGN.md @@ -0,0 +1,199 @@ +# ATLAS Develop + Create/Organize Folders — Design Doc + +Status: **proposal for owner review** (no code written yet). +Informed by research on **Hermes Agent** (NousResearch) and **OpenClaw** (formerly Clawbot). + +--- + +## 1. Goal + +Let ATLAS autonomously **develop** (write + run code) and **create/organize folders**, safely, on the Mac Mini — usable by voice, working with both cloud and local LLMs. This is a **purely additive** capability: it reuses ATLAS's existing L6 tool layer (`orchestration/tools.py`) and L7 permission gate (`connectors/loader.py`) with **no architecture change**. + +## 2. Decisions (locked with owner) + +| Decision | Choice | +|---|---| +| Workspace location | **`~/ATLAS-workspace`** — its own git repo, outside the ATLAS source and MemVault | +| Enable model | **Native, always on** — no connector switch; risky tools still confirm each time | +| Network during `run` | **Allowed** — enables `pip`/`npm`/`git fetch`; exfil defenses hardened to compensate | +| Rollout | **Full design first** (this doc), then phased build with per-phase owner approval | + +## 3. Principles taken from Hermes & OpenClaw + +- **Typed FILE tools, separate from a SHELL/exec tool** (Hermes). Weaker/local models are far more reliable with `read_file`/`write_file`/`edit_file` returning diffs than heredoc-ing into a shell. +- **Two independent gates**: permission (human-in-the-loop) **and** an OS sandbox. ATLAS has the first; we add the second. +- **Un-bypassable hardline blocklist** + **secrets denylist** (Hermes) — some things are refused even when "confirmed". +- **git + markdown workspace, plan-then-apply, move-never-delete** (OpenClaw) — undo is one command; destructive ops are always a two-step "speak the plan → say yes". +- **SKILL.md procedural memory with progressive disclosure** (both) — learned procedures cost almost no context until needed. +- **Short, hand-curated context files** — bloated auto-written `AGENTS.md` measurably *lowers* success; keep them ~15 lines. + +## 4. Workspace model + +One config-driven root, default **`~/ATLAS-workspace/`** (created on first use, its **own git repo**, separate from both the ATLAS product repo and the MemVault repo). + +``` +~/ATLAS-workspace/ + .git/ # git-backed undo (reuse VaultStore.init_git / _git_commit) + AGENTS.md # SHORT, hand-curated workspace rules (setup/test cmds, do-not-touch) + projects// # one folder per project (slug = lowercase-hyphen-ascii, deduped) + README.md AGENTS.md .gitignore LICENSE(AGPL-3.0) + src|pkg/ tests/ Makefile + .venv/ # per-project virtualenv (pip installs land here) + scratch/ # throwaway experiments + .trash/ # soft-deletes land here; ATLAS never `rm`s + skills/ # SKILL.md folders (procedural memory) + .pending/ # self-authored skills await owner review before activation +``` + +- **Git-backed undo**: every WRITE/DESTRUCTIVE fs op **auto-commits the current state first** (aider pattern — you never lose work), enabling a voice-friendly `workspace_undo`. +- **Per-project git init + checkpoint** at creation, so per-project undo works too. +- Templates ship at `atlas/orchestration/templates/{python,node,static,blank}/` (committed, no secrets). +- Boundary discipline: the workspace holds only owner project code + markdown. ATLAS's own config/credentials/registry stay in the ATLAS repo and are **unreachable from the jail** (see §6). + +## 5. Tool set (~14 new tools) + +All are `_t_(args) -> str` handlers on `ToolBox`, dispatched and gated **exactly as today**. Every file/dev handler calls `fsjail.resolve()` on each path first. Risk tag drives approval: **READ** auto-runs · **WRITE** runs then confirms-after (diff shown first) · **DESTRUCTIVE** blocked until you say "yes". + +### FILE (Hermes-style) +| Tool | Risk | Args | Behavior | +|---|---|---|---| +| `list_dir` | READ | `{path?}` | Tree/ls of a workspace dir | +| `read_file` | READ | `{path, offset?, limit?}` | Text with line numbers + pagination | +| `search_files` | READ | `{query, target?:content\|files, regex?}` | ripgrep-backed grep / find | +| `write_file` | WRITE | `{path, content}` | Writes/overwrites whole file, makes parent dirs. **New file** = plain WRITE; **overwrite** returns a unified diff and needs confirmation | +| `edit_file` | WRITE | `{path, old_string, new_string}` | Targeted replace, returns a diff. **Refuses if `old_string` isn't unique** (guards the Hermes patch-corruption failure mode) | +| `create_dir` | WRITE | `{path}` | `mkdir -p` inside the jail | + +### DEV / SCAFFOLD +| Tool | Risk | Args | Behavior | +|---|---|---|---| +| `create_project` | WRITE | `{name, kind?:python\|node\|static\|blank, description?}` | Scaffolds `projects//` from a template, `git init` + checkpoint | +| `run` | DESTRUCTIVE | `{command, cwd?, timeout?, background?}` | Runs a shell/build/test command, cwd pinned in the workspace, wrapped in Seatbelt, captured stdout/stderr (byte-capped) + exit code. The single develop+run primitive (covers `git`, `npm`, `pytest`, `pip install`, …) | +| `run_python` | DESTRUCTIVE | `{script, timeout?}` | Writes a temp script inside the jail, runs it under Seatbelt with a scrubbed env | + +### ORGANIZE (plan/apply — voice-first) +| Tool | Risk | Args | Behavior | +|---|---|---|---| +| `organize_plan` | READ | `{path}` | Classifies files, returns a spoken-friendly manifest (`file → category/ · reason`), stashes it on `ToolBox.last_organize` (mirrors `last_media`). ATLAS **speaks** the plan | +| `organize_apply` | DESTRUCTIVE | `{}` | Consumes the stashed manifest; **move-only** into a sibling `-organized/` dir (source untouched), dedupes, writes `manifest.json` + `.undo-moves.sh`, git checkpoint. **Never deletes** | + +### UNDO / SKILLS +| Tool | Risk | Args | Behavior | +|---|---|---|---| +| `workspace_undo` | WRITE | `{}` | `git revert`/`reset --hard` to the pre-action checkpoint — one-command voice undo | +| `skill_view` | READ | `{name, path?}` | Loads a full `SKILL.md` (or a reference file) on demand | +| `skill_manage` | WRITE | `{action:create\|edit\|write_file, ...}` | Authors/edits a SKILL.md; self-authored skills stage to `skills/.pending/` | + +## 6. Safety model — TWO independent gates + +ATLAS today has **Gate 1** only. This adds **Gate 2** plus a path jail. + +### Gate 1 — permission (reuse `registry.gate` verbatim) +- New tools get `risk` tags in `config.json`; `DESTRUCTIVE`-without-`confirmed` is already refused (`loader.py:59`). +- **Hardline blocklist** (checked inside destructive handlers; `confirmed=true` can NEVER bypass): `rm -rf /`, fork bombs, `dd` to `/dev/*`, `mkfs`, `curl|sh`/`wget|sh` pipe-to-shell. +- **Inline-eval always-confirm**: `python -c`, `node -e`, `bash -c`, `osascript -e` always require confirmation even if the interpreter is allowed. +- **Secrets write-denylist**: refuse writes to any `.env`/`.env.*`/`.envrc`, `~/.ssh`, `~/.aws`, dotfiles, and ATLAS's own `settings.json`/`registry.json`/`owner.md`. +- **User deny-glob list** from `settings.json`. Invariant: *approvals can only tighten, never loosen* (effective = stricter of config + gate). + +### Path jail — `atlas/orchestration/fsjail.py` (new) +- `resolve(p)` = `Path(p).resolve()` (realpath, follows symlinks) then require `.is_relative_to(WORKSPACE_ROOT)`. +- Re-run on **every** call **and after any write** (TOCTOU: an agent that can write can plant an escaping symlink, so a lexical check alone fails). +- Absolute paths outside the root are refused, not silently allowed. + +### Gate 2 — OS sandbox — `atlas/orchestration/sandbox.py` (new) +- `run`/`run_python` execute under **Apple Seatbelt** via `/usr/bin/sandbox-exec -p ` (verified present on this M4 — same approach as Claude Code/Goose). +- Behind a **`SandboxRunner` abstraction** (`run(cmd, cwd, allow_net, timeout)`) so a Docker tier or a post-deprecation replacement drops in. `sandbox-exec` is officially deprecated (still present) — never couple handlers directly to it. +- Backends: `SeatbeltBackend` (default), `DockerBackend` (stub/Phase 5), `NoneBackend` (explicit owner opt-out, never default). + +### Network is ALLOWED — so exfil defenses are the load-bearing protection +Because `run` can reach the network (for `pip`/`npm`), the sandbox can't rely on network isolation. Instead it **prevents secrets from being *read* in the first place**, plus scrubs the environment: +- **Seatbelt denies reads** of `~/.ssh`, `~/.aws`, `~/.gnupg`, `~/.config`, `~/Library/Keychains`, `~/.netrc`, `~/.docker`, the ATLAS repo's `atlas/config/`, and the MemVault dir — even though network is open. +- **Env scrub**: drop any var matching `*KEY*`/`*TOKEN*`/`*SECRET*`/`*PASSWORD*`/`*CREDENTIAL*`/`*AUTH*` before exec. +- **Writes** confined to the workspace + a temp dir. +- Future hardening (not v1): route outbound through a local proxy for a **domain allowlist** (Seatbelt can't filter by domain, only allow/deny sockets). + +### Illustrative Seatbelt profile (finalized + tested in Phase 2) +```scheme +(version 1) +(deny default) +(allow process-fork process-exec) +(allow signal (target self)) +(allow sysctl-read mach-lookup) +;; reads: system + workspace + temp +(allow file-read* + (subpath "/usr") (subpath "/System") (subpath "/Library") + (subpath "/bin") (subpath "/sbin") (subpath "/opt") + (subpath "/private/var") (subpath "/private/tmp") + (subpath "$WORKSPACE_ROOT") (subpath "$TMPDIR")) +;; …but NEVER these (last-match-wins → secrets denied even with network open) +(deny file-read* + (subpath "$HOME/.ssh") (subpath "$HOME/.aws") (subpath "$HOME/.gnupg") + (subpath "$HOME/.config") (subpath "$HOME/Library/Keychains") + (literal "$HOME/.netrc") (subpath "$HOME/.docker") + (subpath "$ATLAS_REPO/atlas/config") (subpath "$MEMVAULT")) +;; writes: workspace + temp only +(allow file-write* (subpath "$WORKSPACE_ROOT") (subpath "$TMPDIR") (subpath "/private/tmp")) +;; network: ALLOWED per owner decision +(allow network*) +``` + +### Resource limits + audit +- Per-call **timeout** (default 120s, SIGTERM→SIGKILL), max stdout/stderr **byte caps**. +- **Per-task tool-call budget** in `router.py` (the two hardcoded `for _ in range(5)` loops at lines 274/355 become a config-driven budget, default 30, with a near-cap warning). +- **Fail-closed**: an unanswered/again-unconfirmed destructive call is DENIED, not run. +- **Audit line** per fs op / execution via the existing `log_event()` (tool, resolved path, bytes, exit code, `confirmed` flag). + +## 7. Skills & learning loop + +SKILL.md-as-folders with **progressive disclosure**, mapped onto MemVault so it costs the local model almost no context: +- `skills_list` — a ~2–3KB index (name + description) injected into `_system_prompt()` alongside the existing vault `context_block()`. Directly helps weaker local models (prose how-to instead of more JSON). +- `skill_view(name)` / `skill_view(name, path)` — load a full skill only when needed. +- Skills are **gated by `requires.bins/env/os`** reusing the existing `openai_tools()` omission mechanism (a skill whose deps don't match is simply not listed). +- **Authoring**: after a complex task (5+ tool calls / error-recovery / novel workflow), ATLAS may write a SKILL.md via `skill_manage`. **Self-authored skills stage to `skills/.pending/` and require owner review before activation** (never auto-activate — 26% of community skills studied had a vulnerability). +- **Promotion** hooks into the L8 nightly `consolidate` job (only with prior owner opt-in). +- "Memory = small durable facts always in context (MemVault); skills = longer procedures loaded only when relevant." + +## 8. Phased plan (each phase shippable + testable; local-test → owner approval per dev-flow) + +- **Phase 0 — Jail + workspace + READ tools** (nothing executes). `fsjail.py` (+ unit tests for `../` and symlink escape), `workspace` settings block, `list_dir`/`read_file`/`search_files`, create `~/ATLAS-workspace` + `git init`. +- **Phase 1 — Write + scaffold + git undo** (WRITE tier, still no shell). `write_file`/`edit_file`/`create_dir`/`create_project` + templates + `workspace_undo`; diff-before-write; auto-commit-first. +- **Phase 2 — SandboxRunner + `run`/`run_python`** (the real dev capability). `sandbox.py` (Seatbelt SBPL, **network allowed**, secrets read-deny, env scrub, timeout, byte caps); hardline blocklist + inline-eval confirm + secrets write-deny; tool-call budget; audit line. +- **Phase 3 — Organize pipeline** (voice payoff). `organize_plan` (READ, stash manifest) + `organize_apply` (DESTRUCTIVE, move-only, `.trash/`, `.undo-moves.sh`, checkpoint). Persist org taxonomy to MemVault. +- **Phase 4 — Skills + learning loop.** `skills_list` in the system prompt, `skill_view` + `skill_manage`, requires-gating, `.pending/` review, nightly promotion hook. +- **Phase 5 (defer)** — Docker tier, Hermes-style `execute_code` RPC, background `process` poll/kill, subagent delegation, outbound domain-allowlist proxy. + +## 9. Exact ATLAS wiring (verified integration points) + +1. `atlas/orchestration/config.json` — add ~14 `tool_schema[]` entries `{name, risk, description, input_schema}`. No `connector` key (native, per decision). +2. `atlas/orchestration/tools.py` — add `_t_` handlers on `ToolBox`; add `self.last_organize` stash (mirror `self.last_media`, line 53). No change to `dispatch()`. +3. **NEW** `atlas/orchestration/fsjail.py` — path jail + hardline blocklist + secrets/inline-eval matchers (pure, unit-tested). +4. **NEW** `atlas/orchestration/sandbox.py` — `SandboxRunner` + `SeatbeltBackend`/`NoneBackend`/`DockerBackend(stub)`. +5. **NEW** `atlas/orchestration/templates/{python,node,static,blank}/` — committed scaffolds (no secrets). +6. `atlas/config/settings.json` + `settings.example.json` — add a `workspace` block: `{"root":"~/ATLAS-workspace","sandbox_backend":"seatbelt","allow_network":true,"run_timeout_s":120,"max_output_bytes":65536,"deny_globs":[],"tool_call_budget":30}`. (`settings.json` stays gitignored; the example carries defaults.) +7. `atlas/settings.py` — add `workspace_root()` accessor (expand `~`, honor absolute), matching `vault_dir()`. +8. `atlas/orchestration/router.py` — replace the two `for _ in range(5)` tool loops (lines 274, 355) with the config-driven budget + near-cap warning. `confirmed` already flows router→dispatch→gate. +9. `atlas/evaluation/logger.py` — audit line per fs op / execution via `log_event()`. +10. `atlas/scheduler/jobs.py` — extend the `consolidate` handler (JOBS line 154) to review `.pending/` skills nightly (plan-only unless pre-approved). +11. Git-undo helper — reuse the `VaultStore.init_git()` / `_git_commit()` pattern (`memory/store.py:510/529`) in a small workspace-repo helper. + +## 10. Risks & mitigations + +| Risk | Mitigation | +|---|---| +| `sandbox-exec` deprecated | `SandboxRunner` abstraction; Docker/None tiers; settings switch | +| TOCTOU / symlink escape | `Path.resolve` re-run every call + after writes; Seatbelt backstop | +| **Network on → exfil channel** | Seatbelt **denies reads** of secrets; env scrub; writes confined; future domain-allowlist proxy | +| `edit_file` ambiguous-match corruption | refuse when `old_string` count ≠ 1; prefer `write_file`+diff | +| Weak local models misuse `run` | optional per-model omission / force-confirm; `create_project`/`read_file` safe everywhere | +| Prompt-injection via a fetched/scaffolded file | secrets read-deny + env scrub even with net on; treat context files as untrusted | +| Runaway loops | per-call timeout + byte caps + per-task tool budget + fork-bomb blocklist | +| Scheduled autonomy writing/running code | scheduled jobs use the SAME jail+Seatbelt+git-undo path; default plan-only | +| Public-repo secret leak | workspace/deny_globs/timeouts live in gitignored `settings.json`, not code; audit every diff | + +## 11. Remaining open questions + +1. **Package installs**: `pip install`/`npm install` run as `run` (DESTRUCTIVE, confirmed) into a per-project `.venv`/`node_modules`. OK, or do you want an explicit `install` tool with its own confirmation copy? +2. **Local-model bar**: which local models (if any) should have `run`/`run_python` hidden or force-always-confirm? +3. **Self-authored skill promotion**: keep OFF by default (you review `.pending/` manually)? Recommended yes. +4. **Tool-call budget**: 30 per task (up from 5)? Warn aloud near the cap on voice turns? +5. **`organize_apply` outside the workspace** (e.g. real `~/Downloads`): confined-to-workspace in v1, with a separate approved-paths list later? Recommended yes. diff --git a/requirements.txt b/requirements.txt index 95f1d3f..6a56bf1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,5 +21,14 @@ pytest>=8.0 # pip install kokoro-onnx soundfile "misaki[en]" # then download kokoro-v1.0.onnx + voices-v1.0.bin into # atlas/interface/voice/models/ (github.com/thewh1teagle/kokoro-onnx releases). -# STT / wake word (real hardware): browser Web Speech today; openwakeword, -# silero-vad, whisper.cpp later. These need a mic, not present in CI. +# Native voice pipeline (L3, opt-in via voice.native_audio) — replaces the browser +# Web Speech loop with an on-device, offline pipeline: mic → wake → STT → Kokoro. +# Optional & heavy (like Kokoro); ATLAS falls back to browser Web Speech without +# them. macOS needs PortAudio (`brew install portaudio`) for sounddevice. +# pip install sounddevice numpy openwakeword onnxruntime pywhispercpp +# Models auto-download on first run: openWakeWord (hey_jarvis + Silero VAD) and +# whisper.cpp (base.en). Not installed in CI (no mic); tests inject fakes. +# Cross-platform secret store (optional): on Windows/Linux the API-key store falls +# back to the OS keyring (Credential Manager / Secret Service) when the macOS +# `security` CLI isn't present. pip install keyring +# Native app packaging (build-time only): pip install pyinstaller (+ Rust/Tauri CLI). diff --git a/src-tauri/.gitignore b/src-tauri/.gitignore new file mode 100644 index 0000000..83fe972 --- /dev/null +++ b/src-tauri/.gitignore @@ -0,0 +1,5 @@ +# Tauri / Rust build artifacts and the frozen sidecar (all rebuilt by build-app.sh) +/target +/gen +/binaries +Cargo.lock diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml new file mode 100644 index 0000000..664c00b --- /dev/null +++ b/src-tauri/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "atlas-app" +version = "0.6.0" +description = "ATLAS — voice-first local assistant (native shell)" +edition = "2021" +rust-version = "1.77" + +[build-dependencies] +tauri-build = { version = "2", features = [] } + +[dependencies] +tauri = { version = "2", features = ["tray-icon"] } +tauri-plugin-shell = "2" +tauri-plugin-single-instance = "2" +tauri-plugin-autostart = "2" +tauri-plugin-positioner = { version = "2", features = ["tray-icon"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" + +[profile.release] +codegen-units = 1 +lto = true +opt-level = "s" +panic = "abort" +strip = true diff --git a/src-tauri/Entitlements.plist b/src-tauri/Entitlements.plist new file mode 100644 index 0000000..255615f --- /dev/null +++ b/src-tauri/Entitlements.plist @@ -0,0 +1,22 @@ + + + + + + com.apple.security.cs.allow-jit + + com.apple.security.cs.allow-unsigned-executable-memory + + com.apple.security.cs.disable-library-validation + + com.apple.security.device.audio-input + + com.apple.security.network.client + + com.apple.security.network.server + + + diff --git a/src-tauri/Info.plist b/src-tauri/Info.plist new file mode 100644 index 0000000..06bce76 --- /dev/null +++ b/src-tauri/Info.plist @@ -0,0 +1,13 @@ + + + + + + NSMicrophoneUsageDescription + ATLAS listens for the wake word and your voice commands, on-device. + LSUIElement + + + diff --git a/src-tauri/build.rs b/src-tauri/build.rs new file mode 100644 index 0000000..d860e1e --- /dev/null +++ b/src-tauri/build.rs @@ -0,0 +1,3 @@ +fn main() { + tauri_build::build() +} diff --git a/src-tauri/capabilities/default.json b/src-tauri/capabilities/default.json new file mode 100644 index 0000000..771913a --- /dev/null +++ b/src-tauri/capabilities/default.json @@ -0,0 +1,19 @@ +{ + "$schema": "../gen/schemas/desktop-schema.json", + "identifier": "default", + "description": "ATLAS shell: spawn the atlas-core sidecar, tray, autostart, positioner.", + "windows": ["main"], + "permissions": [ + "core:default", + "core:window:allow-show", + "core:window:allow-hide", + "core:window:allow-set-focus", + "shell:allow-execute", + { + "identifier": "shell:allow-spawn", + "allow": [{ "name": "binaries/atlas-core", "sidecar": true, "args": true }] + }, + "positioner:default", + "autostart:default" + ] +} diff --git a/src-tauri/icons/128x128.png b/src-tauri/icons/128x128.png new file mode 100644 index 0000000..24eb663 Binary files /dev/null and b/src-tauri/icons/128x128.png differ diff --git a/src-tauri/icons/128x128@2x.png b/src-tauri/icons/128x128@2x.png new file mode 100644 index 0000000..d3fdc08 Binary files /dev/null and b/src-tauri/icons/128x128@2x.png differ diff --git a/src-tauri/icons/32x32.png b/src-tauri/icons/32x32.png new file mode 100644 index 0000000..9c98123 Binary files /dev/null and b/src-tauri/icons/32x32.png differ diff --git a/src-tauri/icons/icon.icns b/src-tauri/icons/icon.icns new file mode 100644 index 0000000..7c35da9 Binary files /dev/null and b/src-tauri/icons/icon.icns differ diff --git a/src-tauri/icons/icon.ico b/src-tauri/icons/icon.ico new file mode 100644 index 0000000..fe1b94d Binary files /dev/null and b/src-tauri/icons/icon.ico differ diff --git a/src-tauri/icons/icon.png b/src-tauri/icons/icon.png new file mode 100644 index 0000000..47bf702 Binary files /dev/null and b/src-tauri/icons/icon.png differ diff --git a/src-tauri/src/main.rs b/src-tauri/src/main.rs new file mode 100644 index 0000000..a883bfb --- /dev/null +++ b/src-tauri/src/main.rs @@ -0,0 +1,112 @@ +// ATLAS native shell (Tauri 2). Personal build: runs the owner's real ATLAS +// install (venv + repo data + full voice stack) as the backend, so the app is the +// complete experience — real memory, settings, and on-device voice (Kokoro + +// wake/STT) — in a native window. (A self-contained frozen-sidecar build for +// distribution is a separate path; see deploy/build-app.sh + atlas-core.spec.) +#![cfg_attr(not(debug_assertions), windows_subsystem = "windows")] + +use std::io::{BufRead, BufReader}; +use std::process::{Child, Command, Stdio}; +use std::sync::Mutex; +use tauri::menu::{Menu, MenuItem}; +use tauri::tray::TrayIconBuilder; +use tauri::{Manager, RunEvent, WebviewUrl, WebviewWindowBuilder, WindowEvent}; + +// The owner's ATLAS checkout (this build targets this machine). +const ATLAS_HOME: &str = "/Users/atlas/ATLAS"; + +struct Backend(Mutex>); + +fn show_main(app: &tauri::AppHandle) { + if let Some(w) = app.get_webview_window("main") { + let _ = w.show(); + let _ = w.set_focus(); + } +} + +fn main() { + tauri::Builder::default() + .plugin(tauri_plugin_single_instance::init(|app, _argv, _cwd| { + show_main(app); + })) + .plugin(tauri_plugin_positioner::init()) + .plugin(tauri_plugin_autostart::init( + tauri_plugin_autostart::MacosLauncher::LaunchAgent, + None, + )) + .manage(Backend(Mutex::new(None))) + .setup(|app| { + let handle = app.handle().clone(); + + // Start the real ATLAS core (venv python) on a free port; it prints + // "ATLAS_READY " once serving. + let py = format!("{ATLAS_HOME}/.venv/bin/python"); + let mut child = Command::new(&py) + .args(["-u", "-m", "atlas.server", "--port", "0"]) + .current_dir(ATLAS_HOME) + .env("ATLAS_SUPERVISED", "1") + .stdout(Stdio::piped()) + .spawn() + .expect("failed to start ATLAS core (venv python)"); + let stdout = child.stdout.take().expect("no stdout"); + app.state::().0.lock().unwrap().replace(child); + + std::thread::spawn(move || { + let reader = BufReader::new(stdout); + for line in reader.lines().map_while(Result::ok) { + if let Some(rest) = line.trim().strip_prefix("ATLAS_READY ") { + if let Ok(port) = rest.trim().parse::() { + let h = handle.clone(); + let url = format!("http://127.0.0.1:{port}"); + let _ = handle.run_on_main_thread(move || { + if h.get_webview_window("main").is_none() { + let _ = WebviewWindowBuilder::new( + &h, + "main", + WebviewUrl::External(url.parse().unwrap()), + ) + .title("ATLAS") + .inner_size(1280.0, 860.0) + .build(); + } + }); + break; + } + } + } + }); + + let quit = MenuItem::with_id(app, "quit", "Quit ATLAS", true, None::<&str>)?; + let menu = Menu::with_items(app, &[&quit])?; + let _tray = TrayIconBuilder::new() + .icon(app.default_window_icon().unwrap().clone()) + .icon_as_template(true) + .menu(&menu) + .show_menu_on_left_click(false) + .on_menu_event(|app, event| { + if event.id.as_ref() == "quit" { + app.exit(0); + } + }) + .on_tray_icon_event(|tray, _event| show_main(tray.app_handle())) + .build(app)?; + + Ok(()) + }) + // Closing the window QUITS ATLAS (stops the backend + voice) — no silent + // hide-to-tray that keeps talking. Quit is also on the tray menu. + .on_window_event(|window, event| { + if let WindowEvent::CloseRequested { .. } = event { + window.app_handle().exit(0); + } + }) + .build(tauri::generate_context!()) + .expect("error building ATLAS") + .run(|app, event| { + if let RunEvent::ExitRequested { .. } = event { + if let Some(mut child) = app.state::().0.lock().unwrap().take() { + let _ = child.kill(); + } + } + }); +} diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json new file mode 100644 index 0000000..84d20ee --- /dev/null +++ b/src-tauri/tauri.conf.json @@ -0,0 +1,34 @@ +{ + "$schema": "https://schema.tauri.app/config/2", + "productName": "ATLAS", + "version": "0.6.0", + "identifier": "ai.atlas.assistant", + "build": { + "frontendDist": "../atlas/interface/web" + }, + "app": { + "withGlobalTauri": true, + "windows": [], + "security": { + "csp": null + } + }, + "bundle": { + "active": true, + "targets": ["app", "dmg"], + "icon": [ + "icons/32x32.png", + "icons/128x128.png", + "icons/128x128@2x.png", + "icons/icon.icns", + "icons/icon.ico" + ], + "macOS": { + "minimumSystemVersion": "12.0", + "entitlements": "Entitlements.plist" + }, + "windows": { + "webviewInstallMode": { "type": "downloadBootstrapper" } + } + } +}