Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 51 additions & 2 deletions examples/god/god.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -178,9 +178,51 @@ const ACTION_PROTOCOL =
"Prefer OPEN, RUN, or POINT over raw CLICK/KEY when a cleaner route exists. Never propose a " +
"destructive action. If no action is warranted, just use [POINT:x,y:label] or no tag.";

// ── guided FORM FILL (docs/FORM-FILL.md's "upgrade path", now built) ───────────────────────────
// "Help me fill this form": God maps the form's fields (the attached clipboard/file reference, or the
// screen) to values it actually HAS — the user's saved identity.json details + the active project's
// context + anything the user said — and emits ONE [FILLGUIDE] tag. Swift turns it into the native
// teach fill-guide: one step per field, that value pre-loaded on the clipboard, the USER pastes each
// (⌘V) — God never types into the form itself. Deliberately read fresh per turn, like identity edits.
function readIdentity() {
try {
const id = JSON.parse(readFileSync(join(REAL_RELAY, "identity.json"), "utf8"));
const kept = Object.entries(id).filter(([, v]) => typeof v === "string" && v.trim());
return kept.length ? Object.fromEntries(kept) : null;
} catch { return null; }
}
function fillProtocol() {
const id = readIdentity();
return (
"\n\nGUIDED FORM FILL — when the user asks you to help FILL a form (its text is usually an attached " +
"clipboard/file reference, or on screen), do NOT [TYPE] into it. Instead end your reply with ONE tag:\n" +
' [FILLGUIDE:[["<field label exactly as the form shows it>","<the value>"],…]]\n' +
"listing the fields in the form's own order. Rules: include ONLY fields you have a REAL value for — " +
"from the saved details below, the active project's context, or something the user told you. NEVER " +
"invent or guess a value; skip fields you don't know. NEVER include passwords, card numbers, or other " +
"secrets — those stay manual. The tag raises a native step-by-step guide where each value is pre-loaded " +
"on the clipboard and the user pastes it themselves. Keep your spoken line to one short sentence " +
"(\"Got it — walking you through the fields.\")." +
(id ? "\nSAVED DETAILS (the user's own, fine to use): " + JSON.stringify(id) : "")
);
}

// Parse the ONE action/point tag off the reply. Priority: an explicit RUN or LOCAL action over a
// bare point. RUN captures a tool name then optional JSON args (or a bare string → {input:string}).
function parseAction(text) {
// [FILLGUIDE:[["label","value"],…]] — parsed FIRST (its JSON body contains "]" so the other tags'
// [^\]]+ bodies must never get a crack at it). Greedy body → the LAST "]…]" pair closes the tag.
const fill = /\[FILLGUIDE:\s*(\[[\s\S]*\])\s*\]/i.exec(text);
if (fill) {
try {
const arr = JSON.parse(fill[1]);
const fields = (Array.isArray(arr) ? arr : [])
.map((e) => Array.isArray(e) ? { label: String(e[0] ?? "").trim(), value: String(e[1] ?? "").trim() }
: e && typeof e === "object" ? { label: String(e.label ?? "").trim(), value: String(e.value ?? "").trim() } : null)
.filter((f) => f && f.label && f.value);
if (fields.length) return { kind: "fillguide", fields };
} catch { /* malformed JSON → fall through to the other tags */ }
}
// [DRIVE:<wrapp> <input>] or [DRIVE:<wrapp>:<command> <input>] — the optional :command lets God pick
// one of a multi-command wrapp's tools from the registry; without it the native side auto-discovers.
const drive = /\[DRIVE:\s*([a-z0-9_-]+)(?::([a-z0-9_]+))?\s+([^\]]+)\]/i.exec(text);
Expand All @@ -206,7 +248,7 @@ function parseToolArgs(raw) {
try { const v = JSON.parse(raw); return v && typeof v === "object" ? v : { input: String(v) }; }
catch { return { input: raw }; }
}
const stripTags = (t) => t.replace(/\[(?:OPEN|TYPE|CLICK|KEY|POINT|DRIVE):[^\]]*\]/gi, "").replace(/\[RUN:[\s\S]*?\]/gi, "").trim();
const stripTags = (t) => t.replace(/\[FILLGUIDE:\s*\[[\s\S]*\]\s*\]/gi, "").replace(/\[(?:OPEN|TYPE|CLICK|KEY|POINT|DRIVE):[^\]]*\]/gi, "").replace(/\[RUN:[\s\S]*?\]/gi, "").trim();

// The desktop's size in POINTS (screencapture gives PIXELS); the ratio maps image coords → clickable
// screen points on retina. Cheap, non-prompting.
Expand All @@ -217,6 +259,7 @@ function screenPointsSize() {
}
function prettyTool(name) { return String(name).replace(/^mcp__[^_]+__/, "").replace(/^wrapp__/, "").replace(/__/g, " · "); }
function describeAction(a, shot) {
if (a.kind === "fillguide") return `guide you through filling ${a.fields.length} field${a.fields.length === 1 ? "" : "s"}`;
if (a.kind === "drive") return `drive the ${a.wrapp} wrapp — “${String(a.input).slice(0, 50)}”`;
if (a.kind === "open") return `open ${a.target}`;
if (a.kind === "type") return `type: “${a.text.slice(0, 60)}”`;
Expand Down Expand Up @@ -256,6 +299,12 @@ async function runAction(a, shot, reg) {
try { writeFileSync(join(REAL_RELAY, "god-action.json"), JSON.stringify({ ...a, describe: describeAction(a, shot) })); } catch { /* best effort */ }
return `handed “drive ${a.wrapp}” to the notch — the widget takes it from here`;
}
if (a.kind === "fillguide") {
// Swift owns the guide runtime (CursorGuide): hand the field→value pairs off and exit; the app
// raises the teach fill-guide (each value pre-loaded on the clipboard, the user pastes each ⌘V).
try { writeFileSync(join(REAL_RELAY, "god-action.json"), JSON.stringify({ ...a, describe: describeAction(a, shot) })); } catch { /* best effort */ }
return `raising the fill guide — ${a.fields.length} field${a.fields.length === 1 ? "" : "s"}, each ready to paste`;
}
if (a.kind === "open") { spawnSync("open", openArgs(a.target)); return `opened ${a.target}`; }
if (a.kind === "type") { spawnSync("osascript", ["-e", `tell application "System Events" to keystroke ${JSON.stringify(a.text)}`]); return `typed`; }
if (a.kind === "key") { const osa = keyComboOsa(a.combo); if (!osa) return `couldn't parse key combo “${a.combo}”`; spawnSync("osascript", ["-e", osa]); return `pressed ${a.combo}`; }
Expand Down Expand Up @@ -832,7 +881,7 @@ async function ask(reg, persona, { instruction, useMic, region, act }) {
}
} catch (e) { log(`skill load skipped: ${e.message}`); }
const baseProtocol = noScreen ? NO_SCREEN_PROTOCOL : PROTOCOL;
const system = `${persona.characteristic}\n\n${baseProtocol}${nameLine}${projLine}${skillBlock}` + (act ? ACTION_PROTOCOL + runBlock : "") + catalogBlock();
const system = `${persona.characteristic}\n\n${baseProtocol}${nameLine}${projLine}${skillBlock}` + (act ? ACTION_PROTOCOL + fillProtocol() + runBlock : "") + catalogBlock();
if (proj) log(`project: ${proj.name}`);

log(`asking ${model} as ${persona.name}${dim(noScreen ? " (voice)" : " (vision)")}…`);
Expand Down
20 changes: 19 additions & 1 deletion packages/menubar/NotchLauncherView.swift
Original file line number Diff line number Diff line change
Expand Up @@ -366,7 +366,7 @@ struct NotchLauncherView: View {
// Opt-in — the clipboard rides along ONLY if the user taps Add. Adding writes it to a temp clipboard.txt
// and stages it exactly like a dropped file, so it flows through the SAME onLaunch(listing, url) path.
private func clipboardOfferRow(_ s: String) -> some View {
HStack(spacing: 0) {
HStack(spacing: 6) {
Spacer(minLength: 0)
Button(action: { withAnimation(.easeOut(duration: 0.16)) { addClipboard(s) } }) {
HStack(spacing: 6) {
Expand All @@ -383,8 +383,26 @@ struct NotchLauncherView: View {
.overlay(Capsule().stroke(Color.edge, lineWidth: 1))
}.buttonStyle(.plain).frame(maxWidth: 240)
.help("Add your clipboard as context for the app you launch")
// "Fill form" — the clipboard IS a form: hand it to the guided form-fill (docs/FORM-FILL.md).
// Writes the same ~/.relay/fill-form trigger the menu item / scripts use; the app's 1 s poll
// picks it up, matches identity.json against the copied form, and raises the teach fill-guide.
Button(action: { triggerFormFill() }) {
HStack(spacing: 4) {
Image(systemName: "square.and.pencil").font(.system(size: 8, weight: .bold))
Text("Fill form").font(.hanken(9.5, .semibold))
}.foregroundColor(.lime).padding(.horizontal, 8).padding(.vertical, 4)
.background(Capsule().fill(Color.white.opacity(0.05)))
.overlay(Capsule().stroke(Color.lime.opacity(0.45), lineWidth: 1))
}.buttonStyle(.plain)
.help("Guide me through filling this form (values from your saved details)")
}
}
// Fire the guided form-fill on the copied form and close the launcher (the guide takes the stage).
private func triggerFormFill() {
let p = (NSHomeDirectory() as NSString).appendingPathComponent(".relay/fill-form")
FileManager.default.createFile(atPath: p, contents: Data())
onClose()
}
private func clipPeek(_ s: String) -> String {
let flat = s.replacingOccurrences(of: "\n", with: " ").replacingOccurrences(of: "\r", with: " ")
.trimmingCharacters(in: .whitespaces)
Expand Down
51 changes: 41 additions & 10 deletions packages/menubar/RelayMenuBar.swift
Original file line number Diff line number Diff line change
Expand Up @@ -3672,22 +3672,31 @@ struct ActionConsentDrop: View {
let clip = (NSPasteboard.general.string(forType: .string) ?? "").lowercased()
guard !clip.isEmpty else { raiseFillNote("Copy the form first (⌘A then ⌘C), then try again."); return }
let id = readIdentity()
var steps: [[String: Any]] = []
var pairs: [(label: String, value: String)] = []
for f in fillFields {
guard let val = id[f.key], !val.isEmpty else { continue } // only fields I actually have
guard f.synonyms.contains(where: { clip.contains($0) }) else { continue } // the form mentions it
// Advance on the PASTE keystroke (reliable, no AX) OR the field filling — whichever first.
steps.append(["id": f.key, "text": "Click the \(f.label) field, then ⌘V",
"copy": val,
"doneWhen": ["any": [["kind": "pasted"], ["kind": "field-non-empty"]]]])
pairs.append((f.label, val))
}
guard !steps.isEmpty else {
guard !pairs.isEmpty else {
raiseFillNote("No fields I have data for matched this form. Add values in ~/.relay/identity.json.")
return
}
let run: [String: Any] = ["mode": "teach", "title": "Fill this form", "source": "Form fill",
"project": model.userName.isEmpty ? "" : "you", "autoClipboard": true, "steps": steps]
writeGuideRunFile(run)
raiseFillGuide(pairs, source: "Form fill")
}
// The one raiser BOTH fill paths share — the deterministic identity match above and God's
// [FILLGUIDE] hand (executeGodAction) — one teach step per (label, value), that value pre-loaded
// on the clipboard. Advance on the PASTE keystroke (reliable, no AX) OR the field filling.
@MainActor private func raiseFillGuide(_ pairs: [(label: String, value: String)], source: String) {
guard !pairs.isEmpty else { return }
let steps: [[String: Any]] = pairs.map { p in
["id": p.label.lowercased().replacingOccurrences(of: " ", with: "-"),
"text": "Click the \(p.label) field, then ⌘V",
"copy": p.value,
"doneWhen": ["any": [["kind": "pasted"], ["kind": "field-non-empty"]]]]
}
writeGuideRunFile(["mode": "teach", "title": "Fill this form", "source": source,
"project": model.userName.isEmpty ? "" : "you", "autoClipboard": true, "steps": steps])
}
// A one-step notch note (used for "copy the form first" / "no matches").
private func raiseFillNote(_ text: String) {
Expand Down Expand Up @@ -5126,7 +5135,10 @@ struct ActionConsentDrop: View {
// Driving an installed wrapp needs NO per-action consent — installing it WAS the consent
// (docs/GOD-HANDS.md #1); it runs straight into the notch. The wrapp's own write-class actions
// still hit the daemon gate. Only local hands God held a key for / risky actions keep the drop.
if (json["kind"] as? String) == "drive" { executeGodAction(json); return }
// A fill-guide rides the same lane: the user ASKED for it, and the guide itself is the consent
// surface — every value is visible per step, pasted by the user's own hand, esc aborts.
let kind = json["kind"] as? String
if kind == "drive" || kind == "fillguide" { executeGodAction(json); return }
showActionConsent(json["describe"] as? String ?? "do something", json)
}

Expand Down Expand Up @@ -5207,6 +5219,25 @@ struct ActionConsentDrop: View {
result: .text("God asked to drive “\(id)” but it isn't in the catalog. Install it from the store first.")),
onOpen: { [weak self] in self?.hideNotchWidget(); self?.showStore() })
}
case "fillguide":
// God mapped the form's fields to values ([FILLGUIDE] tag — docs/FORM-FILL.md upgrade path):
// raise the native teach fill-guide. Values ride the clipboard one step at a time, the USER
// pastes each; nothing is typed into the form by us. god.mjs normalizes fields to
// [{label, value}] before the handoff, so that's the only shape read here.
var pairs: [(label: String, value: String)] = []
for o in (a["fields"] as? [[String: Any]] ?? []) {
if let l = (o["label"] as? String)?.trimmingCharacters(in: .whitespaces),
let v = (o["value"] as? String)?.trimmingCharacters(in: .whitespaces),
!l.isEmpty, !v.isEmpty { pairs.append((l, v)) }
}
if pairs.isEmpty {
godLog("executeGodAction: fillguide with no usable fields — \(a)")
showNotchWidget(WidgetSpec(kicker: "GOD · FILL", title: "Nothing to fill", openLabel: "Open panel",
result: .text("God proposed a form-fill guide but no field→value pairs survived parsing — nothing was raised.")),
onOpen: { [weak self] in self?.hideNotchWidget(); self?.showPanel() })
} else {
raiseFillGuide(pairs, source: "God")
}
case "open":
// DWIM like god.mjs's openArgs: URL/scheme → open it; path → open the file; else it's an
// APP NAME and needs `-a` (bare `open Calendar` looks for a file, not the app, and no-ops).
Expand Down
Binary file modified packages/menubar/Switchboard.app/Contents/MacOS/Relay
Binary file not shown.