@@ -25,34 +25,39 @@ sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)))
2525import prompts # noqa: E402
2626import validate # noqa: E402
2727
28−HOST_VERSION = "0.1.0"
28+HOST_VERSION = "0.1.0" # release builds stamp the extension version here (scripts/build.mjs)
2929HOST_NAME = "io.diagonal.host"
3030ALLOWED_ORIGIN = "chrome-extension://mpnodlalikgeehnlnofdkpgapkmbkjdf/"
31+# The Chrome Web Store gives the listed extension its own ID. Once the listing exists, put its origin here
32+# ("chrome-extension://<store id>/") so the store install and the unpacked one can both use the host.
33+STORE_ORIGIN = ""
3134FM = os.environ.get("DIAGONAL_FM", "/usr/bin/fm") # tests point this at host/tests/fake_fm.py
3235SUPPORT = os.environ.get("DIAGONAL_SUPPORT_DIR", os.path.expanduser("~/Library/Application Support/Diagonal"))
3336SCHEMAS = os.path.join(SUPPORT, "schemas")
3437LOG_DIR = os.environ.get("DIAGONAL_LOG_DIR", os.path.expanduser("~/Library/Logs/Diagonal"))
3538LOG_MAX_BYTES = 5 * 1024 * 1024
36−CHAR_BUDGET = 10_000 # ≈ 2,850 tokens of prompt at 3.5 chars/token (section 9); double once 8,192 is confirmed
39+# fm's context holds prompt and reply together. Measured on macOS 27 with the topics schema: a 7,400-token
40+# prompt worked, 13,100 overflowed. 7,000 tokens leaves room for the reply.
41+TOKEN_BUDGET = 7_000
42+# Tokens per character, measured with `fm count-tokens` on tab lists: English 0.26, Russian 0.31, Chinese 0.54,
43+# Japanese 0.51. The estimate rounds up, so a prompt it calls small enough is; near the limit fm counts exactly.
44+TOKENS_PER_CHAR = {"ascii": 0.3, "cjk": 0.6, "other": 0.36}
45+EXACT_COUNT_ABOVE = 0.75 # of TOKEN_BUDGET
3746MAX_REQUEST = 64 * 1024 * 1024
3847MAX_REPLY = 1024 * 1024
3948OPS = {"ping", "name", "organize"}
4049
41−# `fm schema` argument lists (section 8). The nested form is unverified on a real macOS 27 machine;
42−# when it fails, --install-schemas writes the two flat schemas and `organize` takes two calls.
50+# `fm schema` argument lists (section 8), checked against fm on macOS 27. `--object NAME` takes the nested
51+# object's schema as JSON from `--schema`; "{TabTopic}" is replaced by the output of SUB_SCHEMAS["TabTopic"].
52+SUB_SCHEMAS = {
53+ "TabTopic": ["schema", "object", "--name", "TabTopic", "--integer", "index", "--string", "topic"],
54+}
4355SCHEMA_COMMANDS = {
4456 "name.json": ["schema", "object", "--name", "GroupName", "--string", "title", "--string", "emoji"],
45− "organize.json": ["schema", "object", "--name", "Organized",
46− "--object", "groups", "--array",
47− "--string", "title", "--string", "emoji", "--string", "color",
48− "--integer", "existing", "--integer", "members", "--array",
49− "--integer", "leftovers", "--array"],
50−}
51−FALLBACK_SCHEMA_COMMANDS = {
52− "organize-labels.json": ["schema", "object", "--name", "TopicLabels",
53− "--string", "labels", "--array", "--string", "emojis", "--array", "--string", "colors", "--array"],
54− "organize-assign.json": ["schema", "object", "--name", "Assignment", "--integer", "assignment", "--array"],
57+ "topics.json": ["schema", "object", "--name", "Topics", "--object", "tabs", "--schema", "{TabTopic}", "--array"],
5558}
59+# Organize schemas written by older hosts; removed when the current ones are installed.
60+LEGACY_SCHEMAS = ["organize.json", "organize-labels.json", "organize-assign.json"]
5661
5762
5863class Fail(Exception):
@@ -130,7 +135,7 @@ def validate_request(req):
130135 if not isinstance(it, dict):
131136 raise Fail("BAD_REQUEST", f"item {n} is not an object")
132137 clean.append({"i": n, "title": _trim(it.get("title"), 120), "url": _trim(it.get("url"), 300),
133− "description": _trim(it.get("description"), 300)})
138+ "description": _trim(it.get("description"), 500)})
134139 payload["items"] = clean
135140 for key in ("siblingTitles", "mustDifferFrom"):
136141 payload[key] = [_trim(t, 60) for t in payload.get(key) or [] if isinstance(t, str)][:20]
@@ -143,9 +148,8 @@ def validate_request(req):
143148 {"g": n, "title": _trim(g.get("title"), 60), "samples": [_trim(s, 60) for s in (g.get("samples") or [])[:2]]}
144149 for n, g in enumerate(groups[:12]) if isinstance(g, dict)
145150 ]
146− model = opts.get("model", "system")
147− if model not in ("system", "pcc"):
148− raise Fail("BAD_REQUEST", "opts.model must be system or pcc")
151+ # fm on macOS 27 only has the on-device model; older extensions may still ask for "pcc".
152+ opts["model"] = "system"
149153 return req
150154
151155
@@ -178,7 +182,7 @@ def classify(stderr, returncode=None):
178182
179183
180184def fm_env():
181− return {"PATH": "/usr/bin:/bin", "HOME": os.environ.get("HOME", ""), "LANG": "en_US.UTF-8"}
185+ return {"PATH": "/usr/bin:/bin", "HOME": os.environ.get("HOME", ""), "LANG": "en_US.UTF-8", "NO_COLOR": "1"}
182186
183187
184188def schema_path(name):
@@ -186,10 +190,46 @@ def schema_path(name):
186190 return path if os.path.isfile(path) and os.path.getsize(path) > 0 else None
187191
188192
189−def check_budget(prompt, n_items):
190− if len(prompt) > CHAR_BUDGET:
191− allowed = max(1, int(n_items * CHAR_BUDGET / len(prompt)))
192− raise Fail("OVER_BUDGET", f"prompt is {len(prompt)} chars, budget {CHAR_BUDGET}", retryable=True, allowedItems=allowed)
193+# How much of each page's text to send, most first: when a batch doesn't fit the budget, the pages' text
194+# is shortened, then dropped, before the host gives up and asks for a smaller batch.
195+DESC_CAPS = (500, 200, 0)
196+
197+
198+def estimate_tokens(prompt):
199+ n = 0.0
200+ for ch in prompt.instructions + prompt.text:
201+ n += TOKENS_PER_CHAR["ascii" if ch < "\x80" else "cjk" if validate.CJK.match(ch) else "other"]
202+ return int(n) + 60 # fm's own framing around instructions and prompt
203+
204+
205+def count_tokens(prompt):
206+ """fm's exact count (under 0.1 s), or None when it can't give one."""
207+ try:
208+ p = subprocess.run([FM, "count-tokens", "-q", "-i", prompt.instructions], input=prompt.text,
209+ capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=10, env=fm_env())
210+ except (OSError, subprocess.TimeoutExpired):
211+ return None
212+ out = p.stdout.strip()
213+ return int(out) if p.returncode == 0 and out.isdigit() else None
214+
215+
216+def prompt_tokens(prompt):
217+ estimate = estimate_tokens(prompt)
218+ if estimate <= TOKEN_BUDGET * EXACT_COUNT_ABOVE:
219+ return estimate
220+ return count_tokens(prompt) or estimate
221+
222+
223+def fit(build, n_items):
224+ """The prompt with as much page text as fits TOKEN_BUDGET; OVER_BUDGET with how many items would fit if
225+ even the titles alone are too long."""
226+ for cap in DESC_CAPS:
227+ prompt = build(cap)
228+ tokens = prompt_tokens(prompt)
229+ if tokens <= TOKEN_BUDGET:
230+ return prompt
231+ allowed = max(1, int(n_items * TOKEN_BUDGET / tokens))
232+ raise Fail("OVER_BUDGET", f"prompt is about {tokens} tokens, budget {TOKEN_BUDGET}", retryable=True, allowedItems=allowed)
193233
194234
195235def extract_json(stdout):
@@ -209,49 +249,100 @@ def extract_json(stdout):
209249 raise Fail("BAD_MODEL_OUTPUT", "no JSON object in fm output", raw=text[:2000])
210250
211251
212−def run_fm(prompt, schema, model, timeout_s):
213− args = [FM, "respond", "--model", model, "--no-stream", "--schema", schema, "--", prompt]
252+# fm's safety layer sometimes refuses harmless tab lists. Naming tabs only restates their own text, so a
253+# refused call is retried once in the mode Apple provides for transforming given content.
254+PERMISSIVE = ["--guardrails", "permissive-content-transformations"]
255+USAGE_EXIT = 64
256+
257+
258+# Below this much time left in a request, a second fm call (the permissive retry, naming one more new
259+# group) is not started: it would only be cut off.
260+MIN_CALL_S = 5.0
261+
262+
263+def run_fm(prompt, schema, opts):
264+ """`prompt` is a prompts.Prompt: its fixed rules go to -i, the tab text on stdin. Every call in one
265+ request shares its deadline, so the extension's guard (its timeout plus 10 s) is a real upper bound."""
266+ try:
267+ return _run_fm(prompt, schema, opts, [])
268+ except Fail as f:
269+ if f.code != "GUARDRAIL" or time_left(opts) < MIN_CALL_S:
270+ raise
271+ return _run_fm(prompt, schema, opts, PERMISSIVE)
272+
273+
274+def time_left(opts):
275+ return opts["deadline"] - time.time()
276+
277+
278+def _run_fm(prompt, schema, opts, extra):
279+ # The tab text goes on stdin (fm reads it there when no prompt argument is given), so it never shows up
280+ # in the process list. fm has no timeout of its own; subprocess enforces ours.
281+ # --greedy: the same tabs get the same topics and name on every run (measured on macOS 27: five runs,
282+ # one answer), so groups don't get renamed or reshuffled each time Diagonal looks at them. The strict
283+ # retry after an unusable answer samples instead: greedy would give the same answer again.
284+ greedy = [] if opts.get("strict") else ["--greedy"]
285+ args = [FM, "respond", "--model", opts["model"], "--no-stream", *greedy, "--schema", schema, "-i", prompt.instructions, *extra]
286+ timeout_s = max(1.0, time_left(opts))
214287 try:
215− p = subprocess.run(args, capture_output=True, text=True, timeout=timeout_s, env=fm_env())
288+ p = subprocess.run(args, input=prompt.text, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=timeout_s, env=fm_env())
216289 except FileNotFoundError:
217290 raise Fail("MODEL_UNAVAILABLE", f"fm not found at {FM} — requires macOS 27")
218291 except subprocess.TimeoutExpired:
219− raise Fail("TIMEOUT", f"fm exceeded {timeout_s:g}s", retryable=True)
292+ raise Fail("TIMEOUT", f"fm exceeded {opts['timeout_s']:g}s", retryable=True)
220293 if p.returncode != 0:
221294 code = classify(p.stderr, p.returncode)
222295 if code == "LICENSE_REQUIRED":
223296 raise Fail(code, LICENSE_MESSAGE, raw=(p.stderr or "")[:2000])
224− raise Fail(code, (p.stderr or f"fm exited {p.returncode}").strip()[:500], retryable=True, raw=(p.stderr or "")[:2000])
297+ # 64 is fm's usage error (unknown flag or value): the same call fails the same way again.
298+ raise Fail(code, (p.stderr or f"fm exited {p.returncode}").strip()[:500], retryable=p.returncode != USAGE_EXIT,
299+ raw=(p.stderr or "")[:2000])
225300 return extract_json(p.stdout)
226301
227302
228303# ----- ops --------------------------------------------------------------------------------------
229304
230305def organize_mode():
231− if schema_path("organize.json"):
232− return "nested"
233− if schema_path("organize-labels.json") and schema_path("organize-assign.json"):
234− return "two-call"
235− return "missing"
306+ return "topics" if schema_path("topics.json") else "missing"
236307
237308
238309def op_ping(_payload, _opts):
239310 if not os.path.exists(FM):
240− return {"hostVersion": HOST_VERSION, "fmPath": FM, "fmAvailable": False,
311+ return {"hostVersion": HOST_VERSION, "fmPath": FM, "fmAvailable": False, "licenseRequired": False,
241312 "fmMessage": f"fm not found at {FM} — requires macOS 27", "schemasOk": False, "organizeMode": organize_mode()}
242313 license_required = False
243314 try:
244− avail = subprocess.run([FM, "available", "--model", "system"], capture_output=True, text=True, timeout=15, env=fm_env())
315+ avail = subprocess.run([FM, "available", "--model", "system"], capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=15, env=fm_env())
245316 ok, msg = avail.returncode == 0, (avail.stdout + avail.stderr).strip()[:300]
246317 if not ok and license_needed(avail.returncode, avail.stdout + avail.stderr):
247318 license_required, msg = True, LICENSE_MESSAGE
248319 except subprocess.TimeoutExpired:
249320 ok, msg = False, "fm available timed out"
321+ schema_message = None
250322 if ok:
251− ensure_schemas()
323+ try:
324+ ensure_schemas()
325+ except Fail as f:
326+ schema_message = f.message
252327 mode = organize_mode()
253− return {"hostVersion": HOST_VERSION, "fmPath": FM, "fmAvailable": ok, "fmMessage": msg,
254− "licenseRequired": license_required, "schemasOk": schemas_ok(), "organizeMode": mode}
328+ reply = {"hostVersion": HOST_VERSION, "fmPath": FM, "fmAvailable": ok, "fmMessage": msg,
329+ "licenseRequired": license_required, "schemasOk": schemas_ok(), "organizeMode": mode}
330+ if schema_message:
331+ reply["schemaMessage"] = schema_message
332+ return reply
333+
334+
335+# Bumped when SCHEMA_COMMANDS change, so schemas written by an older host are rewritten. Version 3: organize
336+# asks for one topic per tab (topics.json) instead of whole groups.
337+SCHEMA_VERSION = "3"
338+
339+
340+def schemas_current():
341+ try:
342+ with open(os.path.join(SCHEMAS, "version"), encoding="utf-8") as f:
343+ return f.read().strip() == SCHEMA_VERSION
344+ except OSError:
345+ return False
255346
256347
257348def schemas_ok():
@@ -261,50 +352,86 @@ def schemas_ok():
261352def ensure_schemas():
262353 """Write the schema files on first use, so install order does not matter (e.g. the fm terms were
263354 accepted after the installer ran). stdout is the native-messaging channel: progress lines are swallowed."""
264− if schemas_ok():
355+ if schemas_ok() and schemas_current():
265356 return
266− with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
357+ out = io.StringIO()
358+ with contextlib.redirect_stdout(out), contextlib.redirect_stderr(out):
267359 rc = install_schemas()
268360 if rc == INSTALL_LICENSE_EXIT:
269361 raise Fail("LICENSE_REQUIRED", LICENSE_MESSAGE)
362+ if not schemas_ok():
363+ lines = [ln for ln in out.getvalue().splitlines() if ln.strip()]
364+ why = lines[-1] if lines else f"fm schema exited {rc}"
365+ raise Fail("SCHEMA_MISSING", f"fm could not write Diagonal's schema files: {why}"[:500], raw=out.getvalue()[-2000:])
366+
367+
368+def _fit_overflow(f, n_items):
369+ """fm's own context-size error carries no item count; estimate one so the extension can split the batch."""
370+ if f.code == "OVER_BUDGET" and "allowedItems" not in f.extra:
371+ f.extra["allowedItems"] = max(1, (n_items * 2) // 3)
372+ f.retryable = True
373+ return f
270374
271375
272376def op_name(payload, opts):
273377 ensure_schemas()
274378 schema = schema_path("name.json")
275379 if not schema:
276− raise Fail("SCHEMA_MISSING", os.path.join(SCHEMAS, "name.json"))
277− prompt = prompts.build_name_prompt(payload, strict=opts.get("strict", False))
278− check_budget(prompt, len(payload["items"]))
279− out = run_fm(prompt, schema, opts["model"], opts["timeout_s"])
380+ raise Fail("SCHEMA_MISSING", f"fm could not write {os.path.join(SCHEMAS, 'name.json')}")
381+ prompt = fit(lambda cap: prompts.build_name_prompt(payload, strict=opts.get("strict", False), desc_cap=cap), len(payload["items"]))
382+ try:
383+ out = run_fm(prompt, schema, opts)
384+ except Fail as f:
385+ raise _fit_overflow(f, len(payload["items"]))
280386 try:
281− return validate.validate_name(out, payload), len(prompt), "name"
387+ return validate.validate_name(out, payload), prompts.size(prompt), "name"
282388 except validate.ValidationError as e:
283389 raise Fail("BAD_MODEL_OUTPUT", str(e), retryable=True, raw=json.dumps(out, ensure_ascii=False)[:2000])
284390
285391
286392def op_organize(payload, opts):
287− strict = opts.get("strict", False)
288393 ensure_schemas()
289− mode = organize_mode()
394+ schema = schema_path("topics.json")
395+ if not schema:
396+ raise Fail("SCHEMA_MISSING", f"fm could not write {os.path.join(SCHEMAS, 'topics.json')}")
397+ prompt = fit(lambda cap: prompts.build_topics_prompt(payload, strict=opts.get("strict", False), desc_cap=cap), len(payload["items"]))
398+ try:
399+ out = run_fm(prompt, schema, opts)
400+ except Fail as f:
401+ raise _fit_overflow(f, len(payload["items"]))
402+ _, owner = prompts.topic_items(payload)
290403 try:
291− if mode == "nested":
292− prompt = prompts.build_organize_prompt(payload, strict=strict)
293− check_budget(prompt, len(payload["items"]))
294− out = run_fm(prompt, schema_path("organize.json"), opts["model"], opts["timeout_s"])
295− return validate.validate_organize(out, payload), len(prompt), "nested"
296− if mode == "two-call":
297− p1 = prompts.build_labels_prompt(payload, strict=strict)
298− check_budget(p1, len(payload["items"]))
299− labels = run_fm(p1, schema_path("organize-labels.json"), opts["model"], opts["timeout_s"])
300− names = [x for x in (labels.get("labels") or []) if isinstance(x, str)] if isinstance(labels, dict) else []
301− p2 = prompts.build_assign_prompt(payload, names, strict=strict)
302− check_budget(p2, len(payload["items"]))
303− assign = run_fm(p2, schema_path("organize-assign.json"), opts["model"], opts["timeout_s"])
304− return validate.organize_from_two_calls(labels, assign, payload), len(p1) + len(p2), "two-call"
404+ result = validate.organize_from_topics(out, payload, owner)
305405 except validate.ValidationError as e:
306− raise Fail("BAD_MODEL_OUTPUT", str(e), retryable=True)
307− raise Fail("SCHEMA_MISSING", os.path.join(SCHEMAS, "organize.json"))
406+ raise Fail("BAD_MODEL_OUTPUT", str(e), retryable=True, raw=json.dumps(out, ensure_ascii=False)[:2000])
407+ chars = prompts.size(prompt)
408+ if payload.get("nameGroups") is not False: # the extension's fit check only needs the topics
409+ chars += name_new_groups(result, payload, opts)
410+ return result, chars, "topics"
411+
412+
413+def name_new_groups(result, payload, opts):
414+ """Title each new group with the name op (well under a second each on fm), so groups appear named.
415+ A group that fails, runs past the request's deadline, or gets a title another group in the window
416+ already has keeps "" and the extension's naming loop, which checks every sibling, names it."""
417+ taken = [t for t in [g.get("title") for g in payload.get("existingGroups") or []] + list(payload.get("siblingTitles") or []) if t]
418+ chars = 0
419+ for g in result["groups"]:
420+ if "existing" in g or g.get("title"):
421+ continue
422+ if time_left(opts) < MIN_CALL_S:
423+ break
424+ sub = {"items": [dict(payload["items"][i], i=n) for n, i in enumerate(g["members"])], "siblingTitles": taken[:20]}
425+ try:
426+ named, n, _ = op_name(sub, {**opts, "strict": False})
427+ except Fail:
428+ continue
429+ chars += n
430+ if any(validate.same_label(named["title"], t) for t in taken):
431+ continue
432+ g["title"], g["emoji"] = named["title"], named["emoji"]
433+ taken.append(named["title"])
434+ return chars
308435
309436
310437def handle(req):
@@ -317,8 +444,8 @@ def handle(req):
317444 raw_opts = req.get("opts") or {}
318445 timeout_ms = raw_opts.get("timeoutMs", 45000)
319446 timeout_ms = timeout_ms if isinstance(timeout_ms, (int, float)) and not isinstance(timeout_ms, bool) else 45000
320− opts = {"model": raw_opts.get("model", "system"), "strict": bool(raw_opts.get("strict")),
321− "timeout_s": max(5.0, min(120.0, timeout_ms / 1000))}
447+ timeout_s = max(5.0, min(120.0, timeout_ms / 1000))
448+ opts = {"model": "system", "strict": bool(raw_opts.get("strict")), "timeout_s": timeout_s, "deadline": t0 + timeout_s}
322449 op = req["op"]
323450 if op == "ping":
324451 return {"v": 1, "id": rid, "ok": True, "result": op_ping(req["payload"], opts)}
@@ -367,8 +494,24 @@ def install_schemas():
367494
368495 def write(name, args):
369496 """True when written; "license" or "rejected" (fm ran and refused) or "error" otherwise."""
497+ subs = {}
498+ for key, sub_args in SUB_SCHEMAS.items():
499+ if "{%s}" % key not in args:
500+ continue
501+ try:
502+ p = subprocess.run([FM, *sub_args], capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=30, env=fm_env())
503+ except (FileNotFoundError, subprocess.TimeoutExpired) as e:
504+ print(f" {name}: fm failed ({e})", file=sys.stderr)
505+ return "error"
506+ if license_needed(p.returncode, p.stdout + p.stderr):
507+ return "license"
508+ if p.returncode != 0 or not p.stdout.strip():
509+ print(f" {name}: fm schema ({key}) exited {p.returncode}: {p.stderr.strip()[:200]}", file=sys.stderr)
510+ return "rejected"
511+ subs["{%s}" % key] = p.stdout.strip()
512+ args = [subs.get(a, a) for a in args]
370513 try:
371− p = subprocess.run([FM, *args], capture_output=True, text=True, timeout=30, env=fm_env())
514+ p = subprocess.run([FM, *args], capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=30, env=fm_env())
372515 except (FileNotFoundError, subprocess.TimeoutExpired) as e:
373516 print(f" {name}: fm failed ({e})", file=sys.stderr)
374517 return "error"
@@ -388,24 +531,19 @@ def install_schemas():
388531 print(" Diagonal writes its schemas on its own once that is done.", file=sys.stderr)
389532 return INSTALL_LICENSE_EXIT
390533 ok &= first is True
391− nested = write("organize.json", SCHEMA_COMMANDS["organize.json"])
392− if nested is True:
393− for name in FALLBACK_SCHEMA_COMMANDS:
534+ topics = write("topics.json", SCHEMA_COMMANDS["topics.json"])
535+ if topics == "license":
536+ return INSTALL_LICENSE_EXIT
537+ ok &= topics is True
538+ if ok:
539+ for name in LEGACY_SCHEMAS:
394540 try:
395541 os.remove(os.path.join(SCHEMAS, name))
396542 except FileNotFoundError:
397543 pass
398− elif nested != "rejected":
399− # fm did not get as far as judging the schema: no reason to think nesting is unsupported.
400− return INSTALL_LICENSE_EXIT if nested == "license" else 1
401− else:
402− print(" nested organize schema unsupported; using the two-call fallback", file=sys.stderr)
403− try:
404− os.remove(os.path.join(SCHEMAS, "organize.json"))
405− except FileNotFoundError:
406− pass
407− for name, args in FALLBACK_SCHEMA_COMMANDS.items():
408− ok &= write(name, args) is True
544+ if ok:
545+ with open(os.path.join(SCHEMAS, "version"), "w", encoding="utf-8") as f:
546+ f.write(SCHEMA_VERSION + "\n")
409547 return 0 if ok else 1
410548
411549
@@ -430,14 +568,16 @@ def selftest():
430568
431569 ping = op_ping({}, {})
432570 check("fm found", os.path.exists(FM), FM)
433− if ping["licenseRequired"]:
571+ if not os.path.exists(FM):
572+ return 1 # nothing else can work without it
573+ if ping.get("licenseRequired"):
434574 check("fm terms accepted", False, f"run: {LICENSE_FIX}")
435575 else:
436576 check("model available (fm available --model system)", ping["fmAvailable"], ping["fmMessage"])
437− for name in ["name.json", *(list(FALLBACK_SCHEMA_COMMANDS) if organize_mode() == "two-call" else ["organize.json"])]:
577+ for name in SCHEMA_COMMANDS:
438578 path = os.path.join(SCHEMAS, name)
439579 readable = os.path.isfile(path) and os.path.getsize(path) > 0
440− hint = f"written on their own after {LICENSE_FIX}" if ping["licenseRequired"] else "missing: run diagonal-host --install-schemas"
580+ hint = f"written on their own after {LICENSE_FIX}" if ping.get("licenseRequired") else "missing: run diagonal-host --install-schemas"
441581 check(f"schema {name}", readable, path if readable else hint)
442582 check("allowed origin pinned", "<" not in ALLOWED_ORIGIN, ALLOWED_ORIGIN)
443583 if ping["fmAvailable"] and schema_path("name.json"):
@@ -449,13 +589,137 @@ def selftest():
449589 return 0 if failures == 0 else 1
450590
451591
592+# ----- registering with browsers ------------------------------------------------------------------
593+
594+# Chromium reads per-user host manifests from <user data dir>/NativeMessagingHosts. Known user data
595+# dirs under ~/Library/Application Support; anything else that looks like a Chromium browser profile
596+# is found by browser_dirs()' scan.
597+KNOWN_BROWSER_DIRS = [
598+ "Google/Chrome", "Google/Chrome Beta", "Google/Chrome Dev", "Google/Chrome Canary", "Chromium",
599+ "BraveSoftware/Brave-Browser", "BraveSoftware/Brave-Origin", "BraveSoftware/Brave-Browser-Beta",
600+ "BraveSoftware/Brave-Browser-Nightly", "Microsoft Edge", "Microsoft Edge Beta", "Microsoft Edge Dev",
601+ "Microsoft Edge Canary", "Vivaldi", "Vivaldi Snapshot", "Arc/User Data", "Dia/User Data",
602+ "com.operasoftware.Opera", "com.operasoftware.OperaGX", "Thorium", "net.imput.helium", "Yandex/YandexBrowser",
603+]
604+BROWSER_SCAN_DEPTH = 3
605+REGISTERED_LIST = "registered-browsers.txt" # next to the host: where --register wrote manifests
606+
607+
608+def app_support():
609+ return os.environ.get("DIAGONAL_APP_SUPPORT", os.path.expanduser("~/Library/Application Support"))
610+
611+
612+def looks_like_browser(user_data_dir):
613+ """A Chromium browser's user data dir: `Local State` plus a profile with browsing data.
614+ Electron apps also have `Local State`, but no History, Favicons or Top Sites."""
615+ if not os.path.isfile(os.path.join(user_data_dir, "Local State")):
616+ return False
617+ try:
618+ profiles = [e for e in os.listdir(user_data_dir) if e == "Default" or e.startswith("Profile ")]
619+ except OSError:
620+ return False
621+ return any(os.path.exists(os.path.join(user_data_dir, p, f)) for p in profiles for f in ("History", "Favicons", "Top Sites"))
622+
623+
624+def browser_dirs():
625+ base = app_support()
626+ found = [os.path.join(base, rel) for rel in KNOWN_BROWSER_DIRS if os.path.isdir(os.path.join(base, rel))]
627+ for root, dirs, _files in os.walk(base):
628+ depth = os.path.relpath(root, base).count(os.sep) + (root != base)
629+ if root != base and (root in found or looks_like_browser(root)):
630+ if root not in found:
631+ found.append(root)
632+ dirs[:] = [] # profiles inside need no scan
633+ continue
634+ if depth >= BROWSER_SCAN_DEPTH or os.path.basename(root) == "Diagonal":
635+ dirs[:] = []
636+ return sorted(set(found))
637+
638+
639+def manifest_problems(path, host_path):
640+ try:
641+ m = json.load(open(path, encoding="utf-8"))
642+ except (OSError, ValueError) as e:
643+ return [f"unreadable ({e})"]
644+ problems = []
645+ if m.get("name") != HOST_NAME:
646+ problems.append(f"name is {m.get('name')!r}")
647+ if m.get("path") != host_path or not os.path.isabs(host_path):
648+ problems.append(f"path is {m.get('path')!r}")
649+ elif not os.access(host_path, os.X_OK):
650+ problems.append(f"host {host_path} is missing or not executable")
651+ if m.get("allowed_origins") != allowed_origins():
652+ problems.append(f"allowed_origins is {m.get('allowed_origins')}")
653+ return problems
654+
655+
656+def register(host_path):
657+ """Write this host's manifest into every Chromium browser's NativeMessagingHosts folder, check each
658+ one the way the browser will read it, and remember where they went for --unregister."""
659+ host_path = os.path.abspath(host_path)
660+ body = json.dumps(manifest_for(host_path), indent=2) + "\n"
661+ dirs = browser_dirs()
662+ if not dirs:
663+ print("No Chromium browser found. Open your browser once, then run this again.", file=sys.stderr)
664+ return 1
665+ written, failures = [], 0
666+ for d in dirs:
667+ target_dir = os.path.join(d, "NativeMessagingHosts")
668+ target = os.path.join(target_dir, f"{HOST_NAME}.json")
669+ try:
670+ os.makedirs(target_dir, exist_ok=True)
671+ with open(target + ".tmp", "w", encoding="utf-8") as f:
672+ f.write(body)
673+ os.replace(target + ".tmp", target) # a browser never reads a half-written file
674+ except OSError as e:
675+ print(f" {os.path.relpath(d, app_support())}: could not write ({e})", file=sys.stderr)
676+ failures += 1
677+ continue
678+ problems = manifest_problems(target, host_path)
679+ if problems:
680+ print(f" {os.path.relpath(d, app_support())}: {'; '.join(problems)}", file=sys.stderr)
681+ failures += 1
682+ else:
683+ print(f"registered: {os.path.relpath(d, app_support())}")
684+ written.append(target)
685+ try:
686+ with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), REGISTERED_LIST), "w", encoding="utf-8") as f:
687+ f.write("\n".join(written) + "\n")
688+ except OSError:
689+ pass
690+ # One browser that could not be written (a locked folder, a stale profile) does not undo the others:
691+ # the install goes on and the ones that failed were named above.
692+ return 0 if len(dirs) - failures > 0 else 1
693+
694+
695+def unregister():
696+ """Remove every manifest --register wrote, plus any other copy of ours a browser folder still has."""
697+ listed = []
698+ try:
699+ listed = open(os.path.join(os.path.dirname(os.path.realpath(__file__)), REGISTERED_LIST), encoding="utf-8").read().split("\n")
700+ except OSError:
701+ pass
702+ candidates = set(filter(None, listed)) | {os.path.join(d, "NativeMessagingHosts", f"{HOST_NAME}.json") for d in browser_dirs()}
703+ for path in sorted(candidates):
704+ try:
705+ os.remove(path)
706+ print(f"removed: {path}")
707+ except OSError:
708+ pass
709+ return 0
710+
711+
452712def manifest_for(path):
453713 return {"name": HOST_NAME, "description": "Diagonal: names tab groups with Apple's on-device model",
454− "path": os.path.abspath(path), "type": "stdio", "allowed_origins": [ALLOWED_ORIGIN]}
714+ "path": os.path.abspath(path), "type": "stdio", "allowed_origins": allowed_origins()}
715+
716+
717+def allowed_origins():
718+ return [o for o in (ALLOWED_ORIGIN, STORE_ORIGIN) if o]
455719
456720
457721def origin_ok(arg):
458− return isinstance(arg, str) and arg.rstrip("/") + "/" == ALLOWED_ORIGIN
722+ return isinstance(arg, str) and arg.rstrip("/") + "/" in allowed_origins()
459723
460724
461725def main(argv=None):
@@ -464,6 +728,10 @@ def main(argv=None):
464728 return selftest()
465729 if argv and argv[0] == "--install-schemas":
466730 return install_schemas()
731+ if argv and argv[0] == "--register":
732+ return register(argv[1] if len(argv) > 1 else sys.argv[0])
733+ if argv and argv[0] == "--unregister":
734+ return unregister()
467735 if argv and argv[0] == "--print-manifest":
468736 print(json.dumps(manifest_for(argv[1] if len(argv) > 1 else os.path.realpath(__file__)), indent=2))
469737 return 0