@@ -25,34 +25,39 @@ sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)))
25import prompts # noqa: E402
25import prompts # noqa: E402
26import validate # noqa: E402
26import validate # noqa: E402
28−HOST_VERSION = "0.1.0"
28+HOST_VERSION = "0.1.0" # release builds stamp the extension version here (scripts/build.mjs)
29HOST_NAME = "io.diagonal.host"
29HOST_NAME = "io.diagonal.host"
30ALLOWED_ORIGIN = "chrome-extension://mpnodlalikgeehnlnofdkpgapkmbkjdf/"
30ALLOWED_ORIGIN = "chrome-extension://mpnodlalikgeehnlnofdkpgapkmbkjdf/"
31+# The Chrome Web Store gives the listed extension its own ID. Once the listing exists, put its origin here
32+# ("chrome-extension://<store id>/") so the store install and the unpacked one can both use the host.
31FM = os.environ.get("DIAGONAL_FM", "/usr/bin/fm") # tests point this at host/tests/fake_fm.py
34FM = os.environ.get("DIAGONAL_FM", "/usr/bin/fm") # tests point this at host/tests/fake_fm.py
32SUPPORT = os.environ.get("DIAGONAL_SUPPORT_DIR", os.path.expanduser("~/Library/Application Support/Diagonal"))
35SUPPORT = os.environ.get("DIAGONAL_SUPPORT_DIR", os.path.expanduser("~/Library/Application Support/Diagonal"))
33SCHEMAS = os.path.join(SUPPORT, "schemas")
36SCHEMAS = os.path.join(SUPPORT, "schemas")
34LOG_DIR = os.environ.get("DIAGONAL_LOG_DIR", os.path.expanduser("~/Library/Logs/Diagonal"))
37LOG_DIR = os.environ.get("DIAGONAL_LOG_DIR", os.path.expanduser("~/Library/Logs/Diagonal"))
35LOG_MAX_BYTES = 5 * 1024 * 1024
38LOG_MAX_BYTES = 5 * 1024 * 1024
36−CHAR_BUDGET = 10_000 # ≈ 2,850 tokens of prompt at 3.5 chars/token (section 9); double once 8,192 is confirmed
39+# fm's context holds prompt and reply together. Measured on macOS 27 with the topics schema: a 7,400-token
40+# prompt worked, 13,100 overflowed. 7,000 tokens leaves room for the reply.
42+# Tokens per character, measured with `fm count-tokens` on tab lists: English 0.26, Russian 0.31, Chinese 0.54,
43+# Japanese 0.51. The estimate rounds up, so a prompt it calls small enough is; near the limit fm counts exactly.
44+TOKENS_PER_CHAR = {"ascii": 0.3, "cjk": 0.6, "other": 0.36}
45+EXACT_COUNT_ABOVE = 0.75 # of TOKEN_BUDGET
37MAX_REQUEST = 64 * 1024 * 1024
46MAX_REQUEST = 64 * 1024 * 1024
38MAX_REPLY = 1024 * 1024
47MAX_REPLY = 1024 * 1024
39OPS = {"ping", "name", "organize"}
48OPS = {"ping", "name", "organize"}
41−# `fm schema` argument lists (section 8). The nested form is unverified on a real macOS 27 machine;
50+# `fm schema` argument lists (section 8), checked against fm on macOS 27. `--object NAME` takes the nested
42−# when it fails, --install-schemas writes the two flat schemas and `organize` takes two calls.
51+# object's schema as JSON from `--schema`; "{TabTopic}" is replaced by the output of SUB_SCHEMAS["TabTopic"].
53+ "TabTopic": ["schema", "object", "--name", "TabTopic", "--integer", "index", "--string", "topic"],
43SCHEMA_COMMANDS = {
55SCHEMA_COMMANDS = {
44 "name.json": ["schema", "object", "--name", "GroupName", "--string", "title", "--string", "emoji"],
56 "name.json": ["schema", "object", "--name", "GroupName", "--string", "title", "--string", "emoji"],
45− "organize.json": ["schema", "object", "--name", "Organized",
57+ "topics.json": ["schema", "object", "--name", "Topics", "--object", "tabs", "--schema", "{TabTopic}", "--array"],
46− "--object", "groups", "--array",
47− "--string", "title", "--string", "emoji", "--string", "color",
48− "--integer", "existing", "--integer", "members", "--array",
49− "--integer", "leftovers", "--array"],
51−FALLBACK_SCHEMA_COMMANDS = {
52− "organize-labels.json": ["schema", "object", "--name", "TopicLabels",
53− "--string", "labels", "--array", "--string", "emojis", "--array", "--string", "colors", "--array"],
54− "organize-assign.json": ["schema", "object", "--name", "Assignment", "--integer", "assignment", "--array"],
59+# Organize schemas written by older hosts; removed when the current ones are installed.
60+LEGACY_SCHEMAS = ["organize.json", "organize-labels.json", "organize-assign.json"]
58class Fail(Exception):
63class Fail(Exception):
@@ -130,7 +135,7 @@ def validate_request(req):
130 if not isinstance(it, dict):
135 if not isinstance(it, dict):
131 raise Fail("BAD_REQUEST", f"item {n} is not an object")
136 raise Fail("BAD_REQUEST", f"item {n} is not an object")
132 clean.append({"i": n, "title": _trim(it.get("title"), 120), "url": _trim(it.get("url"), 300),
137 clean.append({"i": n, "title": _trim(it.get("title"), 120), "url": _trim(it.get("url"), 300),
133− "description": _trim(it.get("description"), 300)})
138+ "description": _trim(it.get("description"), 500)})
134 payload["items"] = clean
139 payload["items"] = clean
135 for key in ("siblingTitles", "mustDifferFrom"):
140 for key in ("siblingTitles", "mustDifferFrom"):
136 payload[key] = [_trim(t, 60) for t in payload.get(key) or [] if isinstance(t, str)][:20]
141 payload[key] = [_trim(t, 60) for t in payload.get(key) or [] if isinstance(t, str)][:20]
@@ -143,9 +148,8 @@ def validate_request(req):
143 {"g": n, "title": _trim(g.get("title"), 60), "samples": [_trim(s, 60) for s in (g.get("samples") or [])[:2]]}
148 {"g": n, "title": _trim(g.get("title"), 60), "samples": [_trim(s, 60) for s in (g.get("samples") or [])[:2]]}
144 for n, g in enumerate(groups[:12]) if isinstance(g, dict)
149 for n, g in enumerate(groups[:12]) if isinstance(g, dict)
146− model = opts.get("model", "system")
151+ # fm on macOS 27 only has the on-device model; older extensions may still ask for "pcc".
147− if model not in ("system", "pcc"):
152+ opts["model"] = "system"
148− raise Fail("BAD_REQUEST", "opts.model must be system or pcc")
149 return req
153 return req
@@ -178,7 +182,7 @@ def classify(stderr, returncode=None):
180def fm_env():
184def fm_env():
181− return {"PATH": "/usr/bin:/bin", "HOME": os.environ.get("HOME", ""), "LANG": "en_US.UTF-8"}
185+ return {"PATH": "/usr/bin:/bin", "HOME": os.environ.get("HOME", ""), "LANG": "en_US.UTF-8", "NO_COLOR": "1"}
184def schema_path(name):
188def schema_path(name):
@@ -186,10 +190,46 @@ def schema_path(name):
186 return path if os.path.isfile(path) and os.path.getsize(path) > 0 else None
190 return path if os.path.isfile(path) and os.path.getsize(path) > 0 else None
189−def check_budget(prompt, n_items):
193+# How much of each page's text to send, most first: when a batch doesn't fit the budget, the pages' text
190− if len(prompt) > CHAR_BUDGET:
194+# is shortened, then dropped, before the host gives up and asks for a smaller batch.
195+DESC_CAPS = (500, 200, 0)
198+def estimate_tokens(prompt):
200+ for ch in prompt.instructions + prompt.text:
201+ n += TOKENS_PER_CHAR["ascii" if ch < "\x80" else "cjk" if validate.CJK.match(ch) else "other"]
202+ return int(n) + 60 # fm's own framing around instructions and prompt
205+def count_tokens(prompt):
206+ """fm's exact count (under 0.1 s), or None when it can't give one."""
208+ p = subprocess.run([FM, "count-tokens", "-q", "-i", prompt.instructions], input=prompt.text,
209+ capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=10, env=fm_env())
210+ except (OSError, subprocess.TimeoutExpired):
212+ out = p.stdout.strip()
213+ return int(out) if p.returncode == 0 and out.isdigit() else None
216+def prompt_tokens(prompt):
217+ estimate = estimate_tokens(prompt)
218+ if estimate <= TOKEN_BUDGET * EXACT_COUNT_ABOVE:
220+ return count_tokens(prompt) or estimate
223+def fit(build, n_items):
224+ """The prompt with as much page text as fits TOKEN_BUDGET; OVER_BUDGET with how many items would fit if
225+ even the titles alone are too long."""
226+ for cap in DESC_CAPS:
228+ tokens = prompt_tokens(prompt)
229+ if tokens <= TOKEN_BUDGET:
191− allowed = max(1, int(n_items * CHAR_BUDGET / len(prompt)))
231+ allowed = max(1, int(n_items * TOKEN_BUDGET / tokens))
192− raise Fail("OVER_BUDGET", f"prompt is {len(prompt)} chars, budget {CHAR_BUDGET}", retryable=True, allowedItems=allowed)
232+ raise Fail("OVER_BUDGET", f"prompt is about {tokens} tokens, budget {TOKEN_BUDGET}", retryable=True, allowedItems=allowed)
195def extract_json(stdout):
235def extract_json(stdout):
@@ -209,49 +249,100 @@ def extract_json(stdout):
209 raise Fail("BAD_MODEL_OUTPUT", "no JSON object in fm output", raw=text[:2000])
249 raise Fail("BAD_MODEL_OUTPUT", "no JSON object in fm output", raw=text[:2000])
252+# fm's safety layer sometimes refuses harmless tab lists. Naming tabs only restates their own text, so a
253+# refused call is retried once in the mode Apple provides for transforming given content.
254+PERMISSIVE = ["--guardrails", "permissive-content-transformations"]
258+# Below this much time left in a request, a second fm call (the permissive retry, naming one more new
259+# group) is not started: it would only be cut off.
212−def run_fm(prompt, schema, model, timeout_s):
263+def run_fm(prompt, schema, opts):
264+ """`prompt` is a prompts.Prompt: its fixed rules go to -i, the tab text on stdin. Every call in one
265+ request shares its deadline, so the extension's guard (its timeout plus 10 s) is a real upper bound."""
267+ return _run_fm(prompt, schema, opts, [])
269+ if f.code != "GUARDRAIL" or time_left(opts) < MIN_CALL_S:
271+ return _run_fm(prompt, schema, opts, PERMISSIVE)
275+ return opts["deadline"] - time.time()
278+def _run_fm(prompt, schema, opts, extra):
279+ # The tab text goes on stdin (fm reads it there when no prompt argument is given), so it never shows up
280+ # in the process list. fm has no timeout of its own; subprocess enforces ours.
281+ # --greedy: the same tabs get the same topics and name on every run (measured on macOS 27: five runs,
282+ # one answer), so groups don't get renamed or reshuffled each time Diagonal looks at them. The strict
283+ # retry after an unusable answer samples instead: greedy would give the same answer again.
284+ greedy = [] if opts.get("strict") else ["--greedy"]
213− args = [FM, "respond", "--model", model, "--no-stream", "--schema", schema, "--", prompt]
285+ args = [FM, "respond", "--model", opts["model"], "--no-stream", *greedy, "--schema", schema, "-i", prompt.instructions, *extra]
286+ timeout_s = max(1.0, time_left(opts))
215− p = subprocess.run(args, capture_output=True, text=True, timeout=timeout_s, env=fm_env())
288+ p = subprocess.run(args, input=prompt.text, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=timeout_s, env=fm_env())
216 except FileNotFoundError:
289 except FileNotFoundError:
217 raise Fail("MODEL_UNAVAILABLE", f"fm not found at {FM} — requires macOS 27")
290 raise Fail("MODEL_UNAVAILABLE", f"fm not found at {FM} — requires macOS 27")
218 except subprocess.TimeoutExpired:
291 except subprocess.TimeoutExpired:
219− raise Fail("TIMEOUT", f"fm exceeded {timeout_s:g}s", retryable=True)
292+ raise Fail("TIMEOUT", f"fm exceeded {opts['timeout_s']:g}s", retryable=True)
220 if p.returncode != 0:
293 if p.returncode != 0:
221 code = classify(p.stderr, p.returncode)
294 code = classify(p.stderr, p.returncode)
222 if code == "LICENSE_REQUIRED":
295 if code == "LICENSE_REQUIRED":
223 raise Fail(code, LICENSE_MESSAGE, raw=(p.stderr or "")[:2000])
296 raise Fail(code, LICENSE_MESSAGE, raw=(p.stderr or "")[:2000])
297+ # 64 is fm's usage error (unknown flag or value): the same call fails the same way again.
224− raise Fail(code, (p.stderr or f"fm exited {p.returncode}").strip()[:500], retryable=True, raw=(p.stderr or "")[:2000])
298+ raise Fail(code, (p.stderr or f"fm exited {p.returncode}").strip()[:500], retryable=p.returncode != USAGE_EXIT,
299+ raw=(p.stderr or "")[:2000])
225 return extract_json(p.stdout)
300 return extract_json(p.stdout)
228# ----- ops --------------------------------------------------------------------------------------
303# ----- ops --------------------------------------------------------------------------------------
230def organize_mode():
305def organize_mode():
231− if schema_path("organize.json"):
306+ return "topics" if schema_path("topics.json") else "missing"
233− if schema_path("organize-labels.json") and schema_path("organize-assign.json"):
238def op_ping(_payload, _opts):
309def op_ping(_payload, _opts):
239 if not os.path.exists(FM):
310 if not os.path.exists(FM):
240− return {"hostVersion": HOST_VERSION, "fmPath": FM, "fmAvailable": False,
311+ return {"hostVersion": HOST_VERSION, "fmPath": FM, "fmAvailable": False, "licenseRequired": False,
241 "fmMessage": f"fm not found at {FM} — requires macOS 27", "schemasOk": False, "organizeMode": organize_mode()}
312 "fmMessage": f"fm not found at {FM} — requires macOS 27", "schemasOk": False, "organizeMode": organize_mode()}
242 license_required = False
313 license_required = False
244− avail = subprocess.run([FM, "available", "--model", "system"], capture_output=True, text=True, timeout=15, env=fm_env())
315+ avail = subprocess.run([FM, "available", "--model", "system"], capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=15, env=fm_env())
245 ok, msg = avail.returncode == 0, (avail.stdout + avail.stderr).strip()[:300]
316 ok, msg = avail.returncode == 0, (avail.stdout + avail.stderr).strip()[:300]
246 if not ok and license_needed(avail.returncode, avail.stdout + avail.stderr):
317 if not ok and license_needed(avail.returncode, avail.stdout + avail.stderr):
247 license_required, msg = True, LICENSE_MESSAGE
318 license_required, msg = True, LICENSE_MESSAGE
248 except subprocess.TimeoutExpired:
319 except subprocess.TimeoutExpired:
249 ok, msg = False, "fm available timed out"
320 ok, msg = False, "fm available timed out"
321+ schema_message = None
251− ensure_schemas()
324+ ensure_schemas()
326+ schema_message = f.message
252 mode = organize_mode()
327 mode = organize_mode()
253− return {"hostVersion": HOST_VERSION, "fmPath": FM, "fmAvailable": ok, "fmMessage": msg,
328+ reply = {"hostVersion": HOST_VERSION, "fmPath": FM, "fmAvailable": ok, "fmMessage": msg,
254− "licenseRequired": license_required, "schemasOk": schemas_ok(), "organizeMode": mode}
329+ "licenseRequired": license_required, "schemasOk": schemas_ok(), "organizeMode": mode}
331+ reply["schemaMessage"] = schema_message
335+# Bumped when SCHEMA_COMMANDS change, so schemas written by an older host are rewritten. Version 3: organize
336+# asks for one topic per tab (topics.json) instead of whole groups.
340+def schemas_current():
342+ with open(os.path.join(SCHEMAS, "version"), encoding="utf-8") as f:
343+ return f.read().strip() == SCHEMA_VERSION
257def schemas_ok():
348def schemas_ok():
@@ -261,50 +352,86 @@ def schemas_ok():
261def ensure_schemas():
352def ensure_schemas():
262 """Write the schema files on first use, so install order does not matter (e.g. the fm terms were
353 """Write the schema files on first use, so install order does not matter (e.g. the fm terms were
263 accepted after the installer ran). stdout is the native-messaging channel: progress lines are swallowed."""
354 accepted after the installer ran). stdout is the native-messaging channel: progress lines are swallowed."""
264− if schemas_ok():
355+ if schemas_ok() and schemas_current():
266− with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
358+ with contextlib.redirect_stdout(out), contextlib.redirect_stderr(out):
267 rc = install_schemas()
359 rc = install_schemas()
268 if rc == INSTALL_LICENSE_EXIT:
360 if rc == INSTALL_LICENSE_EXIT:
269 raise Fail("LICENSE_REQUIRED", LICENSE_MESSAGE)
361 raise Fail("LICENSE_REQUIRED", LICENSE_MESSAGE)
362+ if not schemas_ok():
363+ lines = [ln for ln in out.getvalue().splitlines() if ln.strip()]
364+ why = lines[-1] if lines else f"fm schema exited {rc}"
365+ raise Fail("SCHEMA_MISSING", f"fm could not write Diagonal's schema files: {why}"[:500], raw=out.getvalue()[-2000:])
368+def _fit_overflow(f, n_items):
369+ """fm's own context-size error carries no item count; estimate one so the extension can split the batch."""
370+ if f.code == "OVER_BUDGET" and "allowedItems" not in f.extra:
371+ f.extra["allowedItems"] = max(1, (n_items * 2) // 3)
272def op_name(payload, opts):
376def op_name(payload, opts):
273 ensure_schemas()
377 ensure_schemas()
274 schema = schema_path("name.json")
378 schema = schema_path("name.json")
275 if not schema:
379 if not schema:
276− raise Fail("SCHEMA_MISSING", os.path.join(SCHEMAS, "name.json"))
380+ raise Fail("SCHEMA_MISSING", f"fm could not write {os.path.join(SCHEMAS, 'name.json')}")
277− prompt = prompts.build_name_prompt(payload, strict=opts.get("strict", False))
381+ prompt = fit(lambda cap: prompts.build_name_prompt(payload, strict=opts.get("strict", False), desc_cap=cap), len(payload["items"]))
278− check_budget(prompt, len(payload["items"]))
382+ try:
279− out = run_fm(prompt, schema, opts["model"], opts["timeout_s"])
383+ out = run_fm(prompt, schema, opts)
385+ raise _fit_overflow(f, len(payload["items"]))
281− return validate.validate_name(out, payload), len(prompt), "name"
387+ return validate.validate_name(out, payload), prompts.size(prompt), "name"
282 except validate.ValidationError as e:
388 except validate.ValidationError as e:
283 raise Fail("BAD_MODEL_OUTPUT", str(e), retryable=True, raw=json.dumps(out, ensure_ascii=False)[:2000])
389 raise Fail("BAD_MODEL_OUTPUT", str(e), retryable=True, raw=json.dumps(out, ensure_ascii=False)[:2000])
286def op_organize(payload, opts):
392def op_organize(payload, opts):
287− strict = opts.get("strict", False)
288 ensure_schemas()
393 ensure_schemas()
289− mode = organize_mode()
394+ schema = schema_path("topics.json")
396+ raise Fail("SCHEMA_MISSING", f"fm could not write {os.path.join(SCHEMAS, 'topics.json')}")
397+ prompt = fit(lambda cap: prompts.build_topics_prompt(payload, strict=opts.get("strict", False), desc_cap=cap), len(payload["items"]))
399+ out = run_fm(prompt, schema, opts)
401+ raise _fit_overflow(f, len(payload["items"]))
402+ _, owner = prompts.topic_items(payload)
291− if mode == "nested":
404+ result = validate.organize_from_topics(out, payload, owner)
292− prompt = prompts.build_organize_prompt(payload, strict=strict)
293− check_budget(prompt, len(payload["items"]))
294− out = run_fm(prompt, schema_path("organize.json"), opts["model"], opts["timeout_s"])
295− return validate.validate_organize(out, payload), len(prompt), "nested"
296− if mode == "two-call":
297− p1 = prompts.build_labels_prompt(payload, strict=strict)
298− check_budget(p1, len(payload["items"]))
299− labels = run_fm(p1, schema_path("organize-labels.json"), opts["model"], opts["timeout_s"])
300− names = [x for x in (labels.get("labels") or []) if isinstance(x, str)] if isinstance(labels, dict) else []
301− p2 = prompts.build_assign_prompt(payload, names, strict=strict)
302− check_budget(p2, len(payload["items"]))
303− assign = run_fm(p2, schema_path("organize-assign.json"), opts["model"], opts["timeout_s"])
304− return validate.organize_from_two_calls(labels, assign, payload), len(p1) + len(p2), "two-call"
305 except validate.ValidationError as e:
405 except validate.ValidationError as e:
306− raise Fail("BAD_MODEL_OUTPUT", str(e), retryable=True)
406+ raise Fail("BAD_MODEL_OUTPUT", str(e), retryable=True, raw=json.dumps(out, ensure_ascii=False)[:2000])
307− raise Fail("SCHEMA_MISSING", os.path.join(SCHEMAS, "organize.json"))
407+ chars = prompts.size(prompt)
408+ if payload.get("nameGroups") is not False: # the extension's fit check only needs the topics
409+ chars += name_new_groups(result, payload, opts)
410+ return result, chars, "topics"
413+def name_new_groups(result, payload, opts):
414+ """Title each new group with the name op (well under a second each on fm), so groups appear named.
415+ A group that fails, runs past the request's deadline, or gets a title another group in the window
416+ already has keeps "" and the extension's naming loop, which checks every sibling, names it."""
417+ taken = [t for t in [g.get("title") for g in payload.get("existingGroups") or []] + list(payload.get("siblingTitles") or []) if t]
419+ for g in result["groups"]:
420+ if "existing" in g or g.get("title"):
422+ if time_left(opts) < MIN_CALL_S:
424+ sub = {"items": [dict(payload["items"][i], i=n) for n, i in enumerate(g["members"])], "siblingTitles": taken[:20]}
426+ named, n, _ = op_name(sub, {**opts, "strict": False})
430+ if any(validate.same_label(named["title"], t) for t in taken):
432+ g["title"], g["emoji"] = named["title"], named["emoji"]
433+ taken.append(named["title"])
310def handle(req):
437def handle(req):
@@ -317,8 +444,8 @@ def handle(req):
317 raw_opts = req.get("opts") or {}
444 raw_opts = req.get("opts") or {}
318 timeout_ms = raw_opts.get("timeoutMs", 45000)
445 timeout_ms = raw_opts.get("timeoutMs", 45000)
319 timeout_ms = timeout_ms if isinstance(timeout_ms, (int, float)) and not isinstance(timeout_ms, bool) else 45000
446 timeout_ms = timeout_ms if isinstance(timeout_ms, (int, float)) and not isinstance(timeout_ms, bool) else 45000
320− opts = {"model": raw_opts.get("model", "system"), "strict": bool(raw_opts.get("strict")),
321− "timeout_s": max(5.0, min(120.0, timeout_ms / 1000))}
447+ timeout_s = max(5.0, min(120.0, timeout_ms / 1000))
448+ opts = {"model": "system", "strict": bool(raw_opts.get("strict")), "timeout_s": timeout_s, "deadline": t0 + timeout_s}
322 op = req["op"]
449 op = req["op"]
323 if op == "ping":
450 if op == "ping":
324 return {"v": 1, "id": rid, "ok": True, "result": op_ping(req["payload"], opts)}
451 return {"v": 1, "id": rid, "ok": True, "result": op_ping(req["payload"], opts)}
@@ -367,8 +494,24 @@ def install_schemas():
368 def write(name, args):
495 def write(name, args):
369 """True when written; "license" or "rejected" (fm ran and refused) or "error" otherwise."""
496 """True when written; "license" or "rejected" (fm ran and refused) or "error" otherwise."""
498+ for key, sub_args in SUB_SCHEMAS.items():
499+ if "{%s}" % key not in args:
502+ p = subprocess.run([FM, *sub_args], capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=30, env=fm_env())
503+ except (FileNotFoundError, subprocess.TimeoutExpired) as e:
504+ print(f" {name}: fm failed ({e})", file=sys.stderr)
506+ if license_needed(p.returncode, p.stdout + p.stderr):
508+ if p.returncode != 0 or not p.stdout.strip():
509+ print(f" {name}: fm schema ({key}) exited {p.returncode}: {p.stderr.strip()[:200]}", file=sys.stderr)
511+ subs["{%s}" % key] = p.stdout.strip()
512+ args = [subs.get(a, a) for a in args]
371− p = subprocess.run([FM, *args], capture_output=True, text=True, timeout=30, env=fm_env())
514+ p = subprocess.run([FM, *args], capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=30, env=fm_env())
372 except (FileNotFoundError, subprocess.TimeoutExpired) as e:
515 except (FileNotFoundError, subprocess.TimeoutExpired) as e:
373 print(f" {name}: fm failed ({e})", file=sys.stderr)
516 print(f" {name}: fm failed ({e})", file=sys.stderr)
374 return "error"
517 return "error"
@@ -388,24 +531,19 @@ def install_schemas():
388 print(" Diagonal writes its schemas on its own once that is done.", file=sys.stderr)
531 print(" Diagonal writes its schemas on its own once that is done.", file=sys.stderr)
389 return INSTALL_LICENSE_EXIT
532 return INSTALL_LICENSE_EXIT
390 ok &= first is True
533 ok &= first is True
391− nested = write("organize.json", SCHEMA_COMMANDS["organize.json"])
534+ topics = write("topics.json", SCHEMA_COMMANDS["topics.json"])
535+ if topics == "license":
536+ return INSTALL_LICENSE_EXIT
392− if nested is True:
537+ ok &= topics is True
393− for name in FALLBACK_SCHEMA_COMMANDS:
538+ if ok:
539+ for name in LEGACY_SCHEMAS:
395 os.remove(os.path.join(SCHEMAS, name))
541 os.remove(os.path.join(SCHEMAS, name))
396 except FileNotFoundError:
542 except FileNotFoundError:
398− elif nested != "rejected":
544+ if ok:
399− # fm did not get as far as judging the schema: no reason to think nesting is unsupported.
400− return INSTALL_LICENSE_EXIT if nested == "license" else 1
402− print(" nested organize schema unsupported; using the two-call fallback", file=sys.stderr)
404− os.remove(os.path.join(SCHEMAS, "organize.json"))
545+ with open(os.path.join(SCHEMAS, "version"), "w", encoding="utf-8") as f:
405− except FileNotFoundError:
546+ f.write(SCHEMA_VERSION + "\n")
407− for name, args in FALLBACK_SCHEMA_COMMANDS.items():
408− ok &= write(name, args) is True
409 return 0 if ok else 1
547 return 0 if ok else 1
@@ -430,14 +568,16 @@ def selftest():
431 ping = op_ping({}, {})
569 ping = op_ping({}, {})
432 check("fm found", os.path.exists(FM), FM)
570 check("fm found", os.path.exists(FM), FM)
571+ if not os.path.exists(FM):
572+ return 1 # nothing else can work without it
433− if ping["licenseRequired"]:
573+ if ping.get("licenseRequired"):
434 check("fm terms accepted", False, f"run: {LICENSE_FIX}")
574 check("fm terms accepted", False, f"run: {LICENSE_FIX}")
436 check("model available (fm available --model system)", ping["fmAvailable"], ping["fmMessage"])
576 check("model available (fm available --model system)", ping["fmAvailable"], ping["fmMessage"])
437− for name in ["name.json", *(list(FALLBACK_SCHEMA_COMMANDS) if organize_mode() == "two-call" else ["organize.json"])]:
577+ for name in SCHEMA_COMMANDS:
438 path = os.path.join(SCHEMAS, name)
578 path = os.path.join(SCHEMAS, name)
439 readable = os.path.isfile(path) and os.path.getsize(path) > 0
579 readable = os.path.isfile(path) and os.path.getsize(path) > 0
440− hint = f"written on their own after {LICENSE_FIX}" if ping["licenseRequired"] else "missing: run diagonal-host --install-schemas"
580+ hint = f"written on their own after {LICENSE_FIX}" if ping.get("licenseRequired") else "missing: run diagonal-host --install-schemas"
441 check(f"schema {name}", readable, path if readable else hint)
581 check(f"schema {name}", readable, path if readable else hint)
442 check("allowed origin pinned", "<" not in ALLOWED_ORIGIN, ALLOWED_ORIGIN)
582 check("allowed origin pinned", "<" not in ALLOWED_ORIGIN, ALLOWED_ORIGIN)
443 if ping["fmAvailable"] and schema_path("name.json"):
583 if ping["fmAvailable"] and schema_path("name.json"):
@@ -449,13 +589,137 @@ def selftest():
449 return 0 if failures == 0 else 1
589 return 0 if failures == 0 else 1
592+# ----- registering with browsers ------------------------------------------------------------------
594+# Chromium reads per-user host manifests from <user data dir>/NativeMessagingHosts. Known user data
595+# dirs under ~/Library/Application Support; anything else that looks like a Chromium browser profile
596+# is found by browser_dirs()' scan.
597+KNOWN_BROWSER_DIRS = [
598+ "Google/Chrome", "Google/Chrome Beta", "Google/Chrome Dev", "Google/Chrome Canary", "Chromium",
599+ "BraveSoftware/Brave-Browser", "BraveSoftware/Brave-Origin", "BraveSoftware/Brave-Browser-Beta",
600+ "BraveSoftware/Brave-Browser-Nightly", "Microsoft Edge", "Microsoft Edge Beta", "Microsoft Edge Dev",
601+ "Microsoft Edge Canary", "Vivaldi", "Vivaldi Snapshot", "Arc/User Data", "Dia/User Data",
602+ "com.operasoftware.Opera", "com.operasoftware.OperaGX", "Thorium", "net.imput.helium", "Yandex/YandexBrowser",
604+BROWSER_SCAN_DEPTH = 3
605+REGISTERED_LIST = "registered-browsers.txt" # next to the host: where --register wrote manifests
609+ return os.environ.get("DIAGONAL_APP_SUPPORT", os.path.expanduser("~/Library/Application Support"))
612+def looks_like_browser(user_data_dir):
613+ """A Chromium browser's user data dir: `Local State` plus a profile with browsing data.
614+ Electron apps also have `Local State`, but no History, Favicons or Top Sites."""
615+ if not os.path.isfile(os.path.join(user_data_dir, "Local State")):
618+ profiles = [e for e in os.listdir(user_data_dir) if e == "Default" or e.startswith("Profile ")]
621+ return any(os.path.exists(os.path.join(user_data_dir, p, f)) for p in profiles for f in ("History", "Favicons", "Top Sites"))
625+ base = app_support()
626+ found = [os.path.join(base, rel) for rel in KNOWN_BROWSER_DIRS if os.path.isdir(os.path.join(base, rel))]
627+ for root, dirs, _files in os.walk(base):
628+ depth = os.path.relpath(root, base).count(os.sep) + (root != base)
629+ if root != base and (root in found or looks_like_browser(root)):
630+ if root not in found:
632+ dirs[:] = [] # profiles inside need no scan
634+ if depth >= BROWSER_SCAN_DEPTH or os.path.basename(root) == "Diagonal":
636+ return sorted(set(found))
639+def manifest_problems(path, host_path):
641+ m = json.load(open(path, encoding="utf-8"))
642+ except (OSError, ValueError) as e:
643+ return [f"unreadable ({e})"]
645+ if m.get("name") != HOST_NAME:
646+ problems.append(f"name is {m.get('name')!r}")
647+ if m.get("path") != host_path or not os.path.isabs(host_path):
648+ problems.append(f"path is {m.get('path')!r}")
649+ elif not os.access(host_path, os.X_OK):
650+ problems.append(f"host {host_path} is missing or not executable")
651+ if m.get("allowed_origins") != allowed_origins():
652+ problems.append(f"allowed_origins is {m.get('allowed_origins')}")
656+def register(host_path):
657+ """Write this host's manifest into every Chromium browser's NativeMessagingHosts folder, check each
658+ one the way the browser will read it, and remember where they went for --unregister."""
659+ host_path = os.path.abspath(host_path)
660+ body = json.dumps(manifest_for(host_path), indent=2) + "\n"
661+ dirs = browser_dirs()
663+ print("No Chromium browser found. Open your browser once, then run this again.", file=sys.stderr)
665+ written, failures = [], 0
667+ target_dir = os.path.join(d, "NativeMessagingHosts")
668+ target = os.path.join(target_dir, f"{HOST_NAME}.json")
670+ os.makedirs(target_dir, exist_ok=True)
671+ with open(target + ".tmp", "w", encoding="utf-8") as f:
673+ os.replace(target + ".tmp", target) # a browser never reads a half-written file
674+ except OSError as e:
675+ print(f" {os.path.relpath(d, app_support())}: could not write ({e})", file=sys.stderr)
678+ problems = manifest_problems(target, host_path)
680+ print(f" {os.path.relpath(d, app_support())}: {'; '.join(problems)}", file=sys.stderr)
683+ print(f"registered: {os.path.relpath(d, app_support())}")
684+ written.append(target)
686+ with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), REGISTERED_LIST), "w", encoding="utf-8") as f:
687+ f.write("\n".join(written) + "\n")
690+ # One browser that could not be written (a locked folder, a stale profile) does not undo the others:
691+ # the install goes on and the ones that failed were named above.
692+ return 0 if len(dirs) - failures > 0 else 1
696+ """Remove every manifest --register wrote, plus any other copy of ours a browser folder still has."""
699+ listed = open(os.path.join(os.path.dirname(os.path.realpath(__file__)), REGISTERED_LIST), encoding="utf-8").read().split("\n")
702+ candidates = set(filter(None, listed)) | {os.path.join(d, "NativeMessagingHosts", f"{HOST_NAME}.json") for d in browser_dirs()}
703+ for path in sorted(candidates):
706+ print(f"removed: {path}")
452def manifest_for(path):
712def manifest_for(path):
453 return {"name": HOST_NAME, "description": "Diagonal: names tab groups with Apple's on-device model",
713 return {"name": HOST_NAME, "description": "Diagonal: names tab groups with Apple's on-device model",
454− "path": os.path.abspath(path), "type": "stdio", "allowed_origins": [ALLOWED_ORIGIN]}
714+ "path": os.path.abspath(path), "type": "stdio", "allowed_origins": allowed_origins()}
717+def allowed_origins():
718+ return [o for o in (ALLOWED_ORIGIN, STORE_ORIGIN) if o]
457def origin_ok(arg):
721def origin_ok(arg):
458− return isinstance(arg, str) and arg.rstrip("/") + "/" == ALLOWED_ORIGIN
722+ return isinstance(arg, str) and arg.rstrip("/") + "/" in allowed_origins()
461def main(argv=None):
725def main(argv=None):
@@ -464,6 +728,10 @@ def main(argv=None):
464 return selftest()
728 return selftest()
465 if argv and argv[0] == "--install-schemas":
729 if argv and argv[0] == "--install-schemas":
466 return install_schemas()
730 return install_schemas()
731+ if argv and argv[0] == "--register":
732+ return register(argv[1] if len(argv) > 1 else sys.argv[0])
733+ if argv and argv[0] == "--unregister":
467 if argv and argv[0] == "--print-manifest":
735 if argv and argv[0] == "--print-manifest":
468 print(json.dumps(manifest_for(argv[1] if len(argv) > 1 else os.path.realpath(__file__)), indent=2))
736 print(json.dumps(manifest_for(argv[1] if len(argv) > 1 else os.path.realpath(__file__)), indent=2))