@@ -25,39 +25,34 @@ sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)))
25import prompts # noqa: E402
25import prompts # noqa: E402
26import validate # noqa: E402
26import validate # noqa: E402
28−HOST_VERSION = "0.1.0" # release builds stamp the extension version here (scripts/build.mjs)
28+HOST_VERSION = "0.1.0"
29HOST_NAME = "io.diagonal.host"
29HOST_NAME = "io.diagonal.host"
30ALLOWED_ORIGIN = "chrome-extension://mpnodlalikgeehnlnofdkpgapkmbkjdf/"
30ALLOWED_ORIGIN = "chrome-extension://mpnodlalikgeehnlnofdkpgapkmbkjdf/"
31−# The Chrome Web Store gives the listed extension its own ID. Once the listing exists, put its origin here
32−# ("chrome-extension://<store id>/") so the store install and the unpacked one can both use the host.
34FM = os.environ.get("DIAGONAL_FM", "/usr/bin/fm") # tests point this at host/tests/fake_fm.py
31FM = os.environ.get("DIAGONAL_FM", "/usr/bin/fm") # tests point this at host/tests/fake_fm.py
35SUPPORT = os.environ.get("DIAGONAL_SUPPORT_DIR", os.path.expanduser("~/Library/Application Support/Diagonal"))
32SUPPORT = os.environ.get("DIAGONAL_SUPPORT_DIR", os.path.expanduser("~/Library/Application Support/Diagonal"))
36SCHEMAS = os.path.join(SUPPORT, "schemas")
33SCHEMAS = os.path.join(SUPPORT, "schemas")
37LOG_DIR = os.environ.get("DIAGONAL_LOG_DIR", os.path.expanduser("~/Library/Logs/Diagonal"))
34LOG_DIR = os.environ.get("DIAGONAL_LOG_DIR", os.path.expanduser("~/Library/Logs/Diagonal"))
38LOG_MAX_BYTES = 5 * 1024 * 1024
35LOG_MAX_BYTES = 5 * 1024 * 1024
39−# fm's context holds prompt and reply together. Measured on macOS 27 with the topics schema: a 7,400-token
36+CHAR_BUDGET = 10_000 # ≈ 2,850 tokens of prompt at 3.5 chars/token (section 9); double once 8,192 is confirmed
40−# prompt worked, 13,100 overflowed. 7,000 tokens leaves room for the reply.
42−# Tokens per character, measured with `fm count-tokens` on tab lists: English 0.26, Russian 0.31, Chinese 0.54,
43−# Japanese 0.51. The estimate rounds up, so a prompt it calls small enough is; near the limit fm counts exactly.
44−TOKENS_PER_CHAR = {"ascii": 0.3, "cjk": 0.6, "other": 0.36}
45−EXACT_COUNT_ABOVE = 0.75 # of TOKEN_BUDGET
46MAX_REQUEST = 64 * 1024 * 1024
37MAX_REQUEST = 64 * 1024 * 1024
47MAX_REPLY = 1024 * 1024
38MAX_REPLY = 1024 * 1024
48OPS = {"ping", "name", "organize"}
39OPS = {"ping", "name", "organize"}
50−# `fm schema` argument lists (section 8), checked against fm on macOS 27. `--object NAME` takes the nested
41+# `fm schema` argument lists (section 8). The nested form is unverified on a real macOS 27 machine;
51−# object's schema as JSON from `--schema`; "{TabTopic}" is replaced by the output of SUB_SCHEMAS["TabTopic"].
42+# when it fails, --install-schemas writes the two flat schemas and `organize` takes two calls.
53− "TabTopic": ["schema", "object", "--name", "TabTopic", "--integer", "index", "--string", "topic"],
55SCHEMA_COMMANDS = {
43SCHEMA_COMMANDS = {
56 "name.json": ["schema", "object", "--name", "GroupName", "--string", "title", "--string", "emoji"],
44 "name.json": ["schema", "object", "--name", "GroupName", "--string", "title", "--string", "emoji"],
57− "topics.json": ["schema", "object", "--name", "Topics", "--object", "tabs", "--schema", "{TabTopic}", "--array"],
45+ "organize.json": ["schema", "object", "--name", "Organized",
46+ "--object", "groups", "--array",
47+ "--string", "title", "--string", "emoji", "--string", "color",
48+ "--integer", "existing", "--integer", "members", "--array",
49+ "--integer", "leftovers", "--array"],
51+FALLBACK_SCHEMA_COMMANDS = {
52+ "organize-labels.json": ["schema", "object", "--name", "TopicLabels",
53+ "--string", "labels", "--array", "--string", "emojis", "--array", "--string", "colors", "--array"],
54+ "organize-assign.json": ["schema", "object", "--name", "Assignment", "--integer", "assignment", "--array"],
59−# Organize schemas written by older hosts; removed when the current ones are installed.
60−LEGACY_SCHEMAS = ["organize.json", "organize-labels.json", "organize-assign.json"]
63class Fail(Exception):
58class Fail(Exception):
@@ -135,7 +130,7 @@ def validate_request(req):
135 if not isinstance(it, dict):
130 if not isinstance(it, dict):
136 raise Fail("BAD_REQUEST", f"item {n} is not an object")
131 raise Fail("BAD_REQUEST", f"item {n} is not an object")
137 clean.append({"i": n, "title": _trim(it.get("title"), 120), "url": _trim(it.get("url"), 300),
132 clean.append({"i": n, "title": _trim(it.get("title"), 120), "url": _trim(it.get("url"), 300),
138− "description": _trim(it.get("description"), 500)})
133+ "description": _trim(it.get("description"), 300)})
139 payload["items"] = clean
134 payload["items"] = clean
140 for key in ("siblingTitles", "mustDifferFrom"):
135 for key in ("siblingTitles", "mustDifferFrom"):
141 payload[key] = [_trim(t, 60) for t in payload.get(key) or [] if isinstance(t, str)][:20]
136 payload[key] = [_trim(t, 60) for t in payload.get(key) or [] if isinstance(t, str)][:20]
@@ -148,8 +143,9 @@ def validate_request(req):
148 {"g": n, "title": _trim(g.get("title"), 60), "samples": [_trim(s, 60) for s in (g.get("samples") or [])[:2]]}
143 {"g": n, "title": _trim(g.get("title"), 60), "samples": [_trim(s, 60) for s in (g.get("samples") or [])[:2]]}
149 for n, g in enumerate(groups[:12]) if isinstance(g, dict)
144 for n, g in enumerate(groups[:12]) if isinstance(g, dict)
151− # fm on macOS 27 only has the on-device model; older extensions may still ask for "pcc".
146+ model = opts.get("model", "system")
152− opts["model"] = "system"
147+ if model not in ("system", "pcc"):
148+ raise Fail("BAD_REQUEST", "opts.model must be system or pcc")
153 return req
149 return req
@@ -182,7 +178,7 @@ def classify(stderr, returncode=None):
184def fm_env():
180def fm_env():
185− return {"PATH": "/usr/bin:/bin", "HOME": os.environ.get("HOME", ""), "LANG": "en_US.UTF-8", "NO_COLOR": "1"}
181+ return {"PATH": "/usr/bin:/bin", "HOME": os.environ.get("HOME", ""), "LANG": "en_US.UTF-8"}
188def schema_path(name):
184def schema_path(name):
@@ -190,46 +186,10 @@ def schema_path(name):
190 return path if os.path.isfile(path) and os.path.getsize(path) > 0 else None
186 return path if os.path.isfile(path) and os.path.getsize(path) > 0 else None
193−# How much of each page's text to send, most first: when a batch doesn't fit the budget, the pages' text
189+def check_budget(prompt, n_items):
194−# is shortened, then dropped, before the host gives up and asks for a smaller batch.
190+ if len(prompt) > CHAR_BUDGET:
195−DESC_CAPS = (500, 200, 0)
198−def estimate_tokens(prompt):
200− for ch in prompt.instructions + prompt.text:
201− n += TOKENS_PER_CHAR["ascii" if ch < "\x80" else "cjk" if validate.CJK.match(ch) else "other"]
202− return int(n) + 60 # fm's own framing around instructions and prompt
205−def count_tokens(prompt):
206− """fm's exact count (under 0.1 s), or None when it can't give one."""
208− p = subprocess.run([FM, "count-tokens", "-q", "-i", prompt.instructions], input=prompt.text,
209− capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=10, env=fm_env())
210− except (OSError, subprocess.TimeoutExpired):
212− out = p.stdout.strip()
213− return int(out) if p.returncode == 0 and out.isdigit() else None
216−def prompt_tokens(prompt):
217− estimate = estimate_tokens(prompt)
218− if estimate <= TOKEN_BUDGET * EXACT_COUNT_ABOVE:
220− return count_tokens(prompt) or estimate
223−def fit(build, n_items):
224− """The prompt with as much page text as fits TOKEN_BUDGET; OVER_BUDGET with how many items would fit if
225− even the titles alone are too long."""
226− for cap in DESC_CAPS:
228− tokens = prompt_tokens(prompt)
229− if tokens <= TOKEN_BUDGET:
231− allowed = max(1, int(n_items * TOKEN_BUDGET / tokens))
191+ allowed = max(1, int(n_items * CHAR_BUDGET / len(prompt)))
232− raise Fail("OVER_BUDGET", f"prompt is about {tokens} tokens, budget {TOKEN_BUDGET}", retryable=True, allowedItems=allowed)
192+ raise Fail("OVER_BUDGET", f"prompt is {len(prompt)} chars, budget {CHAR_BUDGET}", retryable=True, allowedItems=allowed)
235def extract_json(stdout):
195def extract_json(stdout):
@@ -249,100 +209,49 @@ def extract_json(stdout):
249 raise Fail("BAD_MODEL_OUTPUT", "no JSON object in fm output", raw=text[:2000])
209 raise Fail("BAD_MODEL_OUTPUT", "no JSON object in fm output", raw=text[:2000])
252−# fm's safety layer sometimes refuses harmless tab lists. Naming tabs only restates their own text, so a
253−# refused call is retried once in the mode Apple provides for transforming given content.
254−PERMISSIVE = ["--guardrails", "permissive-content-transformations"]
258−# Below this much time left in a request, a second fm call (the permissive retry, naming one more new
259−# group) is not started: it would only be cut off.
263−def run_fm(prompt, schema, opts):
212+def run_fm(prompt, schema, model, timeout_s):
264− """`prompt` is a prompts.Prompt: its fixed rules go to -i, the tab text on stdin. Every call in one
265− request shares its deadline, so the extension's guard (its timeout plus 10 s) is a real upper bound."""
267− return _run_fm(prompt, schema, opts, [])
269− if f.code != "GUARDRAIL" or time_left(opts) < MIN_CALL_S:
271− return _run_fm(prompt, schema, opts, PERMISSIVE)
275− return opts["deadline"] - time.time()
278−def _run_fm(prompt, schema, opts, extra):
279− # The tab text goes on stdin (fm reads it there when no prompt argument is given), so it never shows up
280− # in the process list. fm has no timeout of its own; subprocess enforces ours.
281− # --greedy: the same tabs get the same topics and name on every run (measured on macOS 27: five runs,
282− # one answer), so groups don't get renamed or reshuffled each time Diagonal looks at them. The strict
283− # retry after an unusable answer samples instead: greedy would give the same answer again.
284− greedy = [] if opts.get("strict") else ["--greedy"]
285− args = [FM, "respond", "--model", opts["model"], "--no-stream", *greedy, "--schema", schema, "-i", prompt.instructions, *extra]
213+ args = [FM, "respond", "--model", model, "--no-stream", "--schema", schema, "--", prompt]
286− timeout_s = max(1.0, time_left(opts))
288− p = subprocess.run(args, input=prompt.text, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=timeout_s, env=fm_env())
215+ p = subprocess.run(args, capture_output=True, text=True, timeout=timeout_s, env=fm_env())
289 except FileNotFoundError:
216 except FileNotFoundError:
290 raise Fail("MODEL_UNAVAILABLE", f"fm not found at {FM} — requires macOS 27")
217 raise Fail("MODEL_UNAVAILABLE", f"fm not found at {FM} — requires macOS 27")
291 except subprocess.TimeoutExpired:
218 except subprocess.TimeoutExpired:
292− raise Fail("TIMEOUT", f"fm exceeded {opts['timeout_s']:g}s", retryable=True)
219+ raise Fail("TIMEOUT", f"fm exceeded {timeout_s:g}s", retryable=True)
293 if p.returncode != 0:
220 if p.returncode != 0:
294 code = classify(p.stderr, p.returncode)
221 code = classify(p.stderr, p.returncode)
295 if code == "LICENSE_REQUIRED":
222 if code == "LICENSE_REQUIRED":
296 raise Fail(code, LICENSE_MESSAGE, raw=(p.stderr or "")[:2000])
223 raise Fail(code, LICENSE_MESSAGE, raw=(p.stderr or "")[:2000])
297− # 64 is fm's usage error (unknown flag or value): the same call fails the same way again.
298− raise Fail(code, (p.stderr or f"fm exited {p.returncode}").strip()[:500], retryable=p.returncode != USAGE_EXIT,
224+ raise Fail(code, (p.stderr or f"fm exited {p.returncode}").strip()[:500], retryable=True, raw=(p.stderr or "")[:2000])
299− raw=(p.stderr or "")[:2000])
300 return extract_json(p.stdout)
225 return extract_json(p.stdout)
303# ----- ops --------------------------------------------------------------------------------------
228# ----- ops --------------------------------------------------------------------------------------
305def organize_mode():
230def organize_mode():
306− return "topics" if schema_path("topics.json") else "missing"
231+ if schema_path("organize.json"):
233+ if schema_path("organize-labels.json") and schema_path("organize-assign.json"):
309def op_ping(_payload, _opts):
238def op_ping(_payload, _opts):
310 if not os.path.exists(FM):
239 if not os.path.exists(FM):
311− return {"hostVersion": HOST_VERSION, "fmPath": FM, "fmAvailable": False, "licenseRequired": False,
240+ return {"hostVersion": HOST_VERSION, "fmPath": FM, "fmAvailable": False,
312 "fmMessage": f"fm not found at {FM} — requires macOS 27", "schemasOk": False, "organizeMode": organize_mode()}
241 "fmMessage": f"fm not found at {FM} — requires macOS 27", "schemasOk": False, "organizeMode": organize_mode()}
313 license_required = False
242 license_required = False
315− avail = subprocess.run([FM, "available", "--model", "system"], capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=15, env=fm_env())
244+ avail = subprocess.run([FM, "available", "--model", "system"], capture_output=True, text=True, timeout=15, env=fm_env())
316 ok, msg = avail.returncode == 0, (avail.stdout + avail.stderr).strip()[:300]
245 ok, msg = avail.returncode == 0, (avail.stdout + avail.stderr).strip()[:300]
317 if not ok and license_needed(avail.returncode, avail.stdout + avail.stderr):
246 if not ok and license_needed(avail.returncode, avail.stdout + avail.stderr):
318 license_required, msg = True, LICENSE_MESSAGE
247 license_required, msg = True, LICENSE_MESSAGE
319 except subprocess.TimeoutExpired:
248 except subprocess.TimeoutExpired:
320 ok, msg = False, "fm available timed out"
249 ok, msg = False, "fm available timed out"
321− schema_message = None
324− ensure_schemas()
251+ ensure_schemas()
326− schema_message = f.message
327 mode = organize_mode()
252 mode = organize_mode()
328− reply = {"hostVersion": HOST_VERSION, "fmPath": FM, "fmAvailable": ok, "fmMessage": msg,
253+ return {"hostVersion": HOST_VERSION, "fmPath": FM, "fmAvailable": ok, "fmMessage": msg,
329− "licenseRequired": license_required, "schemasOk": schemas_ok(), "organizeMode": mode}
254+ "licenseRequired": license_required, "schemasOk": schemas_ok(), "organizeMode": mode}
331− reply["schemaMessage"] = schema_message
335−# Bumped when SCHEMA_COMMANDS change, so schemas written by an older host are rewritten. Version 3: organize
336−# asks for one topic per tab (topics.json) instead of whole groups.
340−def schemas_current():
342− with open(os.path.join(SCHEMAS, "version"), encoding="utf-8") as f:
343− return f.read().strip() == SCHEMA_VERSION
348def schemas_ok():
257def schemas_ok():
@@ -352,86 +261,50 @@ def schemas_ok():
352def ensure_schemas():
261def ensure_schemas():
353 """Write the schema files on first use, so install order does not matter (e.g. the fm terms were
262 """Write the schema files on first use, so install order does not matter (e.g. the fm terms were
354 accepted after the installer ran). stdout is the native-messaging channel: progress lines are swallowed."""
263 accepted after the installer ran). stdout is the native-messaging channel: progress lines are swallowed."""
355− if schemas_ok() and schemas_current():
264+ if schemas_ok():
358− with contextlib.redirect_stdout(out), contextlib.redirect_stderr(out):
266+ with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
359 rc = install_schemas()
267 rc = install_schemas()
360 if rc == INSTALL_LICENSE_EXIT:
268 if rc == INSTALL_LICENSE_EXIT:
361 raise Fail("LICENSE_REQUIRED", LICENSE_MESSAGE)
269 raise Fail("LICENSE_REQUIRED", LICENSE_MESSAGE)
362− if not schemas_ok():
363− lines = [ln for ln in out.getvalue().splitlines() if ln.strip()]
364− why = lines[-1] if lines else f"fm schema exited {rc}"
365− raise Fail("SCHEMA_MISSING", f"fm could not write Diagonal's schema files: {why}"[:500], raw=out.getvalue()[-2000:])
368−def _fit_overflow(f, n_items):
369− """fm's own context-size error carries no item count; estimate one so the extension can split the batch."""
370− if f.code == "OVER_BUDGET" and "allowedItems" not in f.extra:
371− f.extra["allowedItems"] = max(1, (n_items * 2) // 3)
376def op_name(payload, opts):
272def op_name(payload, opts):
377 ensure_schemas()
273 ensure_schemas()
378 schema = schema_path("name.json")
274 schema = schema_path("name.json")
379 if not schema:
275 if not schema:
380− raise Fail("SCHEMA_MISSING", f"fm could not write {os.path.join(SCHEMAS, 'name.json')}")
276+ raise Fail("SCHEMA_MISSING", os.path.join(SCHEMAS, "name.json"))
381− prompt = fit(lambda cap: prompts.build_name_prompt(payload, strict=opts.get("strict", False), desc_cap=cap), len(payload["items"]))
277+ prompt = prompts.build_name_prompt(payload, strict=opts.get("strict", False))
382− try:
278+ check_budget(prompt, len(payload["items"]))
383− out = run_fm(prompt, schema, opts)
279+ out = run_fm(prompt, schema, opts["model"], opts["timeout_s"])
385− raise _fit_overflow(f, len(payload["items"]))
387− return validate.validate_name(out, payload), prompts.size(prompt), "name"
281+ return validate.validate_name(out, payload), len(prompt), "name"
388 except validate.ValidationError as e:
282 except validate.ValidationError as e:
389 raise Fail("BAD_MODEL_OUTPUT", str(e), retryable=True, raw=json.dumps(out, ensure_ascii=False)[:2000])
283 raise Fail("BAD_MODEL_OUTPUT", str(e), retryable=True, raw=json.dumps(out, ensure_ascii=False)[:2000])
392def op_organize(payload, opts):
286def op_organize(payload, opts):
287+ strict = opts.get("strict", False)
393 ensure_schemas()
288 ensure_schemas()
394− schema = schema_path("topics.json")
289+ mode = organize_mode()
396− raise Fail("SCHEMA_MISSING", f"fm could not write {os.path.join(SCHEMAS, 'topics.json')}")
397− prompt = fit(lambda cap: prompts.build_topics_prompt(payload, strict=opts.get("strict", False), desc_cap=cap), len(payload["items"]))
399− out = run_fm(prompt, schema, opts)
401− raise _fit_overflow(f, len(payload["items"]))
402− _, owner = prompts.topic_items(payload)
404− result = validate.organize_from_topics(out, payload, owner)
291+ if mode == "nested":
292+ prompt = prompts.build_organize_prompt(payload, strict=strict)
293+ check_budget(prompt, len(payload["items"]))
294+ out = run_fm(prompt, schema_path("organize.json"), opts["model"], opts["timeout_s"])
295+ return validate.validate_organize(out, payload), len(prompt), "nested"
296+ if mode == "two-call":
297+ p1 = prompts.build_labels_prompt(payload, strict=strict)
298+ check_budget(p1, len(payload["items"]))
299+ labels = run_fm(p1, schema_path("organize-labels.json"), opts["model"], opts["timeout_s"])
300+ names = [x for x in (labels.get("labels") or []) if isinstance(x, str)] if isinstance(labels, dict) else []
301+ p2 = prompts.build_assign_prompt(payload, names, strict=strict)
302+ check_budget(p2, len(payload["items"]))
303+ assign = run_fm(p2, schema_path("organize-assign.json"), opts["model"], opts["timeout_s"])
304+ return validate.organize_from_two_calls(labels, assign, payload), len(p1) + len(p2), "two-call"
405 except validate.ValidationError as e:
305 except validate.ValidationError as e:
406− raise Fail("BAD_MODEL_OUTPUT", str(e), retryable=True, raw=json.dumps(out, ensure_ascii=False)[:2000])
306+ raise Fail("BAD_MODEL_OUTPUT", str(e), retryable=True)
407− chars = prompts.size(prompt)
307+ raise Fail("SCHEMA_MISSING", os.path.join(SCHEMAS, "organize.json"))
408− if payload.get("nameGroups") is not False: # the extension's fit check only needs the topics
409− chars += name_new_groups(result, payload, opts)
410− return result, chars, "topics"
413−def name_new_groups(result, payload, opts):
414− """Title each new group with the name op (well under a second each on fm), so groups appear named.
415− A group that fails, runs past the request's deadline, or gets a title another group in the window
416− already has keeps "" and the extension's naming loop, which checks every sibling, names it."""
417− taken = [t for t in [g.get("title") for g in payload.get("existingGroups") or []] + list(payload.get("siblingTitles") or []) if t]
419− for g in result["groups"]:
420− if "existing" in g or g.get("title"):
422− if time_left(opts) < MIN_CALL_S:
424− sub = {"items": [dict(payload["items"][i], i=n) for n, i in enumerate(g["members"])], "siblingTitles": taken[:20]}
426− named, n, _ = op_name(sub, {**opts, "strict": False})
430− if any(validate.same_label(named["title"], t) for t in taken):
432− g["title"], g["emoji"] = named["title"], named["emoji"]
433− taken.append(named["title"])
437def handle(req):
310def handle(req):
@@ -444,8 +317,8 @@ def handle(req):
444 raw_opts = req.get("opts") or {}
317 raw_opts = req.get("opts") or {}
445 timeout_ms = raw_opts.get("timeoutMs", 45000)
318 timeout_ms = raw_opts.get("timeoutMs", 45000)
446 timeout_ms = timeout_ms if isinstance(timeout_ms, (int, float)) and not isinstance(timeout_ms, bool) else 45000
319 timeout_ms = timeout_ms if isinstance(timeout_ms, (int, float)) and not isinstance(timeout_ms, bool) else 45000
320+ opts = {"model": raw_opts.get("model", "system"), "strict": bool(raw_opts.get("strict")),
447− timeout_s = max(5.0, min(120.0, timeout_ms / 1000))
321+ "timeout_s": max(5.0, min(120.0, timeout_ms / 1000))}
448− opts = {"model": "system", "strict": bool(raw_opts.get("strict")), "timeout_s": timeout_s, "deadline": t0 + timeout_s}
449 op = req["op"]
322 op = req["op"]
450 if op == "ping":
323 if op == "ping":
451 return {"v": 1, "id": rid, "ok": True, "result": op_ping(req["payload"], opts)}
324 return {"v": 1, "id": rid, "ok": True, "result": op_ping(req["payload"], opts)}
@@ -494,24 +367,8 @@ def install_schemas():
495 def write(name, args):
368 def write(name, args):
496 """True when written; "license" or "rejected" (fm ran and refused) or "error" otherwise."""
369 """True when written; "license" or "rejected" (fm ran and refused) or "error" otherwise."""
498− for key, sub_args in SUB_SCHEMAS.items():
499− if "{%s}" % key not in args:
502− p = subprocess.run([FM, *sub_args], capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=30, env=fm_env())
503− except (FileNotFoundError, subprocess.TimeoutExpired) as e:
504− print(f" {name}: fm failed ({e})", file=sys.stderr)
506− if license_needed(p.returncode, p.stdout + p.stderr):
508− if p.returncode != 0 or not p.stdout.strip():
509− print(f" {name}: fm schema ({key}) exited {p.returncode}: {p.stderr.strip()[:200]}", file=sys.stderr)
511− subs["{%s}" % key] = p.stdout.strip()
512− args = [subs.get(a, a) for a in args]
514− p = subprocess.run([FM, *args], capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=30, env=fm_env())
371+ p = subprocess.run([FM, *args], capture_output=True, text=True, timeout=30, env=fm_env())
515 except (FileNotFoundError, subprocess.TimeoutExpired) as e:
372 except (FileNotFoundError, subprocess.TimeoutExpired) as e:
516 print(f" {name}: fm failed ({e})", file=sys.stderr)
373 print(f" {name}: fm failed ({e})", file=sys.stderr)
517 return "error"
374 return "error"
@@ -531,19 +388,24 @@ def install_schemas():
531 print(" Diagonal writes its schemas on its own once that is done.", file=sys.stderr)
388 print(" Diagonal writes its schemas on its own once that is done.", file=sys.stderr)
532 return INSTALL_LICENSE_EXIT
389 return INSTALL_LICENSE_EXIT
533 ok &= first is True
390 ok &= first is True
534− topics = write("topics.json", SCHEMA_COMMANDS["topics.json"])
391+ nested = write("organize.json", SCHEMA_COMMANDS["organize.json"])
535− if topics == "license":
536− return INSTALL_LICENSE_EXIT
537− ok &= topics is True
392+ if nested is True:
538− if ok:
393+ for name in FALLBACK_SCHEMA_COMMANDS:
539− for name in LEGACY_SCHEMAS:
541 os.remove(os.path.join(SCHEMAS, name))
395 os.remove(os.path.join(SCHEMAS, name))
542 except FileNotFoundError:
396 except FileNotFoundError:
544− if ok:
398+ elif nested != "rejected":
399+ # fm did not get as far as judging the schema: no reason to think nesting is unsupported.
400+ return INSTALL_LICENSE_EXIT if nested == "license" else 1
402+ print(" nested organize schema unsupported; using the two-call fallback", file=sys.stderr)
545− with open(os.path.join(SCHEMAS, "version"), "w", encoding="utf-8") as f:
404+ os.remove(os.path.join(SCHEMAS, "organize.json"))
546− f.write(SCHEMA_VERSION + "\n")
405+ except FileNotFoundError:
407+ for name, args in FALLBACK_SCHEMA_COMMANDS.items():
408+ ok &= write(name, args) is True
547 return 0 if ok else 1
409 return 0 if ok else 1
@@ -568,16 +430,14 @@ def selftest():
569 ping = op_ping({}, {})
431 ping = op_ping({}, {})
570 check("fm found", os.path.exists(FM), FM)
432 check("fm found", os.path.exists(FM), FM)
571− if not os.path.exists(FM):
572− return 1 # nothing else can work without it
573− if ping.get("licenseRequired"):
433+ if ping["licenseRequired"]:
574 check("fm terms accepted", False, f"run: {LICENSE_FIX}")
434 check("fm terms accepted", False, f"run: {LICENSE_FIX}")
576 check("model available (fm available --model system)", ping["fmAvailable"], ping["fmMessage"])
436 check("model available (fm available --model system)", ping["fmAvailable"], ping["fmMessage"])
577− for name in SCHEMA_COMMANDS:
437+ for name in ["name.json", *(list(FALLBACK_SCHEMA_COMMANDS) if organize_mode() == "two-call" else ["organize.json"])]:
578 path = os.path.join(SCHEMAS, name)
438 path = os.path.join(SCHEMAS, name)
579 readable = os.path.isfile(path) and os.path.getsize(path) > 0
439 readable = os.path.isfile(path) and os.path.getsize(path) > 0
580− hint = f"written on their own after {LICENSE_FIX}" if ping.get("licenseRequired") else "missing: run diagonal-host --install-schemas"
440+ hint = f"written on their own after {LICENSE_FIX}" if ping["licenseRequired"] else "missing: run diagonal-host --install-schemas"
581 check(f"schema {name}", readable, path if readable else hint)
441 check(f"schema {name}", readable, path if readable else hint)
582 check("allowed origin pinned", "<" not in ALLOWED_ORIGIN, ALLOWED_ORIGIN)
442 check("allowed origin pinned", "<" not in ALLOWED_ORIGIN, ALLOWED_ORIGIN)
583 if ping["fmAvailable"] and schema_path("name.json"):
443 if ping["fmAvailable"] and schema_path("name.json"):
@@ -589,137 +449,13 @@ def selftest():
589 return 0 if failures == 0 else 1
449 return 0 if failures == 0 else 1
592−# ----- registering with browsers ------------------------------------------------------------------
594−# Chromium reads per-user host manifests from <user data dir>/NativeMessagingHosts. Known user data
595−# dirs under ~/Library/Application Support; anything else that looks like a Chromium browser profile
596−# is found by browser_dirs()' scan.
597−KNOWN_BROWSER_DIRS = [
598− "Google/Chrome", "Google/Chrome Beta", "Google/Chrome Dev", "Google/Chrome Canary", "Chromium",
599− "BraveSoftware/Brave-Browser", "BraveSoftware/Brave-Origin", "BraveSoftware/Brave-Browser-Beta",
600− "BraveSoftware/Brave-Browser-Nightly", "Microsoft Edge", "Microsoft Edge Beta", "Microsoft Edge Dev",
601− "Microsoft Edge Canary", "Vivaldi", "Vivaldi Snapshot", "Arc/User Data", "Dia/User Data",
602− "com.operasoftware.Opera", "com.operasoftware.OperaGX", "Thorium", "net.imput.helium", "Yandex/YandexBrowser",
604−BROWSER_SCAN_DEPTH = 3
605−REGISTERED_LIST = "registered-browsers.txt" # next to the host: where --register wrote manifests
609− return os.environ.get("DIAGONAL_APP_SUPPORT", os.path.expanduser("~/Library/Application Support"))
612−def looks_like_browser(user_data_dir):
613− """A Chromium browser's user data dir: `Local State` plus a profile with browsing data.
614− Electron apps also have `Local State`, but no History, Favicons or Top Sites."""
615− if not os.path.isfile(os.path.join(user_data_dir, "Local State")):
618− profiles = [e for e in os.listdir(user_data_dir) if e == "Default" or e.startswith("Profile ")]
621− return any(os.path.exists(os.path.join(user_data_dir, p, f)) for p in profiles for f in ("History", "Favicons", "Top Sites"))
625− base = app_support()
626− found = [os.path.join(base, rel) for rel in KNOWN_BROWSER_DIRS if os.path.isdir(os.path.join(base, rel))]
627− for root, dirs, _files in os.walk(base):
628− depth = os.path.relpath(root, base).count(os.sep) + (root != base)
629− if root != base and (root in found or looks_like_browser(root)):
630− if root not in found:
632− dirs[:] = [] # profiles inside need no scan
634− if depth >= BROWSER_SCAN_DEPTH or os.path.basename(root) == "Diagonal":
636− return sorted(set(found))
639−def manifest_problems(path, host_path):
641− m = json.load(open(path, encoding="utf-8"))
642− except (OSError, ValueError) as e:
643− return [f"unreadable ({e})"]
645− if m.get("name") != HOST_NAME:
646− problems.append(f"name is {m.get('name')!r}")
647− if m.get("path") != host_path or not os.path.isabs(host_path):
648− problems.append(f"path is {m.get('path')!r}")
649− elif not os.access(host_path, os.X_OK):
650− problems.append(f"host {host_path} is missing or not executable")
651− if m.get("allowed_origins") != allowed_origins():
652− problems.append(f"allowed_origins is {m.get('allowed_origins')}")
656−def register(host_path):
657− """Write this host's manifest into every Chromium browser's NativeMessagingHosts folder, check each
658− one the way the browser will read it, and remember where they went for --unregister."""
659− host_path = os.path.abspath(host_path)
660− body = json.dumps(manifest_for(host_path), indent=2) + "\n"
661− dirs = browser_dirs()
663− print("No Chromium browser found. Open your browser once, then run this again.", file=sys.stderr)
665− written, failures = [], 0
667− target_dir = os.path.join(d, "NativeMessagingHosts")
668− target = os.path.join(target_dir, f"{HOST_NAME}.json")
670− os.makedirs(target_dir, exist_ok=True)
671− with open(target + ".tmp", "w", encoding="utf-8") as f:
673− os.replace(target + ".tmp", target) # a browser never reads a half-written file
674− except OSError as e:
675− print(f" {os.path.relpath(d, app_support())}: could not write ({e})", file=sys.stderr)
678− problems = manifest_problems(target, host_path)
680− print(f" {os.path.relpath(d, app_support())}: {'; '.join(problems)}", file=sys.stderr)
683− print(f"registered: {os.path.relpath(d, app_support())}")
684− written.append(target)
686− with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), REGISTERED_LIST), "w", encoding="utf-8") as f:
687− f.write("\n".join(written) + "\n")
690− # One browser that could not be written (a locked folder, a stale profile) does not undo the others:
691− # the install goes on and the ones that failed were named above.
692− return 0 if len(dirs) - failures > 0 else 1
696− """Remove every manifest --register wrote, plus any other copy of ours a browser folder still has."""
699− listed = open(os.path.join(os.path.dirname(os.path.realpath(__file__)), REGISTERED_LIST), encoding="utf-8").read().split("\n")
702− candidates = set(filter(None, listed)) | {os.path.join(d, "NativeMessagingHosts", f"{HOST_NAME}.json") for d in browser_dirs()}
703− for path in sorted(candidates):
706− print(f"removed: {path}")
712def manifest_for(path):
452def manifest_for(path):
713 return {"name": HOST_NAME, "description": "Diagonal: names tab groups with Apple's on-device model",
453 return {"name": HOST_NAME, "description": "Diagonal: names tab groups with Apple's on-device model",
714− "path": os.path.abspath(path), "type": "stdio", "allowed_origins": allowed_origins()}
454+ "path": os.path.abspath(path), "type": "stdio", "allowed_origins": [ALLOWED_ORIGIN]}
717−def allowed_origins():
718− return [o for o in (ALLOWED_ORIGIN, STORE_ORIGIN) if o]
721def origin_ok(arg):
457def origin_ok(arg):
722− return isinstance(arg, str) and arg.rstrip("/") + "/" in allowed_origins()
458+ return isinstance(arg, str) and arg.rstrip("/") + "/" == ALLOWED_ORIGIN
725def main(argv=None):
461def main(argv=None):
@@ -728,10 +464,6 @@ def main(argv=None):
728 return selftest()
464 return selftest()
729 if argv and argv[0] == "--install-schemas":
465 if argv and argv[0] == "--install-schemas":
730 return install_schemas()
466 return install_schemas()
731− if argv and argv[0] == "--register":
732− return register(argv[1] if len(argv) > 1 else sys.argv[0])
733− if argv and argv[0] == "--unregister":
735 if argv and argv[0] == "--print-manifest":
467 if argv and argv[0] == "--print-manifest":
736 print(json.dumps(manifest_for(argv[1] if len(argv) > 1 else os.path.realpath(__file__)), indent=2))
468 print(json.dumps(manifest_for(argv[1] if len(argv) > 1 else os.path.realpath(__file__)), indent=2))