39441a5..master · diagonal

git.iko.soy/diagonal

on master help sign in

.github/workflows/release.yml+23−8

since splitsnapshots

unifiedsplit

@@ -9,8 +9,11 @@ on:

9permissions:

10 contents: write

12+# One release at a time, in push order: an older run finishing last must not become the latest release,

13+# which is what the Homebrew cask downloads.

12concurrency:

14concurrency:

13− group: release-${{ github.sha }}

15+ group: release

16+ cancel-in-progress: false

15jobs:

18jobs:

16 release:

19 release:

@@ -32,8 +35,23 @@ jobs:

33 - name: Version from commit time

36 - name: Version from commit time

34 id: version

37 id: version

38+ env:

39+ GH_TOKEN: ${{ github.token }}

35 run: |

40 run: |

36− version=$(TZ=UTC git log -1 --format=%cd --date=format-local:%Y.%m.%d.%H%M "$GITHUB_SHA")

41+ # The commit's minute, or the next free minute when another commit already released in it. A re-run

42+ # for a commit that already released publishes nothing: a rebuilt zip has new bytes, which would

43+ # break the sha256 the tap already has, and users on that version would never get the new one.

44+ t=$(git log -1 --format=%ct "$GITHUB_SHA")

45+ for _ in $(seq 1 30); do

46+ version=$(date -u -d "@$t" +%Y.%m.%d.%H%M)

47+ target=$(gh release view "$version" --json targetCommitish -q .targetCommitish 2>/dev/null || true)

48+ [ -z "$target" ] && break

49+ if [ "$target" = "$GITHUB_SHA" ]; then

50+ echo "published=true" >> "$GITHUB_OUTPUT"

51+ break

52+ fi

53+ t=$((t + 60))

54+ done

37 echo "version=$version" >> "$GITHUB_OUTPUT"

55 echo "version=$version" >> "$GITHUB_OUTPUT"

38 echo "Version $version"

56 echo "Version $version"

@@ -54,22 +72,19 @@ jobs:

54 npm run zip:store

72 npm run zip:store

56 - name: Publish release

74 - name: Publish release

75+ if: steps.version.outputs.published != 'true'

57 env:

76 env:

58 GH_TOKEN: ${{ github.token }}

77 GH_TOKEN: ${{ github.token }}

59 VERSION: ${{ steps.version.outputs.version }}

78 VERSION: ${{ steps.version.outputs.version }}

60 run: |

79 run: |

61 asset="diagonal-extension-${VERSION}.zip"

80 asset="diagonal-extension-${VERSION}.zip"

62 notes="Built from ${GITHUB_SHA::7}. Install with: brew install iko-soy/tap/diagonal. Or by hand: unzip, load the folder at brave://extensions (Developer mode → Load unpacked), then install the native host once from Terminal: bash <that folder>/install-host.command"

81 notes="Built from ${GITHUB_SHA::7}. Install with: brew install iko-soy/tap/diagonal. Or by hand: unzip, load the folder at brave://extensions (Developer mode → Load unpacked), then install the native host once from Terminal: bash <that folder>/install-host.command"

63− if gh release view "$VERSION" >/dev/null 2>&1; then

64− # Two commits in the same minute share a version: refresh the asset on the existing release.

65− gh release upload "$VERSION" "$asset" diagonal-extension.zip diagonal-store.zip --clobber

66− else

67− gh release create "$VERSION" "$asset" diagonal-extension.zip diagonal-store.zip --target "$GITHUB_SHA" --title "Diagonal $VERSION" --notes "$notes" --latest

82+ gh release create "$VERSION" "$asset" diagonal-extension.zip diagonal-store.zip --target "$GITHUB_SHA" --title "Diagonal $VERSION" --notes "$notes" --latest

68− fi

70 # Uploads the release to the Chrome Web Store and submits it for review, once the listing exists and

84 # Uploads the release to the Chrome Web Store and submits it for review, once the listing exists and

71 # its API credentials are set as repository secrets (docs/store/listing.md). Skipped until then.

85 # its API credentials are set as repository secrets (docs/store/listing.md). Skipped until then.

72 - name: Upload to the Chrome Web Store

86 - name: Upload to the Chrome Web Store

87+ if: steps.version.outputs.published != 'true'

73 env:

88 env:

74 CWS_EXTENSION_ID: ${{ secrets.CWS_EXTENSION_ID }}

89 CWS_EXTENSION_ID: ${{ secrets.CWS_EXTENSION_ID }}

75 CWS_CLIENT_ID: ${{ secrets.CWS_CLIENT_ID }}

90 CWS_CLIENT_ID: ${{ secrets.CWS_CLIENT_ID }}

host/diagonal-host.py+49−30

@@ -206,7 +206,7 @@ def count_tokens(prompt):

206 """fm's exact count (under 0.1 s), or None when it can't give one."""

207 try:

208 p = subprocess.run([FM, "count-tokens", "-q", "-i", prompt.instructions], input=prompt.text,

209− capture_output=True, text=True, timeout=10, env=fm_env())

209+ capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=10, env=fm_env())

210 except (OSError, subprocess.TimeoutExpired):

211 return None

212 out = p.stdout.strip()

@@ -255,28 +255,41 @@ PERMISSIVE = ["--guardrails", "permissive-content-transformations"]

255USAGE_EXIT = 64

256

257

258+# Below this much time left in a request, a second fm call (the permissive retry, naming one more new

259+# group) is not started: it would only be cut off.

260+MIN_CALL_S = 5.0

261+

262+

258−def run_fm(prompt, schema, model, timeout_s):

263+def run_fm(prompt, schema, opts):

259− """`prompt` is a prompts.Prompt: its fixed rules go to -i, the tab text on stdin."""

264+ """`prompt` is a prompts.Prompt: its fixed rules go to -i, the tab text on stdin. Every call in one

265+ request shares its deadline, so the extension's guard (its timeout plus 10 s) is a real upper bound."""

260 try:

266 try:

261− return _run_fm(prompt, schema, model, timeout_s, [])

267+ return _run_fm(prompt, schema, opts, [])

262 except Fail as f:

268 except Fail as f:

263− if f.code != "GUARDRAIL":

269+ if f.code != "GUARDRAIL" or time_left(opts) < MIN_CALL_S:

264 raise

270 raise

265− return _run_fm(prompt, schema, model, timeout_s, PERMISSIVE)

271+ return _run_fm(prompt, schema, opts, PERMISSIVE)

272+

273+

274+def time_left(opts):

275+ return opts["deadline"] - time.time()

266

276

267

277

268−def _run_fm(prompt, schema, model, timeout_s, extra):

278+def _run_fm(prompt, schema, opts, extra):

269 # The tab text goes on stdin (fm reads it there when no prompt argument is given), so it never shows up

279 # The tab text goes on stdin (fm reads it there when no prompt argument is given), so it never shows up

270 # in the process list. fm has no timeout of its own; subprocess enforces ours.

280 # in the process list. fm has no timeout of its own; subprocess enforces ours.

271 # --greedy: the same tabs get the same topics and name on every run (measured on macOS 27: five runs,

281 # --greedy: the same tabs get the same topics and name on every run (measured on macOS 27: five runs,

272− # one answer), so groups don't get renamed or reshuffled each time Diagonal looks at them.

282+ # one answer), so groups don't get renamed or reshuffled each time Diagonal looks at them. The strict

283+ # retry after an unusable answer samples instead: greedy would give the same answer again.

284+ greedy = [] if opts.get("strict") else ["--greedy"]

273− args = [FM, "respond", "--model", model, "--no-stream", "--greedy", "--schema", schema, "-i", prompt.instructions, *extra]

285+ args = [FM, "respond", "--model", opts["model"], "--no-stream", *greedy, "--schema", schema, "-i", prompt.instructions, *extra]

286+ timeout_s = max(1.0, time_left(opts))

274 try:

287 try:

275− p = subprocess.run(args, input=prompt.text, capture_output=True, text=True, timeout=timeout_s, env=fm_env())

288+ p = subprocess.run(args, input=prompt.text, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=timeout_s, env=fm_env())

276 except FileNotFoundError:

289 except FileNotFoundError:

277 raise Fail("MODEL_UNAVAILABLE", f"fm not found at {FM} — requires macOS 27")

290 raise Fail("MODEL_UNAVAILABLE", f"fm not found at {FM} — requires macOS 27")

278 except subprocess.TimeoutExpired:

291 except subprocess.TimeoutExpired:

279− raise Fail("TIMEOUT", f"fm exceeded {timeout_s:g}s", retryable=True)

292+ raise Fail("TIMEOUT", f"fm exceeded {opts['timeout_s']:g}s", retryable=True)

280 if p.returncode != 0:

293 if p.returncode != 0:

281 code = classify(p.stderr, p.returncode)

294 code = classify(p.stderr, p.returncode)

282 if code == "LICENSE_REQUIRED":

295 if code == "LICENSE_REQUIRED":

@@ -295,11 +308,11 @@ def organize_mode():

295

308

296def op_ping(_payload, _opts):

309def op_ping(_payload, _opts):

297 if not os.path.exists(FM):

310 if not os.path.exists(FM):

298− return {"hostVersion": HOST_VERSION, "fmPath": FM, "fmAvailable": False,

311+ return {"hostVersion": HOST_VERSION, "fmPath": FM, "fmAvailable": False, "licenseRequired": False,

299 "fmMessage": f"fm not found at {FM} — requires macOS 27", "schemasOk": False, "organizeMode": organize_mode()}

312 "fmMessage": f"fm not found at {FM} — requires macOS 27", "schemasOk": False, "organizeMode": organize_mode()}

300 license_required = False

313 license_required = False

301 try:

314 try:

302− avail = subprocess.run([FM, "available", "--model", "system"], capture_output=True, text=True, timeout=15, env=fm_env())

315+ avail = subprocess.run([FM, "available", "--model", "system"], capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=15, env=fm_env())

303 ok, msg = avail.returncode == 0, (avail.stdout + avail.stderr).strip()[:300]

316 ok, msg = avail.returncode == 0, (avail.stdout + avail.stderr).strip()[:300]

304 if not ok and license_needed(avail.returncode, avail.stdout + avail.stderr):

317 if not ok and license_needed(avail.returncode, avail.stdout + avail.stderr):

305 license_required, msg = True, LICENSE_MESSAGE

318 license_required, msg = True, LICENSE_MESSAGE

@@ -367,7 +380,7 @@ def op_name(payload, opts):

367 raise Fail("SCHEMA_MISSING", f"fm could not write {os.path.join(SCHEMAS, 'name.json')}")

380 raise Fail("SCHEMA_MISSING", f"fm could not write {os.path.join(SCHEMAS, 'name.json')}")

368 prompt = fit(lambda cap: prompts.build_name_prompt(payload, strict=opts.get("strict", False), desc_cap=cap), len(payload["items"]))

381 prompt = fit(lambda cap: prompts.build_name_prompt(payload, strict=opts.get("strict", False), desc_cap=cap), len(payload["items"]))

369 try:

382 try:

370− out = run_fm(prompt, schema, opts["model"], opts["timeout_s"])

383+ out = run_fm(prompt, schema, opts)

371 except Fail as f:

384 except Fail as f:

372 raise _fit_overflow(f, len(payload["items"]))

385 raise _fit_overflow(f, len(payload["items"]))

373 try:

386 try:

@@ -383,7 +396,7 @@ def op_organize(payload, opts):

383 raise Fail("SCHEMA_MISSING", f"fm could not write {os.path.join(SCHEMAS, 'topics.json')}")

396 raise Fail("SCHEMA_MISSING", f"fm could not write {os.path.join(SCHEMAS, 'topics.json')}")

384 prompt = fit(lambda cap: prompts.build_topics_prompt(payload, strict=opts.get("strict", False), desc_cap=cap), len(payload["items"]))

397 prompt = fit(lambda cap: prompts.build_topics_prompt(payload, strict=opts.get("strict", False), desc_cap=cap), len(payload["items"]))

385 try:

398 try:

386− out = run_fm(prompt, schema, opts["model"], opts["timeout_s"])

399+ out = run_fm(prompt, schema, opts)

387 except Fail as f:

400 except Fail as f:

388 raise _fit_overflow(f, len(payload["items"]))

401 raise _fit_overflow(f, len(payload["items"]))

389 _, owner = prompts.topic_items(payload)

402 _, owner = prompts.topic_items(payload)

@@ -393,29 +406,31 @@ def op_organize(payload, opts):

393 raise Fail("BAD_MODEL_OUTPUT", str(e), retryable=True, raw=json.dumps(out, ensure_ascii=False)[:2000])

406 raise Fail("BAD_MODEL_OUTPUT", str(e), retryable=True, raw=json.dumps(out, ensure_ascii=False)[:2000])

394 chars = prompts.size(prompt)

407 chars = prompts.size(prompt)

395 if payload.get("nameGroups") is not False: # the extension's fit check only needs the topics

408 if payload.get("nameGroups") is not False: # the extension's fit check only needs the topics

396− chars += name_new_groups(result, payload, opts, time.time() + opts["timeout_s"])

409+ chars += name_new_groups(result, payload, opts)

397 return result, chars, "topics"

410 return result, chars, "topics"

398

411

399

412

400−def name_new_groups(result, payload, opts, deadline):

413+def name_new_groups(result, payload, opts):

401 """Title each new group with the name op (well under a second each on fm), so groups appear named.

414 """Title each new group with the name op (well under a second each on fm), so groups appear named.

415+ A group that fails, runs past the request's deadline, or gets a title another group in the window

402− A group that fails, or runs past the deadline, keeps "" and the extension's naming loop names it."""

416+ already has keeps "" and the extension's naming loop, which checks every sibling, names it."""

403− taken = [g.get("title") for g in payload.get("existingGroups") or [] if g.get("title")]

417+ taken = [t for t in [g.get("title") for g in payload.get("existingGroups") or []] + list(payload.get("siblingTitles") or []) if t]

404 chars = 0

418 chars = 0

405 for g in result["groups"]:

419 for g in result["groups"]:

406 if "existing" in g or g.get("title"):

420 if "existing" in g or g.get("title"):

407 continue

421 continue

408− left = deadline - time.time()

422+ if time_left(opts) < MIN_CALL_S:

409− if left < 5:

410 break

423 break

411 sub = {"items": [dict(payload["items"][i], i=n) for n, i in enumerate(g["members"])], "siblingTitles": taken[:20]}

424 sub = {"items": [dict(payload["items"][i], i=n) for n, i in enumerate(g["members"])], "siblingTitles": taken[:20]}

412 try:

425 try:

413− named, n, _ = op_name(sub, {**opts, "timeout_s": min(opts["timeout_s"], left)})

426+ named, n, _ = op_name(sub, {**opts, "strict": False})

414 except Fail:

427 except Fail:

415 continue

428 continue

429+ chars += n

430+ if any(validate.same_label(named["title"], t) for t in taken):

431+ continue

416 g["title"], g["emoji"] = named["title"], named["emoji"]

432 g["title"], g["emoji"] = named["title"], named["emoji"]

417 taken.append(named["title"])

433 taken.append(named["title"])

418− chars += n

419 return chars

434 return chars

420

435

421

436

@@ -429,8 +444,8 @@ def handle(req):

429 raw_opts = req.get("opts") or {}

444 raw_opts = req.get("opts") or {}

430 timeout_ms = raw_opts.get("timeoutMs", 45000)

445 timeout_ms = raw_opts.get("timeoutMs", 45000)

431 timeout_ms = timeout_ms if isinstance(timeout_ms, (int, float)) and not isinstance(timeout_ms, bool) else 45000

446 timeout_ms = timeout_ms if isinstance(timeout_ms, (int, float)) and not isinstance(timeout_ms, bool) else 45000

432− opts = {"model": "system", "strict": bool(raw_opts.get("strict")),

433− "timeout_s": max(5.0, min(120.0, timeout_ms / 1000))}

447+ timeout_s = max(5.0, min(120.0, timeout_ms / 1000))

448+ opts = {"model": "system", "strict": bool(raw_opts.get("strict")), "timeout_s": timeout_s, "deadline": t0 + timeout_s}

434 op = req["op"]

449 op = req["op"]

435 if op == "ping":

450 if op == "ping":

436 return {"v": 1, "id": rid, "ok": True, "result": op_ping(req["payload"], opts)}

451 return {"v": 1, "id": rid, "ok": True, "result": op_ping(req["payload"], opts)}

@@ -484,7 +499,7 @@ def install_schemas():

484 if "{%s}" % key not in args:

499 if "{%s}" % key not in args:

485 continue

500 continue

486 try:

501 try:

487− p = subprocess.run([FM, *sub_args], capture_output=True, text=True, timeout=30, env=fm_env())

502+ p = subprocess.run([FM, *sub_args], capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=30, env=fm_env())

488 except (FileNotFoundError, subprocess.TimeoutExpired) as e:

503 except (FileNotFoundError, subprocess.TimeoutExpired) as e:

489 print(f" {name}: fm failed ({e})", file=sys.stderr)

504 print(f" {name}: fm failed ({e})", file=sys.stderr)

490 return "error"

505 return "error"

@@ -496,7 +511,7 @@ def install_schemas():

496 subs["{%s}" % key] = p.stdout.strip()

511 subs["{%s}" % key] = p.stdout.strip()

497 args = [subs.get(a, a) for a in args]

512 args = [subs.get(a, a) for a in args]

498 try:

513 try:

499− p = subprocess.run([FM, *args], capture_output=True, text=True, timeout=30, env=fm_env())

514+ p = subprocess.run([FM, *args], capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=30, env=fm_env())

500 except (FileNotFoundError, subprocess.TimeoutExpired) as e:

515 except (FileNotFoundError, subprocess.TimeoutExpired) as e:

501 print(f" {name}: fm failed ({e})", file=sys.stderr)

516 print(f" {name}: fm failed ({e})", file=sys.stderr)

502 return "error"

517 return "error"

@@ -553,14 +568,16 @@ def selftest():

553

568

554 ping = op_ping({}, {})

569 ping = op_ping({}, {})

555 check("fm found", os.path.exists(FM), FM)

570 check("fm found", os.path.exists(FM), FM)

571+ if not os.path.exists(FM):

572+ return 1 # nothing else can work without it

556− if ping["licenseRequired"]:

573+ if ping.get("licenseRequired"):

557 check("fm terms accepted", False, f"run: {LICENSE_FIX}")

574 check("fm terms accepted", False, f"run: {LICENSE_FIX}")

558 else:

575 else:

559 check("model available (fm available --model system)", ping["fmAvailable"], ping["fmMessage"])

576 check("model available (fm available --model system)", ping["fmAvailable"], ping["fmMessage"])

560 for name in SCHEMA_COMMANDS:

577 for name in SCHEMA_COMMANDS:

561 path = os.path.join(SCHEMAS, name)

578 path = os.path.join(SCHEMAS, name)

562 readable = os.path.isfile(path) and os.path.getsize(path) > 0

579 readable = os.path.isfile(path) and os.path.getsize(path) > 0

563− hint = f"written on their own after {LICENSE_FIX}" if ping["licenseRequired"] else "missing: run diagonal-host --install-schemas"

580+ hint = f"written on their own after {LICENSE_FIX}" if ping.get("licenseRequired") else "missing: run diagonal-host --install-schemas"

564 check(f"schema {name}", readable, path if readable else hint)

581 check(f"schema {name}", readable, path if readable else hint)

565 check("allowed origin pinned", "<" not in ALLOWED_ORIGIN, ALLOWED_ORIGIN)

582 check("allowed origin pinned", "<" not in ALLOWED_ORIGIN, ALLOWED_ORIGIN)

566 if ping["fmAvailable"] and schema_path("name.json"):

583 if ping["fmAvailable"] and schema_path("name.json"):

@@ -670,7 +687,9 @@ def register(host_path):

670 f.write("\n".join(written) + "\n")

687 f.write("\n".join(written) + "\n")

671 except OSError:

688 except OSError:

672 pass

689 pass

690+ # One browser that could not be written (a locked folder, a stale profile) does not undo the others:

691+ # the install goes on and the ones that failed were named above.

673− return 1 if failures else 0

692+ return 0 if len(dirs) - failures > 0 else 1

674

693

675

694

676def unregister():

695def unregister():

host/tests/golden/labels.jsonnew file+102

@@ -0,0 +1,102 @@

1+[

2+ [

3+ "\"Rust async runtimes.\"",

4+ "Rust async runtimes"

5+ ],

6+ [

7+ "rust async runtimes",

8+ "Rust async runtimes"

9+ ],

10+ [

11+ "Rust tabs and docs",

12+ "Rust and docs"

13+ ],

14+ [

15+ "One two three four five six",

16+ "One two three four"

17+ ],

18+ [

19+ "Machine learning research papers reading list",

20+ "Machine learning research"

21+ ],

22+ [

23+ "iPhone camera tips",

24+ "iPhone camera tips"

25+ ],

26+ [

27+ "macOS updates",

28+ "macOS updates"

29+ ],

30+ [

31+ "Children's books",

32+ "Children's books"

33+ ],

34+ [

35+ "'Quoted' title words",

36+ "Quoted title words"

37+ ],

38+ [

39+ "Planning a trip to Japan",

40+ "Planning a trip"

41+ ],

42+ [

43+ "Recipes for",

44+ null

45+ ],

46+ [

47+ "Python教程",

48+ "Python教程"

49+ ],

50+ [

51+ "React 教程",

52+ "React 教程"

53+ ],

54+ [

55+ "日本旅行",

56+ "日本旅行"

57+ ],

58+ [

59+ "京都の旅行計画",

60+ "京都の旅行計画"

61+ ],

62+ [

63+ "日本旅行。",

64+ "日本旅行"

65+ ],

66+ [

67+ "这是一个非常非常非常非常非常长的标题名字",

68+ "这是一个非常非常非常非常非常长的"

69+ ],

70+ [

71+ "Kubernetes 集群部署与运维指南最佳实践",

72+ "Kubernetes 集群部署与运维指南"

73+ ],

74+ [

75+ "ทริปญี่ปุ่น",

76+ "ทริปญี่ปุ่น"

77+ ],

78+ [

79+ "日",

80+ null

81+ ],

82+ [

83+ "Tabs",

84+ null

85+ ],

86+ [

87+ "Misc stuff",

88+ null

89+ ],

90+ [

91+ "",

92+ null

93+ ],

94+ [

95+ "x",

96+ null

97+ ],

98+ [

99+ "Supercalifragilisticexpialidocious extraordinarily",

100+ null

101+ ]

102+]

host/tests/test_host.py+56

@@ -287,6 +287,19 @@ class ErrorCodes(HostCase):

287 self.assertIn("Reply with only the object", argv[argv.index("-i") + 1])

288

289

290+ def test_strict_retry_samples_instead_of_repeating_the_greedy_answer(self):

291+ self.respond(json.dumps({"title": "Rust async", "emoji": "🦀"}))

292+ self.call("name", {"items": ITEMS3}, strict=True)

293+ self.assertNotIn("--greedy", [a for a in self.argv_log() if a[0] == "respond"][0])

294+

295+ def test_permissive_retry_shares_the_request_deadline(self):

296+ # The first call uses up the request's time: no second call runs past what the extension waits for.

297+ self.control({"respond_queue": [{"stderr": "Error: The model's safety guardrails were triggered.\n", "exit": 1, "sleep": 2.5}]})

298+ r = self.call("name", {"items": ITEMS3}, timeoutMs=5000)

299+ self.assertEqual(r["error"]["code"], "GUARDRAIL")

300+ self.assertEqual(len([a for a in self.argv_log() if a[0] == "respond"]), 1)

301+

302+

290class Organize(HostCase):

303class Organize(HostCase):

291 ITEMS = ITEMS3 + [

304 ITEMS = ITEMS3 + [

292 {"title": "Hotels in Lisbon", "url": "https://booking.com/lisbon"},

305 {"title": "Hotels in Lisbon", "url": "https://booking.com/lisbon"},

@@ -331,6 +344,33 @@ class Organize(HostCase):

331 self.assertIn("Tokio", name_calls[0])

344 self.assertIn("Tokio", name_calls[0])

332 self.assertIn('"Lisbon trip"', name_calls[0]) # existing titles are off limits

345 self.assertIn('"Lisbon trip"', name_calls[0]) # existing titles are off limits

333

346

347+ def test_a_new_group_named_like_another_group_is_left_to_the_naming_loop(self):

348+ self.control({"respond_queue": [

349+ {"stdout": json.dumps({"tabs": [{"index": i, "topic": "Programming"} for i in (0, 1, 2)]})},

350+ {"stdout": json.dumps({"title": "Rust async", "emoji": "🦀"})},

351+ ]})

352+ r = self.call("organize", {"items": self.ITEMS, "siblingTitles": ["rust async"]})

353+ self.assertEqual(r["result"]["groups"][0]["title"], "")

354+ self.assertIn('"rust async"', self.stdin_log()[1]) # the name call was told, too

355+

356+ def test_a_topic_two_groups_share_equally_joins_neither_unless_the_words_say_which(self):

357+ existing = [{"g": 0, "title": "Lisbon trip", "samples": ["Flights to Lisbon"]},

358+ {"g": 1, "title": "Porto trip", "samples": ["Porto restaurants"]}]

359+ # Examples 8 (group 0) and 9 (group 1) both got Travel: a tie.

360+ self.topics([(3, "Travel"), (4, "Travel"), (8, "Travel"), (9, "Travel")])

361+ r = self.call("organize", {"items": self.ITEMS, "existingGroups": existing})

362+ self.assertEqual([g.get("existing") for g in r["result"]["groups"]], [0]) # "Lisbon" is in both tabs' titles

363+ items = [{"title": "Beach day", "url": "https://a.example/"}, {"title": "City walk", "url": "https://b.example/"}]

364+ self.topics([(0, "Travel"), (1, "Travel"), (2, "Travel"), (3, "Travel")])

365+ r = self.call("organize", {"items": items, "existingGroups": existing})

366+ self.assertNotIn("existing", r["result"]["groups"][0])

367+

368+ def test_only_vague_topics_leave_every_tab_loose(self):

369+ self.topics([(0, "Other"), (1, "Misc"), (2, "General")])

370+ r = self.call("organize", {"items": ITEMS3})

371+ self.assertTrue(r["ok"], r)

372+ self.assertEqual(r["result"], {"groups": [], "leftovers": [0, 1, 2]})

373+

334 def test_two_topics_joining_one_group_make_one_join(self):

374 def test_two_topics_joining_one_group_make_one_join(self):

335 # The group's two example tabs (8 and 9) got different topics; both topics still join group 0 once.

375 # The group's two example tabs (8 and 9) got different topics; both topics still join group 0 once.

336 self.topics([(3, "Travel"), (4, "Hotels"), (8, "Travel"), (9, "Hotels"), (0, "Programming")])

376 self.topics([(3, "Travel"), (4, "Hotels"), (8, "Travel"), (9, "Hotels"), (0, "Programming")])

@@ -487,6 +527,15 @@ class Register(HostCase):

487 self.assertEqual(p.returncode, 1)

527 self.assertEqual(p.returncode, 1)

488 self.assertIn("missing or not executable", p.stderr)

528 self.assertIn("missing or not executable", p.stderr)

489

529

530+ def test_one_browser_that_cannot_be_written_does_not_fail_the_rest(self):

531+ chrome = self.make_profile("Google/Chrome")

532+ brave = self.make_profile("BraveSoftware/Brave-Browser")

533+ open(os.path.join(brave, "NativeMessagingHosts"), "w").close() # a file where the folder should be

534+ p = self.run_copy("--register", self.host_copy)

535+ self.assertEqual(p.returncode, 0, p.stderr)

536+ self.assertIn("BraveSoftware/Brave-Browser: could not write", p.stderr)

537+ self.assertTrue(os.path.exists(os.path.join(chrome, "NativeMessagingHosts", "io.diagonal.host.json")))

538+

490 def test_no_browser_found(self):

539 def test_no_browser_found(self):

491 os.makedirs(self.env["DIAGONAL_APP_SUPPORT"])

540 os.makedirs(self.env["DIAGONAL_APP_SUPPORT"])

492 p = self.run_copy("--register", self.host_copy)

541 p = self.run_copy("--register", self.host_copy)

@@ -508,6 +557,13 @@ class SelfTest(HostCase):

508 self.assertEqual(code, 1)

557 self.assertEqual(code, 1)

509 self.assertIn("FAIL model available", p.stdout.decode())

558 self.assertIn("FAIL model available", p.stdout.decode())

510

559

560+ def test_selftest_without_fm_says_so_and_stops(self):

561+ self.env["DIAGONAL_FM"] = os.path.join(self.tmp, "no-such-fm")

562+ code, _, p = self.run_host(args=["--selftest"])

563+ self.assertEqual(code, 1)

564+ self.assertIn("FAIL fm found", p.stdout.decode())

565+ self.assertNotIn("Traceback", p.stderr.decode())

566+

511 def test_selftest_names_the_license_fix(self):

567 def test_selftest_names_the_license_fix(self):

512 self.control({"license": True})

568 self.control({"license": True})

513 code, _, p = self.run_host(args=["--selftest"])

569 code, _, p = self.run_host(args=["--selftest"])

host/tests/test_prompts_validate.py+7−7

@@ -1,3 +1,4 @@

1+import json

1import os

2import os

2import unittest

3import unittest

@@ -73,13 +74,12 @@ class ValidateName(unittest.TestCase):

73 with self.assertRaises(validate.ValidationError, msg=bad):

74 with self.assertRaises(validate.ValidationError, msg=bad):

74 self.v(bad)

75 self.v(bad)

76− def test_chinese_and_japanese_titles_need_no_spaces(self):

77+ def test_same_labels_as_the_extension(self):

77− self.assertEqual(self.v("日本旅行")["title"], "日本旅行")

78+ # test/url.test.ts runs these cases against src/shared/label.ts.

78− self.assertEqual(self.v("京都の旅行計画")["title"], "京都の旅行計画")

79+ with open(os.path.join(GOLDEN, "labels.json"), encoding="utf-8") as f:

79− self.assertEqual(self.v("React 教程")["title"], "React 教程")

80+ cases = json.load(f)

80− for bad in ("日", "这是一个非常非常非常非常非常长的标题名字"):

81+ for raw, fixed in cases:

81− with self.assertRaises(validate.ValidationError, msg=bad):

82+ self.assertEqual(validate.repair_label(raw), fixed, msg=raw)

82− self.v(bad)

84 def test_emoji_default(self):

85 self.assertEqual(self.v("Rust async", "not-an-emoji")["emoji"], "🧭")

host/validate.py+92−14

@@ -7,7 +7,6 @@ from urllib.parse import urlsplit

7HERE = os.path.dirname(os.path.realpath(__file__))

8COLORS = ["grey", "blue", "red", "yellow", "green", "pink", "purple", "cyan", "orange"]

9BANNED_WORDS = {"tab", "tabs", "group", "groups", "misc", "various", "stuff"}

10−QUOTES = "\"'“”‘’«»`"

11TRAILING_PUNCT = re.compile(r"[\s.,;:!?…\-–—。、，！？：；]+$")

10TRAILING_PUNCT = re.compile(r"[\s.,;:!?…\-–—。、，！？：；]+$")

@@ -89,39 +88,82 @@ def default_emoji(items):

89 return CATEGORIES[category_for_host(majority)][0]

88 return CATEGORIES[category_for_host(majority)][0]

91+# Quotes go; an apostrophe between two letters ("Children's", "Rock'n'roll") is part of the word and stays.

92+_QUOTE = re.compile(r"(?<![^\W\d_])['’]|['’](?![^\W\d_])|[\"“”‘«»`]")

93+# A label cut short should not end mid-phrase: "Planning a trip to" reads as "Planning a trip".

94+DANGLING = {"a", "an", "the", "to", "of", "for", "in", "on", "at", "by", "and", "or", "with", "from"}

95+

96+

92def repair_label(raw):

97def repair_label(raw):

93− """Trim, drop quotes, banned words and trailing punctuation, cap at 4 words / 28 chars, capitalise."""

98+ """Trim, drop quotes, banned words and trailing punctuation, cap at 4 words / 28 chars (16 wide characters

99+ for no-space scripts), drop a dangling "to"/"of", capitalise. Kept in step with src/shared/label.ts."""

94 if not isinstance(raw, str):

100 if not isinstance(raw, str):

95 return None

101 return None

96− s = raw.translate({ord(q): None for q in QUOTES})

102+ s = _QUOTE.sub("", raw)

97 s = TRAILING_PUNCT.sub("", " ".join(s.split()))

103 s = TRAILING_PUNCT.sub("", " ".join(s.split()))

98 words = [w for w in s.split(" ") if w and re.sub(r"[^\w]", "", w.lower()) not in BANNED_WORDS]

104 words = [w for w in s.split(" ") if w and re.sub(r"[^\w]", "", w.lower()) not in BANNED_WORDS]

99 words = words[:4]

105 words = words[:4]

106+ if _no_space(" ".join(words)):

107+ while len(words) > 1 and _width(" ".join(words)) > 16:

108+ words.pop()

109+ if _width(" ".join(words)) > 16:

110+ words = [_cut_to_width(" ".join(words), 16)]

111+ else:

100− while len(words) > 2 and len(" ".join(words)) > 28:

112+ while len(words) > 2 and len(" ".join(words)) > 28:

113+ words.pop()

114+ while len(words) > 1 and words[-1].lower() in DANGLING:

101 words.pop()

115 words.pop()

102 label = TRAILING_PUNCT.sub("", " ".join(words))

116 label = TRAILING_PUNCT.sub("", " ".join(words))

103 if not label:

117 if not label:

104 return None

118 return None

119+ # "rust async" becomes "Rust async"; "iPhone tips" and "macOS updates" keep the brand's own casing.

120+ if not any(c.isupper() for c in words[0][1:]):

105− label = label[0].upper() + label[1:]

121+ label = label[0].upper() + label[1:]

106 return label if is_valid_label(label) else None

122 return label if is_valid_label(label) else None

107

123

108

124

109−# Chinese and Japanese titles have no spaces between words, so they're checked by length instead.

125+# Chinese, Japanese, Korean, Thai, Lao, Khmer and Myanmar titles don't put spaces between words, so a label

126+# with any of their characters ("Python教程") is checked by width instead of word count. Each such character

127+# is about as wide as two Latin letters on the tab strip.

128+NO_SPACE = re.compile(r"[\u0e00-\u0eff\u1000-\u109f\u1780-\u17ff\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uac00-\ud7af\uf900-\ufaff]")

129+# Chinese, Japanese and Korean, for the host's token estimate.

110CJK = re.compile(r"[\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uac00-\ud7af\uf900-\ufaff]")

130CJK = re.compile(r"[\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uac00-\ud7af\uf900-\ufaff]")

111

131

112

132

133+def _no_space(label):

134+ return bool(NO_SPACE.search(label))

135+

136+

137+def _width(label):

138+ wide = len(NO_SPACE.findall(label))

139+ return wide + (len(label) - wide) / 2

140+

141+

142+def _cut_to_width(label, most):

143+ out = ""

144+ for ch in label:

145+ if _width(out + ch) > most:

146+ break

147+ out += ch

148+ return out.strip()

149+

150+

113def is_valid_label(label):

151def is_valid_label(label):

114 words = label.split()

152 words = label.split()

115− cjk = len(CJK.findall(label)) * 2 > len(label.replace(" ", ""))

153+ wide = _no_space(label)

116 return (

154 return (

117− (2 <= len(label) <= 16 if cjk else 3 <= len(label) <= 28)

155+ (2 <= len(label) <= 28 and _width(label) <= 16 if wide else 3 <= len(label) <= 28)

118− and (1 <= len(words) <= 4 if cjk else 2 <= len(words) <= 4)

156+ and (1 <= len(words) <= 4 if wide else 2 <= len(words) <= 4)

119− and not any(q in label for q in QUOTES)

157+ and not _QUOTE.search(label)

120 and not TRAILING_PUNCT.search(label)

158 and not TRAILING_PUNCT.search(label)

121 and not any(w.lower() in BANNED_WORDS for w in words)

159 and not any(w.lower() in BANNED_WORDS for w in words)

122 )

160 )

123

161

124

162

163+def same_label(a, b):

164+ return bool(a) and bool(b) and " ".join(a.split()).casefold() == " ".join(b.split()).casefold()

165+

166+

125def validate_name(out, payload):

167def validate_name(out, payload):

126 if not isinstance(out, dict):

168 if not isinstance(out, dict):

127 raise ValidationError("reply is not an object")

169 raise ValidationError("reply is not an object")

@@ -227,16 +269,52 @@ def organize_from_topics(out, payload, owner):

227 votes = group_votes.setdefault(topic, {})

269 votes = group_votes.setdefault(topic, {})

228 votes[owner[i]] = votes.get(owner[i], 0) + 1

270 votes[owner[i]] = votes.get(owner[i], 0) + 1

229 if not tab_topic:

271 if not tab_topic:

272+ if not any(isinstance(t, dict) and _in_range(t.get("index"), n_items) for t in out["tabs"]):

230− raise ValidationError("no tab got a topic")

273+ raise ValidationError("no tab got a topic")

274+ # Every tab got only a vague topic ("Other", "Misc"): the model found nothing in common.

275+ return validate_organize({"groups": [], "leftovers": []}, payload)

231 clusters = {}

276 clusters = {}

232 for i in sorted(tab_topic):

277 for i in sorted(tab_topic):

233 clusters.setdefault(tab_topic[i], []).append(i)

278 clusters.setdefault(tab_topic[i], []).append(i)

234 groups = []

279 groups = []

235 for topic, members in clusters.items():

280 for topic, members in clusters.items():

236− votes = group_votes.get(topic)

281+ g = _winner(group_votes.get(topic) or {}, members, payload)

237− if votes:

282+ if g is not None:

238− g = max(sorted(votes), key=lambda k: votes[k])

239 groups.append({"existing": g, "members": members})

283 groups.append({"existing": g, "members": members})

240 else:

284 else:

241 groups.append({"title": "", "emoji": "", "color": "", "members": members})

285 groups.append({"title": "", "emoji": "", "color": "", "members": members})

242 return validate_organize({"groups": groups, "leftovers": []}, payload)

286 return validate_organize({"groups": groups, "leftovers": []}, payload)

287+

288+

289+def _in_range(i, n):

290+ return isinstance(i, int) and not isinstance(i, bool) and 0 <= i < n

291+

292+

293+_WORD = re.compile(r"\w{3,}")

294+

295+

296+def _words(text):

297+ return {w.casefold() for w in _WORD.findall(text or "")}

298+

299+

300+def _winner(votes, members, payload):

301+ """The existing group a topic's tabs join: the one whose example tabs share it most. When two groups tie,

302+ the one whose title and examples share more words with these tabs; when that ties too, none (the tabs

303+ form a group of their own, or stay loose), rather than whichever group happens to be listed first."""

304+ if not votes:

305+ return None

306+ top = max(votes.values())

307+ tied = sorted(k for k, v in votes.items() if v == top)

308+ if len(tied) == 1:

309+ return tied[0]

310+ tab_words = set().union(*(_words(payload["items"][i].get("title")) for i in members))

311+ groups = {g["g"]: g for g in payload.get("existingGroups") or []}

312+

313+ def overlap(k):

314+ g = groups.get(k) or {}

315+ return len(tab_words & _words(" ".join([g.get("title") or "", *(g.get("samples") or [])])))

316+

317+ scores = {k: overlap(k) for k in tied}

318+ best = max(scores.values())

319+ leaders = [k for k in tied if scores[k] == best]

320+ return leaders[0] if len(leaders) == 1 and best > 0 else None

scripts/install-manifest.sh+11−2

@@ -53,7 +53,16 @@ echo "host: $BIN -> $SHARE/diagonal-host.py"

53"$BIN" --version >/dev/null # the host must start with the python3 it was pinned to

54# Writes the manifest into every Chromium browser's NativeMessagingHosts folder (Chrome, Brave and

55# Brave Origin, Edge, Vivaldi, Arc, Opera, …) and checks each one the way the browser reads it.

56+# A browser that was never opened has no folder yet: the rest of the install still runs, and the end

57+# says how to finish.

58+register_rc=0

56−"$BIN" --register "$BIN"

59+"$BIN" --register "$BIN" || register_rc=$?

60+register_note() {

61+ if [[ $register_rc -ne 0 ]]; then

62+ echo "No browser could be set up for Diagonal yet. Open your browser once, then run: bash '$HERE/$(basename "$0")'"

63+ fi

64+}

65+trap register_note EXIT

58FM="${DIAGONAL_FM:-/usr/bin/fm}"

67FM="${DIAGONAL_FM:-/usr/bin/fm}"

@@ -89,7 +98,7 @@ fi

89# Diagonal also writes these on first use; doing it now lets the self-test below cover the model.

98# Diagonal also writes these on first use; doing it now lets the self-test below cover the model.

90schemas_out=$("$BIN" --install-schemas 2>&1) || true

99schemas_out=$("$BIN" --install-schemas 2>&1) || true

91if selftest_out=$("$BIN" --selftest 2>&1); then

100if selftest_out=$("$BIN" --selftest 2>&1); then

92− echo "Diagonal's host is ready."

101+ [[ $register_rc -ne 0 ]] || echo "Diagonal's host is ready."

93else

102else

94 echo "Diagonal's host is installed, but its self-test found a problem:"

103 echo "Diagonal's host is installed, but its self-test found a problem:"

95 echo "$schemas_out"

104 echo "$schemas_out"

src/background/engine.ts+3−1

@@ -1,6 +1,7 @@

1import type { GroupColor } from "../shared/colors";

2import { isColor } from "../shared/colors";

3import { colorFor, isInternalUrl, parse, provisionalTitle, trimText } from "../shared/url";

4+import { titleKey } from "../shared/label";

4import type { Settings } from "./settings";

5import type { Settings } from "./settings";

5import { markDirty, newGroupRecord, type GroupOrigin, type GroupRecord, type RemovedGroup, type State, type TabRecord } from "./state";

6import { markDirty, newGroupRecord, type GroupOrigin, type GroupRecord, type RemovedGroup, type State, type TabRecord } from "./state";

@@ -113,7 +114,7 @@ const handlers: Handlers = {

113 else loose(rec, ctx, out);

114 else loose(rec, ctx, out);

114 return;

115 return;

115 }

116 }

116− const titleChanged = before.title !== rec.title;

117+ const titleChanged = titleKey(before.title) !== titleKey(rec.title); // not an unread count ticking over

117 const pathChanged = pathKey(before.url) !== pathKey(rec.url);

118 const pathChanged = pathKey(before.url) !== pathKey(rec.url);

118 if (pathChanged) rec.keepLoose = undefined; // a new page is fair game again

119 if (pathChanged) rec.keepLoose = undefined; // a new page is fair game again

119 if (before.groupId !== rec.groupId) {

120 if (before.groupId !== rec.groupId) {

@@ -423,6 +424,7 @@ function upsertTab(s: State, tab: TabSnapshot, now: number): TabRecord {

423 if (before?.parkedFrom) rec.parkedFrom = before.parkedFrom;

424 if (before?.parkedFrom) rec.parkedFrom = before.parkedFrom;

424 if (before?.parkedAt) rec.parkedAt = before.parkedAt;

425 if (before?.parkedAt) rec.parkedAt = before.parkedAt;

425 if (before?.organizedKey) rec.organizedKey = before.organizedKey;

426 if (before?.organizedKey) rec.organizedKey = before.organizedKey;

427+ if (before?.organizeRefused) rec.organizeRefused = before.organizeRefused;

426 if (before?.handPlaced) rec.handPlaced = true;

428 if (before?.handPlaced) rec.handPlaced = true;

427 if (before?.fitPending) rec.fitPending = true;

429 if (before?.fitPending) rec.fitPending = true;

428 s.tabs[tab.id] = rec;

430 s.tabs[tab.id] = rec;

src/background/host.ts+3

@@ -102,6 +102,9 @@ export function normalizeReply<T>(reply: unknown, at: number): HostReply<T> {

102 };

103}

104

105+/** Errors about what one request carried, not about the host: they don't count toward pausing it. */

106+export const REQUEST_ERRORS = new Set<HostErrorCode>(["GUARDRAIL", "BAD_MODEL_OUTPUT", "OVER_BUDGET", "BAD_REQUEST"]);

107+

105/** Errors that mean "fix the setup", not "try this group again later". */

108/** Errors that mean "fix the setup", not "try this group again later". */

106export const SETUP_ERRORS = new Set<HostErrorCode>([

109export const SETUP_ERRORS = new Set<HostErrorCode>([

107 "HOST_NOT_FOUND", "HOST_FORBIDDEN", "FORBIDDEN_ORIGIN", "SCHEMA_MISSING", "MODEL_UNAVAILABLE", "LICENSE_REQUIRED",

110 "HOST_NOT_FOUND", "HOST_FORBIDDEN", "FORBIDDEN_ORIGIN", "SCHEMA_MISSING", "MODEL_UNAVAILABLE", "LICENSE_REQUIRED",

src/background/index.ts+27−10

@@ -1,6 +1,6 @@

1import { stripTitle } from "../shared/label";

2import { applyEvent, bury, type Action, type EngineEvent, type GroupSnapshot, type TabSnapshot } from "./engine";

3−import { callHost, chromeSender, explain, HOST_MANIFEST_PATH, SETUP_ERRORS, type HostOpts, type HostReply, type Op } from "./host";

3+import { callHost, chromeSender, explain, HOST_MANIFEST_PATH, REQUEST_ERRORS, SETUP_ERRORS, type HostOpts, type HostReply, type Op } from "./host";

4import { GLOBAL_PAUSE_AFTER, GLOBAL_PAUSE_MS, Naming, RATE_LIMIT_PAUSES_MS, type Member, type NamePayload, type NameResult } from "./naming";

5import { AutoOrganizer, organizeWindow, undoOrganize, UNDO_WINDOW_MS as ORGANIZE_UNDO_MS } from "./organize";

6import { FitChecker } from "./fit";

@@ -93,7 +93,11 @@ async function host<T>(op: Op, payload: object, extra: Partial<HostOpts> = {}):

93 h.lastOkAt = Date.now();

94 h.consecutiveFailures = 0;

95 h.consecutiveTimeouts = 0;

96− h.rateLimitStep = 0;

96+ if (op !== "ping") {

97+ h.rateLimitStep = 0; // a ping doesn't use the model, so it says nothing about its limit

98+ // A model answer (a manual Organize or Name now during a pause) proves the helper works again.

99+ if (h.pausedUntil) recovered();

100+ }

97 if (reply.meta?.ms !== undefined) void pushRing("latency", reply.meta.ms, 100);

101 if (reply.meta?.ms !== undefined) void pushRing("latency", reply.meta.ms, 100);

98 } else {

102 } else {

99 recordHostError(reply.error);

103 recordHostError(reply.error);

@@ -107,6 +111,8 @@ async function host<T>(op: Op, payload: object, extra: Partial<HostOpts> = {}):

107}

111}

108

112

109function recordHostError(e: HostError): void {

113function recordHostError(e: HostError): void {

114+ // About one request's tabs, not the host's health: the caller retries or sets those tabs aside.

115+ if (REQUEST_ERRORS.has(e.code)) return;

110 const h = state.host;

116 const h = state.host;

111 const now = Date.now();

117 const now = Date.now();

112 h.lastError = { ...e, at: now };

118 h.lastError = { ...e, at: now };

@@ -138,11 +144,7 @@ async function ping(): Promise<HostReply<PingResult>> {

138 } else if (!reply.result.schemasOk) {

144 } else if (!reply.result.schemasOk) {

139 recordHostError({ code: "SCHEMA_MISSING", message: reply.result.schemaMessage || "fm could not write Diagonal's schema files" });

145 recordHostError({ code: "SCHEMA_MISSING", message: reply.result.schemaMessage || "fm could not write Diagonal's schema files" });

140 } else {

146 } else {

141− h.pausedUntil = undefined;

147+ recovered();

142− h.lastError = undefined;

143− chrome.alarms.clear(ALARM_HOST);

144− void naming.sweepDirty();

145− void autoOrganizer.sweep();

146 }

148 }

147 }

149 }

148 commit();

150 commit();

@@ -150,6 +152,16 @@ async function ping(): Promise<HostReply<PingResult>> {

150 return reply;

152 return reply;

151}

153}

152

154

155+/** The helper answers again: lift the pause and pick up what waited for it. */

156+function recovered(): void {

157+ const h = state.host;

158+ h.pausedUntil = undefined;

159+ h.lastError = undefined;

160+ chrome.alarms.clear(ALARM_HOST);

161+ void naming.sweepDirty();

162+ void autoOrganizer.sweep();

163+}

164+

153// ----- badge -----------------------------------------------------------------------------------

165// ----- badge -----------------------------------------------------------------------------------

154

166

155function busy(on: boolean): void {

167function busy(on: boolean): void {

@@ -161,7 +173,6 @@ function hostUnhealthy(): boolean {

161 const h = state.host;

173 const h = state.host;

162 const now = Date.now();

174 const now = Date.now();

163 if (h.pausedUntil && h.pausedUntil > now) return true;

175 if (h.pausedUntil && h.pausedUntil > now) return true;

164− if (h.consecutiveFailures >= 3) return true;

165 return !!h.lastError && SETUP_ERRORS.has(h.lastError.code) && (!h.lastOkAt || h.lastOkAt < (h.lastError.at ?? 0));

176 return !!h.lastError && SETUP_ERRORS.has(h.lastError.code) && (!h.lastOkAt || h.lastOkAt < (h.lastError.at ?? 0));

166}

177}

167

178

@@ -381,6 +392,7 @@ async function reconcile(reason: "startup" | "installed"): Promise<void> {

381 parkedAt: prev?.parkedAt,

392 parkedAt: prev?.parkedAt,

382 keepLoose: prev?.keepLoose,

393 keepLoose: prev?.keepLoose,

383 organizedKey: prev?.organizedKey,

394 organizedKey: prev?.organizedKey,

395+ organizeRefused: prev?.organizeRefused,

384 handPlaced: prev?.handPlaced,

396 handPlaced: prev?.handPlaced,

385 fitPending: prev?.fitPending,

397 fitPending: prev?.fitPending,

386 createdAt: prev?.createdAt ?? now,

398 createdAt: prev?.createdAt ?? now,

@@ -500,12 +512,14 @@ if (GROUPS_SUPPORTED) {

500

512

501 chrome.tabs.onReplaced.addListener((added, removed) =>

513 chrome.tabs.onReplaced.addListener((added, removed) =>

502 void serial("tabs.onReplaced", async () => {

514 void serial("tabs.onReplaced", async () => {

515+ // Chromium swapped in a new tab (a prerendered page, a discarded tab coming back): it is the same

516+ // tab to the user, so it keeps what Diagonal knew about it, including choices the user made.

503 const rec = state.tabs[removed];

517 const rec = state.tabs[removed];

504 delete state.tabs[removed];

518 delete state.tabs[removed];

505 const t = await tabById(added);

519 const t = await tabById(added);

506 if (t) {

520 if (t) {

521+ if (rec && !state.tabs[added]) state.tabs[added] = { ...rec, id: added };

507 await feed({ type: "tabUpdated", tab: snapTab(t) });

522 await feed({ type: "tabUpdated", tab: snapTab(t) });

508− if (rec?.description && state.tabs[added]) state.tabs[added].description = rec.description;

509 }

523 }

510 }),

524 }),

511 );

525 );

@@ -600,9 +614,12 @@ async function rewriteTitles(before: Settings): Promise<void> {

600async function nameGroupNow(groupId: number): Promise<void> {

614async function nameGroupNow(groupId: number): Promise<void> {

601 const g = state.groups[groupId];

615 const g = state.groups[groupId];

602 if (!g) return;

616 if (!g) return;

603− // An explicit request overrides "don't name" and a hand-typed title for this group.

617+ // An explicit request overrides "don't name" and a hand-typed title for this group, and the model's

618+ // answer goes on the strip even when it is close to the label it gave before.

604 g.managed = true;

619 g.managed = true;

605 g.userNamed = false;

620 g.userNamed = false;

621+ g.title = undefined;

622+ g.lastNamedAt = undefined;

606 commit();

623 commit();

607 await naming.nameNow(groupId);

624 await naming.nameNow(groupId);

608}

625}

src/background/naming.ts+77−25

@@ -1,6 +1,7 @@

1−import { labelOf, repairLabel, safeEmoji, sameLabel, stripTitle, LABEL_MAX_CHARS } from "../shared/label";

1+import { labelOf, repairLabel, safeEmoji, sameLabel, stripTitle, titleKey, LABEL_MAX_CHARS } from "../shared/label";

2import { sha1 } from "../shared/sha1";

3import { isInternalUrl, promptUrl, trimText } from "../shared/url";

4+import { pathKey } from "./engine";

4import type { HostOpts, HostReply } from "./host";

5import type { HostOpts, HostReply } from "./host";

5import { SETUP_ERRORS } from "./host";

6import { SETUP_ERRORS } from "./host";

6import type { Settings } from "./settings";

7import type { Settings } from "./settings";

@@ -60,7 +61,7 @@ export const settled = (t: { status?: string }): boolean => t.status !== "loadin

61/** sha-1 of the sorted `url|title` lines: reorder-proof, changes when a member or its title does. */

62/** sha-1 of the sorted `url|title` lines: reorder-proof, changes when a member or its title does. */

62export const membersHash = (members: Pick<Member, "url" | "title">[]): string =>

63export const membersHash = (members: Pick<Member, "url" | "title">[]): string =>

63− sha1(members.map((m) => `${m.url}|${m.title}`).sort().join("\n"));

64+ sha1(members.map((m) => `${m.url}|${titleKey(m.title)}`).sort().join("\n"));

65export const memberUrls = (members: Pick<Member, "url">[]): string[] => members.map((m) => m.url).sort();

66export const memberUrls = (members: Pick<Member, "url">[]): string[] => members.map((m) => m.url).sort();

@@ -113,8 +114,9 @@ export function shouldApply(g: GroupRecord, label: string, urls: string[]): bool

113 if (g.userNamed) return false;

114 if (g.userNamed) return false;

114 if (!g.lastNamedAt || !g.title) return true; // first name replaces the provisional hostname

115 if (!g.lastNamedAt || !g.title) return true; // first name replaces the provisional hostname

115 if (sameLabel(label, g.title)) return false;

116 if (sameLabel(label, g.title)) return false;

117+ // By page, not by exact address: a tab that moved to ?page=2 or #comments is the same member.

116− const before = new Set(g.memberUrls ?? []);

118+ const before = new Set((g.memberUrls ?? []).map(pathKey));

117− const after = new Set(urls);

119+ const after = new Set(urls.map(pathKey));

118 const changed = [...after].some((u) => !before.has(u)) || [...before].some((u) => !after.has(u));

120 const changed = [...after].some((u) => !before.has(u)) || [...before].some((u) => !after.has(u));

119 return changed; // a paraphrase with the same members keeps the old title

121 return changed; // a paraphrase with the same members keeps the old title

120}

122}

@@ -214,18 +216,32 @@ export class Naming {

214 }

216 }

215 const now = this.d.now();

217 const now = this.d.now();

216 if (!force) {

218 if (!force) {

219+ // A refusal waits for the members to change, which only a look at them can tell.

217− if (g.nextAttemptAt && g.nextAttemptAt > now) return;

220+ if (g.nextAttemptAt && g.nextAttemptAt > now && !g.refusedHash) return;

218 if (s.host.pausedUntil && s.host.pausedUntil > now) return;

221 if (s.host.pausedUntil && s.host.pausedUntil > now) return;

219 if (s.inFlight?.groupId === groupId && this.current !== groupId && now - s.inFlight.startedAt < settings.timeoutMs + 5000) return;

222 if (s.inFlight?.groupId === groupId && this.current !== groupId && now - s.inFlight.startedAt < settings.timeoutMs + 5000) return;

220 }

223 }

221 const members = (await this.d.liveMembers(groupId))?.filter(nameable);

224 const members = (await this.d.liveMembers(groupId))?.filter(nameable);

222 if (!members) return;

225 if (!members) return;

223− if (members.length < 2 || !members.every(settled)) return; // stay dirty; the load finishing touches it again

226+ if (!members.every(settled)) return; // stay dirty; the load finishing touches it again

224− if (!force && membersHash(members) === g.membersHash) {

227+ if (members.length < 2) {

228+ // Nothing to name from yet. A tab joining or loading a page marks the group dirty again.

229+ if (g.dirty && !force) {

225− g.dirty = false;

230+ g.dirty = false;

231+ this.d.commit();

232+ }

233+ return;

234+ }

235+ const hash = membersHash(members);

236+ if (!force && hash === g.membersHash) {

237+ settle(g);

226 this.d.commit();

238 this.d.commit();

227 return;

239 return;

228 }

240 }

241+ if (g.refusedHash && !force) {

242+ if (hash === g.refusedHash && g.nextAttemptAt && g.nextAttemptAt > now) return;

243+ clearFailures(g); // different tabs now: Apple's filter may well take them

244+ }

229 if (this.current === groupId && !force) return; // re-queued when the in-flight reply lands

245 if (this.current === groupId && !force) return; // re-queued when the in-flight reply lands

230 if (!this.queue.some((q) => q.groupId === groupId)) this.queue.push({ groupId, force });

246 if (!this.queue.some((q) => q.groupId === groupId)) this.queue.push({ groupId, force });

231 void this.pump();

247 void this.pump();

@@ -261,7 +277,7 @@ export class Naming {

261 if (!members || members.length < 2 || !members.every(settled)) return;

277 if (!members || members.length < 2 || !members.every(settled)) return;

262 const hash = membersHash(members);

278 const hash = membersHash(members);

263 if (!force && hash === g.membersHash) {

279 if (!force && hash === g.membersHash) {

264− g.dirty = false;

280+ settle(g);

265 this.d.commit();

281 this.d.commit();

266 return;

282 return;

267 }

283 }

@@ -296,9 +312,10 @@ export class Naming {

296 }

312 }

297 if (e.code === "BAD_MODEL_OUTPUT" && !retried.output) {

313 if (e.code === "BAD_MODEL_OUTPUT" && !retried.output) {

298 retried.output = true;

314 retried.output = true;

315+ mustDifferFrom = withRejected(mustDifferFrom, e.raw);

299 continue;

316 continue;

300 }

317 }

301− this.fail(g, e);

318+ this.fail(g, e, hash);

302 return;

319 return;

303 }

320 }

304

321

@@ -307,9 +324,10 @@ export class Naming {

307 const e: HostError = { code: "BAD_MODEL_OUTPUT", message: "label failed worker validation", raw: JSON.stringify(reply.result).slice(0, 500), at: this.d.now() };

324 const e: HostError = { code: "BAD_MODEL_OUTPUT", message: "label failed worker validation", raw: JSON.stringify(reply.result).slice(0, 500), at: this.d.now() };

308 if (!retried.output) {

325 if (!retried.output) {

309 retried.output = true;

326 retried.output = true;

327+ mustDifferFrom = withRejected(mustDifferFrom, e.raw);

310 continue;

328 continue;

311 }

329 }

312− this.fail(g, e);

330+ this.fail(g, e, hash);

313 return;

331 return;

314 }

332 }

315 const emoji = safeEmoji(reply.result.emoji);

333 const emoji = safeEmoji(reply.result.emoji);

@@ -324,12 +342,15 @@ export class Naming {

324 finalLabel = suffixLabel(label, siblings);

342 finalLabel = suffixLabel(label, siblings);

325 }

343 }

326 this.d.hostOk();

344 this.d.hostOk();

327− await this.apply(g, finalLabel, emoji, members, hash);

345+ const written = await this.apply(g, finalLabel, emoji, members, hash);

328 g = this.d.state().groups[groupId];

346 g = this.d.state().groups[groupId];

329− if (g) {

347+ if (!g) return;

348+ if (!written) {

349+ // The strip refused the title (a drag in progress, the window closing): try again shortly.

350+ g.nextAttemptAt = this.d.now() + BACKOFF_MS[0];

351+ this.d.scheduleFallback(g.nextAttemptAt);

330− if (g.dirtySeq === seq) g.dirty = false;

352+ } else if (g.dirtySeq === seq) g.dirty = false;

331− else this.touch(groupId); // dirtied while in flight: go again with the new membership

353+ else this.touch(groupId); // dirtied while in flight: go again with the new membership

332− }

333 return;

354 return;

334 }

355 }

335 } finally {

356 } finally {

@@ -340,16 +361,20 @@ export class Naming {

340 }

361 }

341 }

362 }

342

363

364+ /** Put the label on the strip if the policy allows; false when the strip would not take the title. */

343− private async apply(g: GroupRecord, label: string, emoji: string, members: Member[], hash: string): Promise<void> {

365+ private async apply(g: GroupRecord, label: string, emoji: string, members: Member[], hash: string): Promise<boolean> {

344 const settings = this.d.settings();

366 const settings = this.d.settings();

345 const urls = memberUrls(members);

367 const urls = memberUrls(members);

346− if (g.userNamed) return; // renamed by hand while the model was thinking

368+ if (g.userNamed) return true; // renamed by hand while the model was thinking

347 if (shouldApply(g, label, urls)) {

369 if (shouldApply(g, label, urls)) {

348 const title = stripTitle(label, emoji, settings.emoji);

370 const title = stripTitle(label, emoji, settings.emoji);

349 this.d.state().ownWrites[g.id] = { title, at: this.d.now() };

371 this.d.state().ownWrites[g.id] = { title, at: this.d.now() };

350 this.d.commit();

372 this.d.commit();

351 const ok = await this.d.writeTitle(g.id, title);

373 const ok = await this.d.writeTitle(g.id, title);

352− if (!ok) return;

374+ if (!ok) {

375+ delete this.d.state().ownWrites[g.id];

376+ return false;

377+ }

353 g.title = label;

378 g.title = label;

354 g.emoji = emoji;

379 g.emoji = emoji;

355 g.stripTitle = title;

380 g.stripTitle = title;

@@ -357,21 +382,24 @@ export class Naming {

357 g.lastNamedAt = this.d.now();

382 g.lastNamedAt = this.d.now();

358 g.membersHash = hash;

383 g.membersHash = hash;

359 g.memberUrls = urls;

384 g.memberUrls = urls;

360− g.nameAttempts = 0;

385+ clearFailures(g);

361− g.firstFailureAt = undefined;

386+ return true;

362− g.nextAttemptAt = undefined;

363 }

387 }

364

388

365− private fail(g: GroupRecord, e: HostError): void {

389+ private fail(g: GroupRecord, e: HostError, hash: string): void {

366 const now = this.d.now();

390 const now = this.d.now();

367 this.d.hostFailed(e);

391 this.d.hostFailed(e);

368 if (e.code === "BAD_REQUEST") {

392 if (e.code === "BAD_REQUEST") {

369 g.dirty = false; // a bug on our side: log and drop

393 g.dirty = false; // a bug on our side: log and drop

370 return;

394 return;

371 }

395 }

396+ // Setup errors pause every model call until the host is fixed, and its recovery picks this group up

397+ // again: they say nothing about this group, so they don't use up its retries.

398+ if (SETUP_ERRORS.has(e.code)) return;

372 if (e.code === "GUARDRAIL") {

399 if (e.code === "GUARDRAIL") {

373 g.nameAttempts = MAX_ATTEMPTS;

400 g.nameAttempts = MAX_ATTEMPTS;

374 g.nextAttemptAt = now + GIVE_UP_AFTER_MS;

401 g.nextAttemptAt = now + GIVE_UP_AFTER_MS;

402+ g.refusedHash = hash;

375 this.d.scheduleFallback(g.nextAttemptAt);

403 this.d.scheduleFallback(g.nextAttemptAt);

376 return;

404 return;

377 }

405 }

@@ -381,11 +409,35 @@ export class Naming {

381 g.dirty = false; // retried for a day: keep the provisional title

409 g.dirty = false; // retried for a day: keep the provisional title

382 return;

410 return;

383 }

411 }

384− const wait = SETUP_ERRORS.has(e.code) ? Math.max(GLOBAL_PAUSE_MS, backoffFor(g.nameAttempts)) : backoffFor(g.nameAttempts);

385− g.nextAttemptAt = now + wait;

412+ g.nextAttemptAt = now + backoffFor(g.nameAttempts);

386 this.d.scheduleFallback(g.nextAttemptAt);

413 this.d.scheduleFallback(g.nextAttemptAt);

387 }

414 }

388}

415}

389

416

417+/** The retry after an unusable title lists that title as not allowed, so the model doesn't give it again. */

418+function withRejected(list: string[] | undefined, raw: string | undefined): string[] | undefined {

419+ let title: unknown;

420+ try {

421+ title = raw ? JSON.parse(raw)?.title : undefined;

422+ } catch {

423+ return list;

424+ }

425+ return typeof title === "string" && title.trim() ? [...(list ?? []), title.trim().slice(0, 60)] : list;

426+}

427+

428+/** The retry bookkeeping of earlier failures, which no longer applies. */

429+function clearFailures(g: GroupRecord): void {

430+ g.nameAttempts = 0;

431+ g.firstFailureAt = undefined;

432+ g.nextAttemptAt = undefined;

433+ g.refusedHash = undefined;

434+}

435+

436+/** The members are the ones last named: nothing to do, and nothing left to retry. */

437+function settle(g: GroupRecord): void {

438+ g.dirty = false;

439+ clearFailures(g);

440+}

441+

390export const eligible = (g: GroupRecord, settings: Settings): boolean =>

442export const eligible = (g: GroupRecord, settings: Settings): boolean =>

391 settings.naming && g.managed && !g.userNamed && g.origin !== "tidy";

443 settings.naming && g.managed && !g.userNamed && g.origin !== "tidy";

src/background/organize.ts+194−107

@@ -1,10 +1,11 @@

1import { COLORS, isColor, type GroupColor } from "../shared/colors";

2−import { labelOf, repairLabel, safeEmoji, stripTitle } from "../shared/label";

2+import { labelOf, repairLabel, safeEmoji, stripTitle, titleKey } from "../shared/label";

3import { colorFor, isInternalUrl, promptAddress, provisionalTitle } from "../shared/url";

4import { pathKey } from "./engine";

5import { membersHash, memberUrls, settled, toItems, type Member, type NameItem } from "./naming";

6import { addToGroup, createManagedGroup, groupExists, ungroup, type Runtime } from "./runtime";

7+import type { HostReply } from "./host";

7−import { markDirty, type State } from "./state";

8+import { markDirty, type GroupRecord, type HostError, type HostErrorCode, type State, type TabRecord } from "./state";

9/** Section 3/9: "Organize all tabs" — cluster ungrouped tabs in batches sized to the model's window. */

10/** Section 3/9: "Organize all tabs" — cluster ungrouped tabs in batches sized to the model's window. */

@@ -26,6 +27,8 @@ export interface ExistingGroup {

26export interface OrganizePayload {

27export interface OrganizePayload {

27 items: NameItem[];

28 items: NameItem[];

28 existingGroups: ExistingGroup[];

29 existingGroups: ExistingGroup[];

30+ /** Every group label in the window: a new group the host names must not repeat one. */

31+ siblingTitles?: string[];

29 allowNew: boolean;

32 allowNew: boolean;

30 maxGroups: number;

33 maxGroups: number;

31 minGroupSize: number;

34 minGroupSize: number;

@@ -151,88 +154,198 @@ export interface OrganizeReport {

151 error?: string;

154 error?: string;

152}

155}

153

156

154−/** Existing groups a batch may add tabs to: extension-managed topic groups in this window. */

157+/** Existing groups a batch may add tabs to: Diagonal's topic groups in this window, the ones in use first. */

155export function existingGroupsFor(state: State, windowId: number, extraIds: number[] = []): { ids: number[]; list: ExistingGroup[] } {

158export function existingGroupsFor(state: State, windowId: number, extraIds: number[] = []): { ids: number[]; list: ExistingGroup[] } {

159+ const members = new Map<number, TabRecord[]>();

160+ for (const t of Object.values(state.tabs)) if (t.groupId !== -1) members.set(t.groupId, [...(members.get(t.groupId) ?? []), t]);

161+ const inUse = (g: GroupRecord): number =>

162+ Math.max(g.dirtyAt ?? 0, g.lastNamedAt ?? 0, ...(members.get(g.id) ?? []).map((t) => t.lastActivatedAt ?? 0));

156 const groups = Object.values(state.groups)

163 const groups = Object.values(state.groups)

157− .filter((g) => g.windowId === windowId && g.managed && g.origin !== "tidy" && (g.stripTitle || g.title))

164+ .filter((g) => g.windowId === windowId && g.managed && g.origin !== "tidy" && !g.restored && (g.stripTitle || g.title))

165+ .map((g) => ({ g, first: extraIds.includes(g.id), at: inUse(g) }))

158− .sort((a, b) => Number(extraIds.includes(b.id)) - Number(extraIds.includes(a.id)))

166+ .sort((a, b) => Number(b.first) - Number(a.first) || b.at - a.at)

159− .slice(0, EXISTING_GROUPS_CAP);

167+ .slice(0, EXISTING_GROUPS_CAP)

168+ .map(({ g }) => g);

160 const list = groups.map((g, i) => ({

169 const list = groups.map((g, i) => ({

161 g: i,

170 g: i,

162 title: labelOf(g.stripTitle || g.title),

171 title: labelOf(g.stripTitle || g.title),

163− samples: Object.values(state.tabs)

164− .filter((t) => t.groupId === g.id)

165− .slice(0, 2)

166− .map((t) => t.title.slice(0, 60)),

172+ samples: (members.get(g.id) ?? []).slice(0, 2).map((t) => t.title.slice(0, 60)),

167 }));

173 }));

168 return { ids: groups.map((g) => g.id), list };

174 return { ids: groups.map((g) => g.id), list };

169}

175}

170

176

177+/** Errors about what was sent rather than about the host: a smaller or plainer request may go through. */

178+const CONTENT_ERRORS = new Set<HostErrorCode>(["OVER_BUDGET", "GUARDRAIL", "BAD_MODEL_OUTPUT"]);

179+const MAX_TIMEOUT_SPLITS = 1;

180+

181+/** One organize call, with the retries that can rescue the same tabs: strict output, then no page text. */

171−export async function organizeWindow(rt: Runtime, windowId: number): Promise<OrganizeReport> {

182+async function askOrganize(rt: Runtime, part: Member[], windowId: number, existingGroups: ExistingGroup[], withDescriptions: boolean): Promise<HostReply<OrganizeResult>> {

183+ const settings = rt.settings();

184+ const s = rt.state();

185+ const payload: OrganizePayload = {

186+ items: toItems(part, settings, withDescriptions),

187+ existingGroups,

188+ siblingTitles: Object.values(s.groups)

189+ .filter((g) => g.windowId === windowId && g.origin !== "tidy")

190+ .map((g) => labelOf(g.stripTitle || g.title))

191+ .filter(Boolean)

192+ .slice(0, 20),

193+ allowNew: part.length >= 2,

194+ maxGroups: maxGroupsFor(part.length),

195+ minGroupSize: settings.organizeMinGroupSize,

196+ };

197+ let reply = await rt.host<OrganizeResult>("organize", payload);

198+ if (!reply.ok && reply.error.code === "BAD_MODEL_OUTPUT") reply = await rt.host<OrganizeResult>("organize", payload, { strict: true });

199+ if (!reply.ok && reply.error.code === "GUARDRAIL" && payload.items.some((i) => i.description)) {

200+ // What a page says is the likeliest thing to trip Apple's filter; its title and address rarely do.

201+ payload.items = toItems(part, settings, false);

202+ reply = await rt.host<OrganizeResult>("organize", payload);

203+ }

204+ return reply;

205+}

206+

207+/** One organize run over a window: what it made, what it moved, and whether the host stopped it. */

208+export interface SortRun {

209+ windowId: number;

210+ createdIds: number[];

211+ previous: Record<number, number>;

212+ grouped: number;

213+ left: number;

214+ timeoutSplits: number;

215+ /** Auto-organize: remember what was looked at, so the same tabs are not sent again unchanged. */

216+ stamp: boolean;

217+ stopped?: HostError;

218+}

219+

220+export const newRun = (windowId: number, stamp: boolean): SortRun => ({ windowId, createdIds: [], previous: {}, grouped: 0, left: 0, timeoutSplits: 0, stamp });

221+

222+/**

223+ * Sort one part of a window. A reply about the tabs themselves (too long, refused by Apple's filter, not

224+ * valid output) splits the part in halves, so one page the model won't handle leaves the others sorted; a

225+ * tab that fails alone is set aside until its page or title changes. Tabs that changed while the model was

226+ * thinking are left out of its answer.

227+ */

228+export async function sortPart(rt: Runtime, part: Member[], run: SortRun, probe = false): Promise<void> {

229+ if (!part.length || run.stopped) return;

230+ const { ids: existingIds, list: existingGroups } = existingGroupsFor(rt.state(), run.windowId, run.createdIds);

231+ if (part.length < 2 && !existingIds.length && !probe) {

232+ run.left += part.length;

233+ return; // one tab and nothing to join: no call can place it (a probe still learns whether the model takes it)

234+ }

235+ const reply = await askOrganize(rt, part, run.windowId, existingGroups, true);

236+ if (!reply.ok) {

237+ const e = reply.error;

238+ const timedOut = e.code === "TIMEOUT" && part.length > 4 && run.timeoutSplits < MAX_TIMEOUT_SPLITS;

239+ if (part.length > 1 && (CONTENT_ERRORS.has(e.code) || timedOut)) {

240+ if (timedOut) run.timeoutSplits++;

241+ const half = Math.ceil(part.length / 2);

242+ await sortPart(rt, part.slice(0, half), run, true);

243+ await sortPart(rt, part.slice(half), run, true);

244+ return;

245+ }

246+ if (CONTENT_ERRORS.has(e.code)) {

247+ const rec = rt.state().tabs[part[0].id];

248+ if (rec) rec.organizeRefused = organizedKeyOf(part[0]);

249+ rt.log("organize: set aside a tab the model would not sort", e.code);

250+ run.left += 1;

251+ return;

252+ }

253+ run.stopped = e;

254+ return;

255+ }

172 const settings = rt.settings();

256 const settings = rt.settings();

257+ const plan = planFromResult(reply.result, part.map((m) => m.id), existingIds, settings.organizeMinGroupSize, maxGroupsFor(part.length), (ids) =>

258+ colorFor(part.find((m) => m.id === ids[0])?.url ?? ""),

259+ );

260+ const unchanged = await stillAsSent(rt, part, run.windowId);

261+ const trimmed = trimPlan(rt.state(), plan, unchanged, settings.organizeMinGroupSize);

262+ const failed = await applyPlan(rt, trimmed, run.windowId, part, run.previous, run.createdIds);

263+ const placed = [...trimmed.joins, ...trimmed.creates].reduce((n, x) => n + x.tabIds.filter((id) => !failed.has(id)).length, 0);

264+ run.grouped += placed;

265+ run.left += part.length - placed;

266+ if (run.stamp) {

267+ const s = rt.state();

268+ for (const m of part) {

269+ const rec = s.tabs[m.id];

270+ if (rec && unchanged.has(m.id) && !failed.has(m.id)) rec.organizedKey = organizedKeyOf(m);

271+ }

272+ }

273+}

274+

275+/** The tabs of a part that are still loose, in this window, on the page and title the model saw. */

276+async function stillAsSent(rt: Runtime, part: Member[], windowId: number): Promise<Set<number>> {

277+ const live = new Map((await chrome.tabs.query({ windowId }).catch(() => [])).map((t) => [t.id!, t]));

278+ const s = rt.state();

279+ return new Set(

280+ part

281+ .filter((m) => {

282+ const t = live.get(m.id);

283+ return !!t && organizable(t) && !s.tabs[m.id]?.keepLoose && organizedKeyOf(t) === organizedKeyOf(m);

284+ })

285+ .map((m) => m.id),

286+ );

287+}

288+

289+/** A plan with the tabs that moved on taken out: a new group left too small is not made, a join into a group gone is dropped. */

290+export function trimPlan(state: State, plan: Plan, keep: Set<number>, minGroupSize: number): Plan {

291+ const out: Plan = { joins: [], creates: [], leftovers: plan.leftovers.filter((id) => keep.has(id)) };

292+ for (const j of plan.joins) {

293+ const tabIds = j.tabIds.filter((id) => keep.has(id));

294+ const g = state.groups[j.groupId];

295+ if (tabIds.length && g?.managed && !g.restored) out.joins.push({ ...j, tabIds });

296+ else out.leftovers.push(...tabIds);

297+ }

298+ for (const c of plan.creates) {

299+ const tabIds = c.tabIds.filter((id) => keep.has(id));

300+ if (tabIds.length >= Math.max(2, minGroupSize)) out.creates.push({ ...c, tabIds });

301+ else out.leftovers.push(...tabIds);

302+ }

303+ return out;

304+}

305+

306+export async function organizeWindow(rt: Runtime, windowId: number): Promise<OrganizeReport> {

307+ const s0 = rt.state();

173 const tabs = (await chrome.tabs.query({ windowId })).filter((t) => organizable(t)).sort((a, b) => a.index - b.index);

308 const tabs = (await chrome.tabs.query({ windowId })).filter((t) => organizable(t)).sort((a, b) => a.index - b.index);

174 const report: OrganizeReport = { grouped: 0, groups: 0, left: 0 };

309 const report: OrganizeReport = { grouped: 0, groups: 0, left: 0 };

310+ // A tab the model already would not sort, on the same page, is not sent again.

311+ const sendable = tabs.filter((t) => s0.tabs[t.id!]?.organizeRefused !== organizedKeyOf(t));

175− if (tabs.length < 2) {

312+ if (sendable.length < 2) {

176 report.left = tabs.length;

313 report.left = tabs.length;

177 return report;

314 return report;

178 }

315 }

179− const members: Member[] = tabs.map((t) => ({

316+ const run = newRun(windowId, true); // auto-organize need not send the same tabs again right after

180− id: t.id!,

181− index: t.index,

182− url: t.url ?? "",

183− title: t.title ?? "",

184− status: t.status,

185− description: rt.state().tabs[t.id!]?.description,

186− lastActive: t.lastAccessed ?? 0,

187− }));

188− const previous: Record<number, number> = {};

189− const createdIds: number[] = [];

190− const queue = makeBatches(members);

191 rt.state().inFlight = { groupId: -1, startedAt: rt.now() };

317 rt.state().inFlight = { groupId: -1, startedAt: rt.now() };

192 rt.refreshBadge();

318 rt.refreshBadge();

193 try {

319 try {

194− while (queue.length) {

320+ for (const batch of makeBatches(sendable.map((t) => toMember(rt.state(), t)))) {

195− const batch = queue.shift()!;

321+ await sortPart(rt, batch, run);

196− const { ids: existingIds, list: existingGroups } = existingGroupsFor(rt.state(), windowId, createdIds);

197− const maxGroups = maxGroupsFor(batch.length);

198− const payload: OrganizePayload = {

199− items: toItems(batch, settings),

200− existingGroups,

201− allowNew: true,

202− maxGroups,

203− minGroupSize: settings.organizeMinGroupSize,

204− };

205− let reply = await rt.host<OrganizeResult>("organize", payload);

206− if (!reply.ok && reply.error.code === "OVER_BUDGET" && batch.length > 1) {

207− const half = Math.ceil(batch.length / 2);

208− queue.unshift(batch.slice(0, half), batch.slice(half));

209− continue;

210− }

211− if (!reply.ok && reply.error.code === "BAD_MODEL_OUTPUT") reply = await rt.host<OrganizeResult>("organize", payload, { strict: true });

212− if (!reply.ok) {

213− report.error = `${reply.error.code}: ${reply.error.message}`;

214− report.left += batch.length + queue.flat().length;

215− break;

322+ if (run.stopped) break;

216− }

217− const batchIds = batch.map((m) => m.id);

218− const plan = planFromResult(reply.result, batchIds, existingIds, settings.organizeMinGroupSize, maxGroups, (ids) =>

219− colorFor(members.find((m) => m.id === ids[0])?.url ?? ""),

220− );

221− await applyPlan(rt, plan, windowId, members, previous, createdIds);

222− report.grouped += plan.joins.reduce((n, j) => n + j.tabIds.length, 0) + plan.creates.reduce((n, c) => n + c.tabIds.length, 0);

223− report.left += plan.leftovers.length;

224 }

323 }

225 } finally {

324 } finally {

226 const s = rt.state();

325 const s = rt.state();

227 if (s.inFlight?.groupId === -1) s.inFlight = undefined;

326 if (s.inFlight?.groupId === -1) s.inFlight = undefined;

327+ report.grouped = run.grouped;

228− report.groups = createdIds.length;

328+ report.groups = run.createdIds.length;

329+ report.left = tabs.length - run.grouped;

330+ if (run.stopped) report.error = `${run.stopped.code}: ${run.stopped.message}`;

229− if (Object.keys(previous).length) s.lastOrganize = { at: rt.now(), previous };

331+ if (Object.keys(run.previous).length) s.lastOrganize = { at: rt.now(), previous: run.previous };

230 rt.commit();

332 rt.commit();

231 rt.refreshBadge();

333 rt.refreshBadge();

232 }

334 }

233 return report;

335 return report;

234}

336}

235

337

338+const toMember = (s: State, t: chrome.tabs.Tab): Member => ({

339+ id: t.id!,

340+ index: t.index,

341+ url: t.url ?? "",

342+ title: t.title ?? "",

343+ status: t.status,

344+ description: s.tabs[t.id!]?.description,

345+ lastActive: t.lastAccessed ?? 0,

346+});

347+

348+/** Carry out a plan; returns the tabs whose move failed, which stay as they were. */

236export async function applyPlan(

349export async function applyPlan(

237 rt: Runtime,

350 rt: Runtime,

238 plan: Plan,

351 plan: Plan,

@@ -240,11 +353,15 @@ export async function applyPlan(

240 members: Member[],

353 members: Member[],

241 previous: Record<number, number>,

354 previous: Record<number, number>,

242 createdIds: number[],

355 createdIds: number[],

243−): Promise<void> {

356+): Promise<Set<number>> {

244 const settings = rt.settings();

357 const settings = rt.settings();

245 const byId = new Map(members.map((m) => [m.id, m]));

358 const byId = new Map(members.map((m) => [m.id, m]));

359+ const failed = new Set<number>();

246 for (const j of plan.joins) {

360 for (const j of plan.joins) {

247− if (!(await addToGroup(rt, j.tabIds, j.groupId))) continue;

361+ if (!(await addToGroup(rt, j.tabIds, j.groupId))) {

362+ for (const id of j.tabIds) failed.add(id);

363+ continue;

364+ }

248 for (const id of j.tabIds) previous[id] = -1;

365 for (const id of j.tabIds) previous[id] = -1;

249 const g = rt.state().groups[j.groupId];

366 const g = rt.state().groups[j.groupId];

250 if (g) {

367 if (g) {

@@ -265,7 +382,10 @@ export async function applyPlan(

265 lastNamedAt: named ? rt.now() : undefined,

382 lastNamedAt: named ? rt.now() : undefined,

266 dirty: false,

383 dirty: false,

267 });

384 });

268− if (groupId === undefined) continue;

385+ if (groupId === undefined) {

386+ for (const id of c.tabIds) failed.add(id);

387+ continue;

388+ }

269 createdIds.push(groupId);

389 createdIds.push(groupId);

270 for (const id of c.tabIds) previous[id] = -1;

390 for (const id of c.tabIds) previous[id] = -1;

271 const g = rt.state().groups[groupId];

391 const g = rt.state().groups[groupId];

@@ -274,6 +394,7 @@ export async function applyPlan(

274 rt.naming.touch(groupId);

394 rt.naming.touch(groupId);

275 }

395 }

276 }

396 }

397+ return failed;

277}

398}

278

399

279/** Undo within the hour: every tab goes back to the group it was in (organize only takes loose tabs). */

400/** Undo within the hour: every tab goes back to the group it was in (organize only takes loose tabs). */

@@ -301,7 +422,7 @@ export async function undoOrganize(rt: Runtime): Promise<number> {

301}

422}

302

423

303/** What auto-organize remembers about a tab: it looks again only when the page or title changes. */

424/** What auto-organize remembers about a tab: it looks again only when the page or title changes. */

304−export const organizedKeyOf = (t: { url?: string; title?: string }): string => `${pathKey(t.url)}\n${t.title ?? ""}`;

425+export const organizedKeyOf = (t: { url?: string; title?: string }): string => `${pathKey(t.url)}\n${titleKey(t.title)}`;

305

426

306export const AUTO_RETRY_MS = 300_000;

427export const AUTO_RETRY_MS = 300_000;

307export const AUTO_CONTINUE_MS = 1_500;

428export const AUTO_CONTINUE_MS = 1_500;

@@ -379,59 +500,25 @@ export class AutoOrganizer {

379 const s = rt.state();

500 const s = rt.state();

380 // Still-loading tabs are skipped here; finishing the load touches the window again. Discarded tabs keep

501 // Still-loading tabs are skipped here; finishing the load touches the window again. Discarded tabs keep

381 // their title and address, so they're sorted like loaded ones.

502 // their title and address, so they're sorted like loaded ones.

382− const loose = tabs.filter((t) => organizable(t) && settled(t) && !s.tabs[t.id!]?.keepLoose);

503+ const loose = tabs.filter((t) => {

504+ const rec = s.tabs[t.id!];

505+ return organizable(t) && settled(t) && !rec?.keepLoose && rec?.organizeRefused !== organizedKeyOf(t);

506+ });

383 const fresh = loose.filter((t) => s.tabs[t.id!]?.organizedKey !== organizedKeyOf(t));

507 const fresh = loose.filter((t) => s.tabs[t.id!]?.organizedKey !== organizedKeyOf(t));

384 if (!fresh.length) return;

508 if (!fresh.length) return;

385− const { ids: existingIds, list: existingGroups } = existingGroupsFor(s, windowId);

386− if (loose.length < 2 && !existingIds.length) return; // one loose tab and nothing to join: wait for company

509+ if (loose.length < 2 && !existingGroupsFor(s, windowId).ids.length) return; // one loose tab and nothing to join: wait for company

387− const toMember = (t: chrome.tabs.Tab): Member => ({

388− id: t.id!,

389− index: t.index,

390− url: t.url ?? "",

391− title: t.title ?? "",

392− status: t.status,

393− description: s.tabs[t.id!]?.description,

394− lastActive: t.lastAccessed ?? 0,

395− });

396− const ordered = [...fresh, ...loose.filter((t) => !fresh.includes(t))].map(toMember);

510+ const ordered = [...fresh, ...loose.filter((t) => !fresh.includes(t))].map((t) => toMember(s, t));

397 const batch = makeBatches(ordered)[0];

511 const batch = makeBatches(ordered)[0];

398− const maxGroups = maxGroupsFor(batch.length);

512+ const run = newRun(windowId, true);

399− const payload: OrganizePayload = {

513+ await sortPart(rt, batch, run);

400− items: toItems(batch, settings),

514+ rt.commit();

401− existingGroups,

515+ if (run.stopped) {

402− allowNew: batch.length >= 2,

403− maxGroups,

404− minGroupSize: settings.organizeMinGroupSize,

405− };

406− let reply = await rt.host<OrganizeResult>("organize", payload);

407− if (!reply.ok && reply.error.code === "OVER_BUDGET" && batch.length > 1) {

408− // Rare with the budgeted batch; try once with half, the rest follows on the next run.

409− const half = batch.slice(0, Math.ceil(batch.length / 2));

410− payload.items = toItems(half, settings);

411− payload.maxGroups = maxGroupsFor(half.length);

412− payload.allowNew = half.length >= 2;

413− batch.length = half.length;

414− reply = await rt.host<OrganizeResult>("organize", payload);

415− }

416− if (!reply.ok && reply.error.code === "BAD_MODEL_OUTPUT") reply = await rt.host<OrganizeResult>("organize", payload, { strict: true });

417− if (!reply.ok) {

418− rt.log("auto-organize", reply.error.code);

516+ rt.log("auto-organize", run.stopped.code);

419 this.deps.scheduleFallback(rt.now() + AUTO_RETRY_MS);

517 this.deps.scheduleFallback(rt.now() + AUTO_RETRY_MS);

420 return;

518 return;

421 }

519 }

422− const batchIds = batch.map((m) => m.id);

520+ const sent = new Set(batch.map((m) => m.id));

423− const plan = planFromResult(reply.result, batchIds, existingIds, settings.organizeMinGroupSize, payload.maxGroups, (ids) =>

424− colorFor(batch.find((m) => m.id === ids[0])?.url ?? ""),

425− );

426− await applyPlan(rt, plan, windowId, batch, {}, []);

427− const live = rt.state();

428− for (const m of batch) {

429− const rec = live.tabs[m.id];

430− if (rec) rec.organizedKey = organizedKeyOf(m);

431− }

432− rt.commit();

433− const remaining = fresh.filter((t) => !batchIds.includes(t.id!));

434− if (remaining.length) this.touch(windowId, AUTO_CONTINUE_MS);

521+ if (fresh.some((t) => !sent.has(t.id!))) this.touch(windowId, AUTO_CONTINUE_MS);

435 }

522 }

436}

523}

437

524

src/background/state.ts+3

@@ -21,6 +21,7 @@ export interface TabRecord {

21 parkedAt?: number; // when the tidy sweep parked it: the archive clock starts here

22 keepLoose?: string; // the user took this tab out of a group on this page: auto-organize leaves it be

23 organizedKey?: string; // page + title when auto-organize last considered this tab

24+ organizeRefused?: string; // page + title the model would not sort even alone: not sent again until it changes

24 handPlaced?: boolean; // the user put this tab into its group: Diagonal won't take it out

25 handPlaced?: boolean; // the user put this tab into its group: Diagonal won't take it out

25 fitPending?: boolean; // moved to a new page inside a Diagonal group: check it still fits (fit.ts)

26 fitPending?: boolean; // moved to a new page inside a Diagonal group: check it still fits (fit.ts)

26}

27}

@@ -46,8 +47,10 @@ export interface GroupRecord {

46 nameAttempts: number;

47 nameAttempts: number;

47 firstFailureAt?: number;

48 firstFailureAt?: number;

48 nextAttemptAt?: number;

49 nextAttemptAt?: number;

50+ refusedHash?: string; // the members Apple's filter refused to name: not asked again until they change

49 registeredAt?: number; // first seen as a group Diagonal did not know: it may yet turn out to be one of Diagonal's coming back

51 registeredAt?: number; // first seen as a group Diagonal did not know: it may yet turn out to be one of Diagonal's coming back

50 revivedAt?: number; // came back after Chromium removed it (window move, reopen, restart)

52 revivedAt?: number; // came back after Chromium removed it (window move, reopen, restart)

53+ restored?: boolean; // "Restored": the archive brought back, not a topic other tabs should join

51}

54}

53/** A group Chromium removed, kept for a day: it may come back (moved to another window, reopened, restored). */

56/** A group Chromium removed, kept for a day: it may come back (moved to another window, reopened, restored). */

src/background/tidy.ts+1−1

@@ -361,7 +361,7 @@ export async function restoreAll(rt: Runtime, windowId: number): Promise<number>

361 if (t.id !== undefined) ids.push(t.id);

362 }

363 if (ids.length) {

364− await createManagedGroup(rt, ids, windowId, "organize", { title: stripTitle("Restored", RESTORED_EMOJI, rt.settings().emoji), color: "grey" }, { userNamed: true });

364+ await createManagedGroup(rt, ids, windowId, "organize", { title: stripTitle("Restored", RESTORED_EMOJI, rt.settings().emoji), color: "grey" }, { userNamed: true, restored: true });

365 }

366 return ids.length;

367}

src/shared/health.ts+4−2

@@ -46,7 +46,9 @@ export function notices(s: Health): Notice[] {

46 case "HOST_NOT_FOUND":

47 out.push({

48 tone: "bad", title: "Finish setting up Diagonal",

49+ body: chrome.runtime.getManifest().update_url

49− body: "Diagonal's helper isn't registered with this browser yet. Run this in Terminal, then reopen Diagonal.",

50+ ? "Diagonal's helper isn't registered with this browser yet. Run this in Terminal, then reopen Diagonal."

51+ : "Diagonal's helper isn't registered with this browser yet. Installed with Homebrew? Run this in Terminal, then reopen Diagonal. Installed from a zip? Run: bash <the Diagonal folder>/install-host.command",

50 // A Web Store install has an update_url and may never have had the cask; an unpacked one came from it.

52 // A Web Store install has an update_url and may never have had the cask; an unpacked one came from it.

51 command: chrome.runtime.getManifest().update_url ? "brew install iko-soy/tap/diagonal" : "brew reinstall diagonal",

53 command: chrome.runtime.getManifest().update_url ? "brew install iko-soy/tap/diagonal" : "brew reinstall diagonal",

52 });

54 });

@@ -80,7 +82,7 @@ export function notices(s: Health): Notice[] {

80 out.push({ tone: "bad", title: "Naming is paused", body: s.message || "The helper didn't answer. Diagonal retries on its own." });

82 out.push({ tone: "bad", title: "Naming is paused", body: s.message || "The helper didn't answer. Diagonal retries on its own." });

81 }

83 }

82 }

84 }

83− if (s.timeoutHint && s.health !== "degraded") {

85+ if (s.timeoutHint) {

84 out.push({ tone: "info", title: "The model is slow right now", body: "It timed out three times in a row. You can give it longer under Naming in Settings." });

86 out.push({ tone: "info", title: "The model is slow right now", body: "It timed out three times in a row. You can give it longer under Naming in Settings." });

85 }

87 }

86 return out;

88 return out;

src/shared/label.ts+42−9

@@ -7,9 +7,12 @@ export const LABEL_MAX_CHARS = 28;

7export const LABEL_MIN_WORDS = 2;

8export const LABEL_MAX_WORDS = 4;

10+// Quotes go; an apostrophe between two letters ("Children's", "Rock'n'roll") is part of the word and stays.

10−const QUOTES = /["'“”‘’«»`]/g;

11+const QUOTES = /(?<!\p{L})['’]|['’](?!\p{L})|["“”‘«»`]/gu;

11−const HAS_QUOTE = /["'“”‘’«»`]/;

12+const HAS_QUOTE = /(?<!\p{L})['’]|['’](?!\p{L})|["“”‘«»`]/u;

12const TRAILING_PUNCT = /[\s.,;:!?…\-–—。、，！？：；]+$/u;

13const TRAILING_PUNCT = /[\s.,;:!?…\-–—。、，！？：；]+$/u;

14+// A label cut short should not end mid-phrase: "Planning a trip to" reads as "Planning a trip".

15+const DANGLING = new Set(["a", "an", "the", "to", "of", "for", "in", "on", "at", "by", "and", "or", "with", "from"]);

14/** Repair what is repairable; return undefined when the label cannot be made valid. */

17/** Repair what is repairable; return undefined when the label cannot be made valid. */

15export function repairLabel(raw: unknown): string | undefined {

18export function repairLabel(raw: unknown): string | undefined {

@@ -17,26 +20,49 @@ export function repairLabel(raw: unknown): string | undefined {

17 let words = raw.replace(QUOTES, "").replace(/\s+/g, " ").trim().replace(TRAILING_PUNCT, "").split(" ").filter(Boolean);

20 let words = raw.replace(QUOTES, "").replace(/\s+/g, " ").trim().replace(TRAILING_PUNCT, "").split(" ").filter(Boolean);

18 words = words.filter((w) => !BANNED_WORDS.has(w.toLowerCase().replace(/[^\p{L}\p{N}]/gu, "")));

21 words = words.filter((w) => !BANNED_WORDS.has(w.toLowerCase().replace(/[^\p{L}\p{N}]/gu, "")));

19 if (words.length > LABEL_MAX_WORDS) words = words.slice(0, LABEL_MAX_WORDS);

22 if (words.length > LABEL_MAX_WORDS) words = words.slice(0, LABEL_MAX_WORDS);

23+ if (noSpace(words.join(" "))) {

24+ while (words.length > 1 && widthOf(words.join(" ")) > CJK_MAX_CHARS) words.pop();

25+ if (widthOf(words.join(" ")) > CJK_MAX_CHARS) words = [cutToWidth(words.join(" "), CJK_MAX_CHARS)];

26+ } else {

20− while (words.length > LABEL_MIN_WORDS && words.join(" ").length > LABEL_MAX_CHARS) words.pop();

27+ while (words.length > LABEL_MIN_WORDS && words.join(" ").length > LABEL_MAX_CHARS) words.pop();

28+ }

29+ while (words.length > 1 && DANGLING.has(words[words.length - 1].toLowerCase())) words.pop();

21 let label = words.join(" ").replace(TRAILING_PUNCT, "");

30 let label = words.join(" ").replace(TRAILING_PUNCT, "");

22 if (!label) return undefined;

31 if (!label) return undefined;

32+ // "rust async" becomes "Rust async"; "iPhone tips" and "macOS updates" keep the brand's own casing.

23− label = label[0].toLocaleUpperCase() + label.slice(1);

33+ if (!/\p{Lu}/u.test(words[0].slice(1))) label = label[0].toLocaleUpperCase() + label.slice(1);

24 return isValidLabel(label) ? label : undefined;

34 return isValidLabel(label) ? label : undefined;

25}

35}

37+/**

27−/** Chinese, Japanese and Korean titles have no spaces between words, so they're checked by length instead. */

38+ * Chinese, Japanese, Korean, Thai, Lao, Khmer and Myanmar titles don't put spaces between words, so a label

39+ * with any of their characters ("Python教程") is checked by width instead of word count. Each such character

40+ * is about as wide as two Latin letters on the tab strip.

41+ */

28−const CJK = /[\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uac00-\ud7af\uf900-\ufaff]/g;

42+const NO_SPACE = /[\u0e00-\u0eff\u1000-\u109f\u1780-\u17ff\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uac00-\ud7af\uf900-\ufaff]/g;

29export const CJK_MIN_CHARS = 2;

43export const CJK_MIN_CHARS = 2;

30export const CJK_MAX_CHARS = 16;

44export const CJK_MAX_CHARS = 16;

32−const isCjk = (label: string): boolean => (label.match(CJK)?.length ?? 0) * 2 > label.replace(/ /g, "").length;

46+const noSpace = (label: string): boolean => (label.match(NO_SPACE)?.length ?? 0) > 0;

47+const widthOf = (label: string): number => {

48+ const wide = label.match(NO_SPACE)?.length ?? 0;

49+ return wide + (label.length - wide) / 2;

50+};

51+const cutToWidth = (label: string, max: number): string => {

52+ let out = "";

53+ for (const ch of label) {

54+ if (widthOf(out + ch) > max) break;

55+ out += ch;

56+ }

57+ return out.trim();

58+};

34export function isValidLabel(label: string): boolean {

60export function isValidLabel(label: string): boolean {

35 const words = label.split(" ").filter(Boolean);

61 const words = label.split(" ").filter(Boolean);

36− const cjk = isCjk(label);

62+ const cjk = noSpace(label);

37 return (

63 return (

38 label.length >= (cjk ? CJK_MIN_CHARS : LABEL_MIN_CHARS) &&

64 label.length >= (cjk ? CJK_MIN_CHARS : LABEL_MIN_CHARS) &&

39− label.length <= (cjk ? CJK_MAX_CHARS : LABEL_MAX_CHARS) &&

65+ (cjk ? widthOf(label) <= CJK_MAX_CHARS && label.length <= LABEL_MAX_CHARS : label.length <= LABEL_MAX_CHARS) &&

40 words.length >= (cjk ? 1 : LABEL_MIN_WORDS) &&

66 words.length >= (cjk ? 1 : LABEL_MIN_WORDS) &&

41 words.length <= LABEL_MAX_WORDS &&

67 words.length <= LABEL_MAX_WORDS &&

42 !HAS_QUOTE.test(label) &&

68 !HAS_QUOTE.test(label) &&

@@ -61,3 +87,10 @@ export function labelOf(title: string | undefined): string {

62export const sameLabel = (a: string | undefined, b: string | undefined): boolean =>

88export const sameLabel = (a: string | undefined, b: string | undefined): boolean =>

63 !!a && !!b && labelOf(a).trim().toLocaleLowerCase() === labelOf(b).trim().toLocaleLowerCase();

89 !!a && !!b && labelOf(a).trim().toLocaleLowerCase() === labelOf(b).trim().toLocaleLowerCase();

90+

91+/**

92+ * A page title without the unread counts and bullets sites add to it, "(3) Inbox" or "• Slack", so a count

93+ * ticking over does not read as a new page. Only for noticing change: the model still sees the full title.

94+ */

95+export const titleKey = (title: string | undefined): string =>

96+ (title ?? "").replace(/[(\[]\d[\d,.]*\+?[)\]]/g, "").replace(/^[\s•●▶*]+/u, "").replace(/\s+/g, " ").trim();

test/auto.test.ts+84−2

@@ -15,6 +15,7 @@ let state: State;

15let calls: OrganizePayload[];

16let replies: HostReply<unknown>[];

17let fallbacks: number[];

18+let respond: ((payload: OrganizePayload, opts?: object) => HostReply<unknown> | undefined) | undefined;

19const liveTab = (id: number, over: Partial<LiveTab> = {}): LiveTab => ({

20const liveTab = (id: number, over: Partial<LiveTab> = {}): LiveTab => ({

20 id, windowId: 1, index: id, groupId: -1, url: `https://site${id}.com/page`, title: `Tab ${id}`, status: "complete", pinned: false, ...over,

21 id, windowId: 1, index: id, groupId: -1, url: `https://site${id}.com/page`, title: `Tab ${id}`, status: "complete", pinned: false, ...over,

@@ -32,9 +33,9 @@ function setup(tabs: LiveTab[]): AutoOrganizer {

32 settings: () => makeSettings(),

33 settings: () => makeSettings(),

33 commit: () => undefined,

34 commit: () => undefined,

34 now: () => 1_000_000,

35 now: () => 1_000_000,

35− host: async <T,>(_op: string, payload: object) => {

36+ host: async <T,>(_op: string, payload: object, opts?: object) => {

36 calls.push(JSON.parse(JSON.stringify(payload)));

37 calls.push(JSON.parse(JSON.stringify(payload)));

37− return (replies.shift() ?? { ok: true, result: { groups: [], leftovers: [] } }) as HostReply<T>;

38+ return (respond?.(payload as OrganizePayload, opts) ?? replies.shift() ?? { ok: true, result: { groups: [], leftovers: [] } }) as HostReply<T>;

38 },

39 },

39 naming: { touch: () => undefined } as unknown as Runtime["naming"],

40 naming: { touch: () => undefined } as unknown as Runtime["naming"],

40 refreshBadge: () => undefined,

41 refreshBadge: () => undefined,

@@ -48,6 +49,7 @@ beforeEach(() => {

48 calls = [];

49 calls = [];

49 replies = [];

50 replies = [];

50 fallbacks = [];

51 fallbacks = [];

52+ respond = undefined;

51 nextGroup = 100;

53 nextGroup = 100;

52 (globalThis as any).chrome = {

54 (globalThis as any).chrome = {

53 tabs: {

55 tabs: {

@@ -92,6 +94,7 @@ describe("auto-organize", () => {

92 await auto.run(1);

94 await auto.run(1);

93 expect(calls[0].items.map((i) => i.title)).toEqual(["Tab 3", "Tab 2"]);

95 expect(calls[0].items.map((i) => i.title)).toEqual(["Tab 3", "Tab 2"]);

94 expect(calls[0].existingGroups).toEqual([expect.objectContaining({ g: 0, title: "Rust" })]);

96 expect(calls[0].existingGroups).toEqual([expect.objectContaining({ g: 0, title: "Rust" })]);

97+ expect(calls[0].siblingTitles).toEqual(["Rust"]);

95 expect(live.find((t) => t.id === 3)!.groupId).toBe(50);

98 expect(live.find((t) => t.id === 3)!.groupId).toBe(50);

96 });

99 });

100

@@ -149,4 +152,83 @@ describe("auto-organize", () => {

149 vi.useRealTimers();

152 vi.useRealTimers();

150 }

153 }

151 });

154 });

155+

156+ it("one tab the model refuses is found and set aside; the others are still sorted", async () => {

157+ const auto = setup([liveTab(1), liveTab(2), liveTab(3, { title: "Refused" }), liveTab(4), liveTab(5)]);

158+ // Apple's filter refuses any request carrying tab 3; anything else becomes one topic.

159+ respond = (p) =>

160+ p.items.some((i) => i.title === "Refused")

161+ ? { ok: false, error: { code: "GUARDRAIL", message: "blocked" } }

162+ : { ok: true, result: { groups: [{ title: "Site pages", emoji: "📚", color: "blue", existing: p.existingGroups.length ? 0 : undefined, members: p.items.map((i) => i.i) }], leftovers: [] } };

163+ await auto.run(1);

164+ expect(live.find((t) => t.id === 3)!.groupId).toBe(-1);

165+ expect(live.filter((t) => t.id !== 3).every((t) => t.groupId === 100)).toBe(true);

166+ expect(state.tabs[3].organizeRefused).toBe(organizedKeyOf(live[2]));

167+ expect(fallbacks).toHaveLength(0); // not a host problem: no retry scheduled

168+

169+ const n = calls.length;

170+ live.push(liveTab(6));

171+ remember(live[5]);

172+ await auto.run(1);

173+ expect(calls.slice(n).flatMap((c) => c.items.map((i) => i.title))).toEqual(["Tab 6"]); // the refused tab is not sent again

174+

175+ const m = calls.length;

176+ live[2].title = "Readable now";

177+ await auto.run(1);

178+ expect(calls.length).toBeGreaterThan(m); // a new title: it gets another chance

179+ });

180+

181+ it("a refused request is tried once more without what the pages say", async () => {

182+ const auto = setup([liveTab(1), liveTab(2)]);

183+ state.tabs[1].description = "Something the filter dislikes";

184+ respond = (p) =>

185+ p.items.some((i) => i.description)

186+ ? { ok: false, error: { code: "GUARDRAIL", message: "blocked" } }

187+ : { ok: true, result: { groups: [{ title: "Pair", emoji: "📚", color: "blue", members: [0, 1] }], leftovers: [] } };

188+ await auto.run(1);

189+ expect(calls).toHaveLength(2);

190+ expect(live.map((t) => t.groupId)).toEqual([100, 100]);

191+ });

192+

193+ it("tabs that changed while the model was thinking are left out of its answer", async () => {

194+ const auto = setup([liveTab(1), liveTab(2), liveTab(3), liveTab(4)]);

195+ respond = () => {

196+ live[1].url = "https://elsewhere.com/"; // tab 2 went to another page

197+ live = live.filter((t) => t.id !== 4); // tab 4 was closed

198+ return { ok: true, result: { groups: [{ title: "Site pages", emoji: "📚", color: "blue", members: [0, 1, 2] }, { title: "Other", emoji: "📚", color: "red", members: [3] }], leftovers: [] } };

199+ };

200+ await auto.run(1);

201+ expect(live.map((t) => [t.id, t.groupId])).toEqual([[1, 100], [2, -1], [3, 100]]);

202+ expect(state.tabs[1].organizedKey).toBeDefined();

203+ expect(state.tabs[2].organizedKey).toBeUndefined(); // looked at again on its new page

204+ });

205+

206+ it("a new group left with one tab after the others moved on is not made", async () => {

207+ const auto = setup([liveTab(1), liveTab(2), liveTab(3)]);

208+ respond = () => {

209+ live[1].groupId = 77; // you dragged tab 2 into a group of yours meanwhile

210+ return { ok: true, result: { groups: [{ title: "Pair", emoji: "📚", color: "blue", members: [0, 1] }], leftovers: [2] } };

211+ };

212+ await auto.run(1);

213+ expect(live.find((t) => t.id === 1)!.groupId).toBe(-1);

214+ expect(nextGroup).toBe(100);

215+ });

216+

217+ it("an unread count ticking over is not a new page", async () => {

218+ const auto = setup([liveTab(1, { title: "(3) Inbox" }), liveTab(2)]);

219+ await auto.run(1);

220+ expect(calls).toHaveLength(1);

221+ live[0].title = "(4) Inbox";

222+ await auto.run(1);

223+ expect(calls).toHaveLength(1);

224+ });

225+

226+ it("a host that stops answering stops the run after one smaller try", async () => {

227+ const auto = setup(Array.from({ length: 10 }, (_, i) => liveTab(i + 1)));

228+ respond = () => ({ ok: false, error: { code: "TIMEOUT", message: "slow" } });

229+ await auto.run(1);

230+ expect(calls.map((c) => c.items.length)).toEqual([10, 5]);

231+ expect(fallbacks).toHaveLength(1);

232+ expect(Object.values(state.tabs).some((t) => t.organizeRefused)).toBe(false);

233+ });

152});

234});

test/naming.test.ts+72−2

@@ -50,6 +50,10 @@ describe("evolving-name policy", () => {

50 it("same label is never rewritten", () => {

51 expect(shouldApply(g(), "rust async runtimes", ["a", "b", "c"])).toBe(false);

52 });

53+ it("a member that only moved within its page is not a membership change", () => {

54+ const g2 = newGroupRecord(1, 1, "opener", "blue", { title: "Rust async runtimes", lastNamedAt: 1, memberUrls: ["https://a.com/x", "https://b.com/y"] });

55+ expect(shouldApply(g2, "Tokio and async-std", ["https://a.com/x?page=2", "https://b.com/y#top"])).toBe(false);

56+ });

53 it("user-named groups are skipped", () => {

57 it("user-named groups are skipped", () => {

54 expect(shouldApply({ ...g(), userNamed: true }, "Anything else", ["x"])).toBe(false);

58 expect(shouldApply({ ...g(), userNamed: true }, "Anything else", ["x"])).toBe(false);

55 });

59 });

@@ -151,15 +155,25 @@ describe("naming queue", () => {

151 expect(state.groups[7].dirty).toBe(false);

155 expect(state.groups[7].dirty).toBe(false);

152 });

156 });

153

157

154− it("waits while a member is still loading or the group has one tab", async () => {

158+ it("waits while a member is still loading", async () => {

155 members = [member(1), member(2, { status: "loading" })];

159 members = [member(1), member(2, { status: "loading" })];

156 naming.touch(7);

160 naming.touch(7);

157 await settle();

161 await settle();

162+ expect(calls).toHaveLength(0);

163+ expect(state.groups[7].dirty).toBe(true);

164+ });

165+

166+ it("a group with one tab to name from is not kept waiting; the next tab to join names it", async () => {

158 members = [member(1)];

167 members = [member(1)];

159 naming.touch(7);

168 naming.touch(7);

160 await settle();

169 await settle();

161 expect(calls).toHaveLength(0);

170 expect(calls).toHaveLength(0);

162− expect(state.groups[7].dirty).toBe(true);

171+ expect(state.groups[7].dirty).toBe(false);

172+ members = [member(1), member(2)];

173+ markDirty(state.groups[7], 0);

174+ naming.touch(7);

175+ await settle();

176+ expect(calls).toHaveLength(1);

163 });

177 });

164

178

165 it("names a group whose member the browser discarded to save memory", async () => {

179 it("names a group whose member the browser discarded to save memory", async () => {

@@ -206,6 +220,54 @@ describe("naming queue", () => {

206 expect(state.groups[7].nextAttemptAt).toBe(now + 24 * 3_600_000);

220 expect(state.groups[7].nextAttemptAt).toBe(now + 24 * 3_600_000);

207 });

221 });

208

222

223+ it("a group Apple's filter refused is asked again as soon as its tabs change, not only after a day", async () => {

224+ replies.push({ ok: false, error: { code: "GUARDRAIL", message: "no" } }, { ok: false, error: { code: "GUARDRAIL", message: "no" } });

225+ naming.touch(7);

226+ await settle();

227+ expect(calls).toHaveLength(2);

228+ naming.touch(7);

229+ await settle();

230+ expect(calls).toHaveLength(2); // the same tabs: still refused

231+ members = [member(1), member(3)];

232+ naming.touch(7);

233+ await settle();

234+ expect(calls).toHaveLength(3);

235+ expect(state.groups[7]).toMatchObject({ title: "Rust crate docs", dirty: false, nameAttempts: 0 });

236+ expect(state.groups[7].refusedHash).toBeUndefined();

237+ });

238+

239+ it("setup errors don't use up a group's retries; the host's recovery picks it up", async () => {

240+ replies.push({ ok: false, error: { code: "LICENSE_REQUIRED", message: "terms" } });

241+ naming.touch(7);

242+ await settle();

243+ expect(state.groups[7]).toMatchObject({ dirty: true, nameAttempts: 0 });

244+ expect(state.groups[7].firstFailureAt).toBeUndefined();

245+ expect(state.groups[7].nextAttemptAt).toBeUndefined();

246+ });

247+

248+ it("a title the strip would not take is tried again shortly", async () => {

249+ let refuse = true;

250+ naming = new Naming({ ...(naming as any).d, writeTitle: async () => !refuse });

251+ naming.touch(7);

252+ await settle();

253+ expect(state.groups[7]).toMatchObject({ dirty: true, nextAttemptAt: now + 30_000 });

254+ expect(state.groups[7].title).toBeUndefined();

255+ expect(state.ownWrites[7]).toBeUndefined();

256+ refuse = false;

257+ now += 30_000;

258+ await naming.consider(7);

259+ await settle();

260+ expect(state.groups[7]).toMatchObject({ dirty: false, title: "Rust crate docs" });

261+ });

262+

263+ it("members back as they were last named clear old failures", async () => {

264+ Object.assign(state.groups[7], { membersHash: membersHash(members), nameAttempts: 3, firstFailureAt: 1, nextAttemptAt: 1 });

265+ naming.touch(7);

266+ await settle();

267+ expect(state.groups[7]).toMatchObject({ dirty: false, nameAttempts: 0 });

268+ expect(state.groups[7].firstFailureAt).toBeUndefined();

269+ });

270+

209 it("failures back off 30 s, 2 min, 10 min, then hourly; the title is left alone", async () => {

271 it("failures back off 30 s, 2 min, 10 min, then hourly; the title is left alone", async () => {

210 const fail: HostReply<NameResult> = { ok: false, error: { code: "FM_ERROR", message: "boom" } };

272 const fail: HostReply<NameResult> = { ok: false, error: { code: "FM_ERROR", message: "boom" } };

211 replies.push(fail);

273 replies.push(fail);

@@ -237,6 +299,14 @@ describe("naming queue", () => {

237 expect(writes).toEqual([[7, "🦀 Rust crates"]]);

299 expect(writes).toEqual([[7, "🦀 Rust crates"]]);

238 });

300 });

239

301

302+ it("the retry after a title the host rejected lists that title as not allowed", async () => {

303+ replies.push({ ok: false, error: { code: "BAD_MODEL_OUTPUT", message: "rules", raw: JSON.stringify({ title: "Python", emoji: "🐍" }) } });

304+ naming.touch(7);

305+ await settle();

306+ expect(calls).toHaveLength(2);

307+ expect(calls[1].mustDifferFrom).toEqual(["Python"]);

308+ });

309+

240 it("a group renamed by hand while the model runs keeps the user's title", async () => {

310 it("a group renamed by hand while the model runs keeps the user's title", async () => {

241 const n = new Naming({

311 const n = new Naming({

242 ...(naming as any).d,

312 ...(naming as any).d,

test/organize.test.ts+17

@@ -107,6 +107,23 @@ describe("existing groups", () => {

107 expect(list[0]).toEqual({ g: 0, title: "Trip 0", samples: ["Hotel"] });

108 });

109

110+ it("offers the groups in use when there are more than 12, not the oldest", () => {

111+ const s = emptyState();

112+ for (let i = 0; i < 15; i++) s.groups[i] = newGroupRecord(i, 1, "organize", "red", { stripTitle: `G ${i}`, lastNamedAt: 1000 - i });

113+ s.tabs[1] = { id: 1, windowId: 1, groupId: 14, index: 0, url: "u", title: "Now", pinned: false, createdAt: 0, lastActivatedAt: 5000 };

114+ const { ids } = existingGroupsFor(s, 1);

115+ expect(ids[0]).toBe(14);

116+ expect(ids).toContain(0);

117+ expect(ids).not.toContain(13);

118+ });

119+

120+ it("leaves out the Restored group, which is not a topic", () => {

121+ const s = emptyState();

122+ s.groups[1] = newGroupRecord(1, 1, "organize", "grey", { userNamed: true, stripTitle: "♻️ Restored", restored: true });

123+ s.groups[2] = newGroupRecord(2, 1, "organize", "red", { stripTitle: "🦀 Rust" });

124+ expect(existingGroupsFor(s, 1).ids).toEqual([2]);

125+ });

126+

110 it("groups created by batch n come first for batch n+1", () => {

127 it("groups created by batch n come first for batch n+1", () => {

111 const s = emptyState();

128 const s = emptyState();

112 for (let i = 0; i < 15; i++) s.groups[i] = newGroupRecord(i, 1, "organize", "red", { stripTitle: `G ${i}` });

129 for (let i = 0; i < 15; i++) s.groups[i] = newGroupRecord(i, 1, "organize", "red", { stripTitle: `G ${i}` });

test/url.test.ts+16−13

@@ -1,7 +1,8 @@

1import { describe, expect, it } from "vitest";

2−import { isValidLabel, labelOf, repairLabel, safeEmoji } from "../src/shared/label";

2+import { isValidLabel, labelOf, repairLabel, safeEmoji, titleKey } from "../src/shared/label";

3import { colorFor, hostname, isInternalUrl, promptAddress, promptUrl, provisionalTitle, registrableDomain, trimText } from "../src/shared/url";

4import { withDefaults } from "../src/background/settings";

5+import labelCases from "../host/tests/golden/labels.json";

6describe("url helpers", () => {

7describe("url helpers", () => {

7 it("strips www.", () => {

8 it("strips www.", () => {

@@ -45,19 +46,12 @@ describe("url helpers", () => {

45 });

46 });

46});

47});

48−describe("labels", () => {

49+// The same cases run against the host's validate.repair_label, so the two sides agree on every label.

49− it.each([

50+const LABEL_CASES = labelCases as [string, string | null][];

50− ['"Rust async runtimes."', "Rust async runtimes"],

51− ["rust async runtimes", "Rust async runtimes"],

52− ["Rust tabs and docs", "Rust and docs"],

53− ["One two three four five six", "One two three four"],

54− ["Machine learning research papers reading list", "Machine learning research"],

55− ])("repairs %s", (raw, fixed) => {

56− expect(repairLabel(raw)).toBe(fixed);

57− });

59− it.each(["Tabs", "Misc stuff", "", "x", "Supercalifragilisticexpialidocious extraordinarily"])("rejects %s", (raw) => {

52+describe("labels", () => {

53+ it.each(LABEL_CASES)("repairs %s", (raw, fixed) => {

60− expect(repairLabel(raw)).toBeUndefined();

54+ expect(repairLabel(raw) ?? null).toBe(fixed);

61 });

55 });

63 it("validates", () => {

57 it("validates", () => {

@@ -74,6 +68,15 @@ describe("labels", () => {

74 expect(safeEmoji("🦄")).toBe("🧭");

68 expect(safeEmoji("🦄")).toBe("🧭");

75 });

69 });

71+ it("titleKey ignores the unread counts and bullets sites put in titles", () => {

72+ expect(titleKey("(3) Inbox - Mail")).toBe(titleKey("(12) Inbox - Mail"));

73+ expect(titleKey("Inbox (1,204) - me@example.com - Gmail")).toBe("Inbox - me@example.com - Gmail");

74+ expect(titleKey("• Slack | general")).toBe("Slack | general");

75+ expect(titleKey("[99+] Chat")).toBe("Chat");

76+ expect(titleKey("Rust async")).toBe("Rust async");

77+ expect(titleKey("Rust async")).not.toBe(titleKey("Rust sync"));

78+ });

79+

77 it("labelOf strips our emoji prefix", () => {

80 it("labelOf strips our emoji prefix", () => {

78 expect(labelOf("🦀 Rust async")).toBe("Rust async");

81 expect(labelOf("🦀 Rust async")).toBe("Rust async");

79 expect(labelOf("Plain title")).toBe("Plain title");

82 expect(labelOf("Plain title")).toBe("Plain title");

tsconfig.json+1

@@ -9,6 +9,7 @@

9 "noEmit": true,

10 "skipLibCheck": true,

11 "isolatedModules": true,

12+ "resolveJsonModule": true,

12 "noFallthroughCasesInSwitch": true,

13 "noFallthroughCasesInSwitch": true,

13 "forceConsistentCasingInFileNames": true

14 "forceConsistentCasingInFileNames": true

14 },

15 },