@@ -1,16 +1,19 @@
11import { COLORS, isColor, type GroupColor } from "../shared/colors";
2−import { labelOf, repairLabel, safeEmoji, stripTitle } from "../shared/label";
2+import { labelOf, repairLabel, safeEmoji, stripTitle, titleKey } from "../shared/label";
33import { colorFor, isInternalUrl, promptAddress, provisionalTitle } from "../shared/url";
44import { pathKey } from "./engine";
5−import { membersHash, memberUrls, toItems, type Member, type NameItem } from "./naming";
5+import { membersHash, memberUrls, settled, toItems, type Member, type NameItem } from "./naming";
66import { addToGroup, createManagedGroup, groupExists, ungroup, type Runtime } from "./runtime";
7−import { markDirty, type State } from "./state";
7+import type { HostReply } from "./host";
8+import { markDirty, type GroupRecord, type HostError, type HostErrorCode, type State, type TabRecord } from "./state";
89
910/** Section 3/9: "Organize all tabs" — cluster ungrouped tabs in batches sized to the model's window. */
1011
1112export const ORGANIZE_ITEM_CAP = 36;
1213export const EXISTING_GROUPS_CAP = 12;
13−export const CHAR_BUDGET = 24_000; // matches the host's budget for fm's ~8k-token context
14+// About the host's 7,000-token budget in English characters. Chinese and Japanese cost about twice the tokens
15+// per character, so their characters count double (weightedLength); the host measures exactly and shortens.
16+export const CHAR_BUDGET = 24_000;
1417export const INSTRUCTION_CHARS = 1_400; // ~320 tokens of rules, emoji and colour lists
1518export const EXISTING_GROUP_CHARS = 90;
1619export const UNDO_WINDOW_MS = 3_600_000;
@@ -24,6 +27,8 @@ export interface ExistingGroup {
2427export interface OrganizePayload {
2528 items: NameItem[];
2629 existingGroups: ExistingGroup[];
30+ /** Every group label in the window: a new group the host names must not repeat one. */
31+ siblingTitles?: string[];
2732 allowNew: boolean;
2833 maxGroups: number;
2934 minGroupSize: number;
@@ -53,9 +58,12 @@ export const maxGroupsFor = (n: number): number => Math.min(8, Math.ceil(n / 3))
5358export const organizable = (t: { groupId?: number; pinned?: boolean; url?: string; incognito?: boolean }): boolean =>
5459 (t.groupId ?? -1) === -1 && !t.pinned && !t.incognito && !isInternalUrl(t.url);
5560
61+const CJK = /[\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uac00-\ud7af\uf900-\ufaff]/g;
62+export const weightedLength = (s: string): number => s.length + (s.match(CJK)?.length ?? 0);
63+
5664/** Rough rendered size of one item line, the unit the character budget is spent in. */
5765export const itemChars = (m: Pick<Member, "title" | "url" | "description">): number =>
58− Math.min(120, m.title.length) + promptAddress(m.url).length + Math.min(500, m.description?.length ?? 1) + 12;
66+ weightedLength(m.title.slice(0, 120)) + promptAddress(m.url).length + weightedLength((m.description ?? " ").slice(0, 500)) + 12;
5967
6068/** Split tabs (in strip order) into batches of ≤ cap items that also fit the character budget. */
6169export function makeBatches<T extends Pick<Member, "title" | "url" | "description">>(
@@ -146,88 +154,198 @@ export interface OrganizeReport {
146154 error?: string;
147155}
148156
149−/** Existing groups a batch may add tabs to: extension-managed topic groups in this window. */
157+/** Existing groups a batch may add tabs to: Diagonal's topic groups in this window, the ones in use first. */
150158export function existingGroupsFor(state: State, windowId: number, extraIds: number[] = []): { ids: number[]; list: ExistingGroup[] } {
159+ const members = new Map<number, TabRecord[]>();
160+ for (const t of Object.values(state.tabs)) if (t.groupId !== -1) members.set(t.groupId, [...(members.get(t.groupId) ?? []), t]);
161+ const inUse = (g: GroupRecord): number =>
162+ Math.max(g.dirtyAt ?? 0, g.lastNamedAt ?? 0, ...(members.get(g.id) ?? []).map((t) => t.lastActivatedAt ?? 0));
151163 const groups = Object.values(state.groups)
152− .filter((g) => g.windowId === windowId && g.managed && g.origin !== "tidy" && (g.stripTitle || g.title))
153− .sort((a, b) => Number(extraIds.includes(b.id)) - Number(extraIds.includes(a.id)))
154− .slice(0, EXISTING_GROUPS_CAP);
164+ .filter((g) => g.windowId === windowId && g.managed && g.origin !== "tidy" && !g.restored && (g.stripTitle || g.title))
165+ .map((g) => ({ g, first: extraIds.includes(g.id), at: inUse(g) }))
166+ .sort((a, b) => Number(b.first) - Number(a.first) || b.at - a.at)
167+ .slice(0, EXISTING_GROUPS_CAP)
168+ .map(({ g }) => g);
155169 const list = groups.map((g, i) => ({
156170 g: i,
157171 title: labelOf(g.stripTitle || g.title),
158− samples: Object.values(state.tabs)
159− .filter((t) => t.groupId === g.id)
160− .slice(0, 2)
161− .map((t) => t.title.slice(0, 60)),
172+ samples: (members.get(g.id) ?? []).slice(0, 2).map((t) => t.title.slice(0, 60)),
162173 }));
163174 return { ids: groups.map((g) => g.id), list };
164175}
165176
166−export async function organizeWindow(rt: Runtime, windowId: number): Promise<OrganizeReport> {
177+/** Errors about what was sent rather than about the host: a smaller or plainer request may go through. */
178+const CONTENT_ERRORS = new Set<HostErrorCode>(["OVER_BUDGET", "GUARDRAIL", "BAD_MODEL_OUTPUT"]);
179+const MAX_TIMEOUT_SPLITS = 1;
180+
181+/** One organize call, with the retries that can rescue the same tabs: strict output, then no page text. */
182+async function askOrganize(rt: Runtime, part: Member[], windowId: number, existingGroups: ExistingGroup[], withDescriptions: boolean): Promise<HostReply<OrganizeResult>> {
183+ const settings = rt.settings();
184+ const s = rt.state();
185+ const payload: OrganizePayload = {
186+ items: toItems(part, settings, withDescriptions),
187+ existingGroups,
188+ siblingTitles: Object.values(s.groups)
189+ .filter((g) => g.windowId === windowId && g.origin !== "tidy")
190+ .map((g) => labelOf(g.stripTitle || g.title))
191+ .filter(Boolean)
192+ .slice(0, 20),
193+ allowNew: part.length >= 2,
194+ maxGroups: maxGroupsFor(part.length),
195+ minGroupSize: settings.organizeMinGroupSize,
196+ };
197+ let reply = await rt.host<OrganizeResult>("organize", payload);
198+ if (!reply.ok && reply.error.code === "BAD_MODEL_OUTPUT") reply = await rt.host<OrganizeResult>("organize", payload, { strict: true });
199+ if (!reply.ok && reply.error.code === "GUARDRAIL" && payload.items.some((i) => i.description)) {
200+ // What a page says is the likeliest thing to trip Apple's filter; its title and address rarely do.
201+ payload.items = toItems(part, settings, false);
202+ reply = await rt.host<OrganizeResult>("organize", payload);
203+ }
204+ return reply;
205+}
206+
207+/** One organize run over a window: what it made, what it moved, and whether the host stopped it. */
208+export interface SortRun {
209+ windowId: number;
210+ createdIds: number[];
211+ previous: Record<number, number>;
212+ grouped: number;
213+ left: number;
214+ timeoutSplits: number;
215+ /** Auto-organize: remember what was looked at, so the same tabs are not sent again unchanged. */
216+ stamp: boolean;
217+ stopped?: HostError;
218+}
219+
220+export const newRun = (windowId: number, stamp: boolean): SortRun => ({ windowId, createdIds: [], previous: {}, grouped: 0, left: 0, timeoutSplits: 0, stamp });
221+
222+/**
223+ * Sort one part of a window. A reply about the tabs themselves (too long, refused by Apple's filter, not
224+ * valid output) splits the part in halves, so one page the model won't handle leaves the others sorted; a
225+ * tab that fails alone is set aside until its page or title changes. Tabs that changed while the model was
226+ * thinking are left out of its answer.
227+ */
228+export async function sortPart(rt: Runtime, part: Member[], run: SortRun, probe = false): Promise<void> {
229+ if (!part.length || run.stopped) return;
230+ const { ids: existingIds, list: existingGroups } = existingGroupsFor(rt.state(), run.windowId, run.createdIds);
231+ if (part.length < 2 && !existingIds.length && !probe) {
232+ run.left += part.length;
233+ return; // one tab and nothing to join: no call can place it (a probe still learns whether the model takes it)
234+ }
235+ const reply = await askOrganize(rt, part, run.windowId, existingGroups, true);
236+ if (!reply.ok) {
237+ const e = reply.error;
238+ const timedOut = e.code === "TIMEOUT" && part.length > 4 && run.timeoutSplits < MAX_TIMEOUT_SPLITS;
239+ if (part.length > 1 && (CONTENT_ERRORS.has(e.code) || timedOut)) {
240+ if (timedOut) run.timeoutSplits++;
241+ const half = Math.ceil(part.length / 2);
242+ await sortPart(rt, part.slice(0, half), run, true);
243+ await sortPart(rt, part.slice(half), run, true);
244+ return;
245+ }
246+ if (CONTENT_ERRORS.has(e.code)) {
247+ const rec = rt.state().tabs[part[0].id];
248+ if (rec) rec.organizeRefused = organizedKeyOf(part[0]);
249+ rt.log("organize: set aside a tab the model would not sort", e.code);
250+ run.left += 1;
251+ return;
252+ }
253+ run.stopped = e;
254+ return;
255+ }
167256 const settings = rt.settings();
257+ const plan = planFromResult(reply.result, part.map((m) => m.id), existingIds, settings.organizeMinGroupSize, maxGroupsFor(part.length), (ids) =>
258+ colorFor(part.find((m) => m.id === ids[0])?.url ?? ""),
259+ );
260+ const unchanged = await stillAsSent(rt, part, run.windowId);
261+ const trimmed = trimPlan(rt.state(), plan, unchanged, settings.organizeMinGroupSize);
262+ const failed = await applyPlan(rt, trimmed, run.windowId, part, run.previous, run.createdIds);
263+ const placed = [...trimmed.joins, ...trimmed.creates].reduce((n, x) => n + x.tabIds.filter((id) => !failed.has(id)).length, 0);
264+ run.grouped += placed;
265+ run.left += part.length - placed;
266+ if (run.stamp) {
267+ const s = rt.state();
268+ for (const m of part) {
269+ const rec = s.tabs[m.id];
270+ if (rec && unchanged.has(m.id) && !failed.has(m.id)) rec.organizedKey = organizedKeyOf(m);
271+ }
272+ }
273+}
274+
275+/** The tabs of a part that are still loose, in this window, on the page and title the model saw. */
276+async function stillAsSent(rt: Runtime, part: Member[], windowId: number): Promise<Set<number>> {
277+ const live = new Map((await chrome.tabs.query({ windowId }).catch(() => [])).map((t) => [t.id!, t]));
278+ const s = rt.state();
279+ return new Set(
280+ part
281+ .filter((m) => {
282+ const t = live.get(m.id);
283+ return !!t && organizable(t) && !s.tabs[m.id]?.keepLoose && organizedKeyOf(t) === organizedKeyOf(m);
284+ })
285+ .map((m) => m.id),
286+ );
287+}
288+
289+/** A plan with the tabs that moved on taken out: a new group left too small is not made, a join into a group gone is dropped. */
290+export function trimPlan(state: State, plan: Plan, keep: Set<number>, minGroupSize: number): Plan {
291+ const out: Plan = { joins: [], creates: [], leftovers: plan.leftovers.filter((id) => keep.has(id)) };
292+ for (const j of plan.joins) {
293+ const tabIds = j.tabIds.filter((id) => keep.has(id));
294+ const g = state.groups[j.groupId];
295+ if (tabIds.length && g?.managed && !g.restored) out.joins.push({ ...j, tabIds });
296+ else out.leftovers.push(...tabIds);
297+ }
298+ for (const c of plan.creates) {
299+ const tabIds = c.tabIds.filter((id) => keep.has(id));
300+ if (tabIds.length >= Math.max(2, minGroupSize)) out.creates.push({ ...c, tabIds });
301+ else out.leftovers.push(...tabIds);
302+ }
303+ return out;
304+}
305+
306+export async function organizeWindow(rt: Runtime, windowId: number): Promise<OrganizeReport> {
307+ const s0 = rt.state();
168308 const tabs = (await chrome.tabs.query({ windowId })).filter((t) => organizable(t)).sort((a, b) => a.index - b.index);
169309 const report: OrganizeReport = { grouped: 0, groups: 0, left: 0 };
170− if (tabs.length < 2) {
310+ // A tab the model already would not sort, on the same page, is not sent again.
311+ const sendable = tabs.filter((t) => s0.tabs[t.id!]?.organizeRefused !== organizedKeyOf(t));
312+ if (sendable.length < 2) {
171313 report.left = tabs.length;
172314 return report;
173315 }
174− const members: Member[] = tabs.map((t) => ({
175− id: t.id!,
176− index: t.index,
177− url: t.url ?? "",
178− title: t.title ?? "",
179− status: t.status,
180− description: rt.state().tabs[t.id!]?.description,
181− lastActive: t.lastAccessed ?? 0,
182− }));
183− const previous: Record<number, number> = {};
184− const createdIds: number[] = [];
185− const queue = makeBatches(members);
316+ const run = newRun(windowId, true); // auto-organize need not send the same tabs again right after
186317 rt.state().inFlight = { groupId: -1, startedAt: rt.now() };
187318 rt.refreshBadge();
188319 try {
189− while (queue.length) {
190− const batch = queue.shift()!;
191− const { ids: existingIds, list: existingGroups } = existingGroupsFor(rt.state(), windowId, createdIds);
192− const maxGroups = maxGroupsFor(batch.length);
193− const payload: OrganizePayload = {
194− items: toItems(batch, settings),
195− existingGroups,
196− allowNew: true,
197− maxGroups,
198− minGroupSize: settings.organizeMinGroupSize,
199− };
200− let reply = await rt.host<OrganizeResult>("organize", payload);
201− if (!reply.ok && reply.error.code === "OVER_BUDGET" && batch.length > 1) {
202− const half = Math.ceil(batch.length / 2);
203− queue.unshift(batch.slice(0, half), batch.slice(half));
204− continue;
205− }
206− if (!reply.ok && reply.error.code === "BAD_MODEL_OUTPUT") reply = await rt.host<OrganizeResult>("organize", payload, { strict: true });
207− if (!reply.ok) {
208− report.error = `${reply.error.code}: ${reply.error.message}`;
209− report.left += batch.length + queue.flat().length;
210− break;
211− }
212− const batchIds = batch.map((m) => m.id);
213− const plan = planFromResult(reply.result, batchIds, existingIds, settings.organizeMinGroupSize, maxGroups, (ids) =>
214− colorFor(members.find((m) => m.id === ids[0])?.url ?? ""),
215− );
216− await applyPlan(rt, plan, windowId, members, previous, createdIds);
217− report.grouped += plan.joins.reduce((n, j) => n + j.tabIds.length, 0) + plan.creates.reduce((n, c) => n + c.tabIds.length, 0);
218− report.left += plan.leftovers.length;
320+ for (const batch of makeBatches(sendable.map((t) => toMember(rt.state(), t)))) {
321+ await sortPart(rt, batch, run);
322+ if (run.stopped) break;
219323 }
220324 } finally {
221325 const s = rt.state();
222326 if (s.inFlight?.groupId === -1) s.inFlight = undefined;
223− report.groups = createdIds.length;
224− if (Object.keys(previous).length) s.lastOrganize = { at: rt.now(), previous };
327+ report.grouped = run.grouped;
328+ report.groups = run.createdIds.length;
329+ report.left = tabs.length - run.grouped;
330+ if (run.stopped) report.error = `${run.stopped.code}: ${run.stopped.message}`;
331+ if (Object.keys(run.previous).length) s.lastOrganize = { at: rt.now(), previous: run.previous };
225332 rt.commit();
226333 rt.refreshBadge();
227334 }
228335 return report;
229336}
230337
338+const toMember = (s: State, t: chrome.tabs.Tab): Member => ({
339+ id: t.id!,
340+ index: t.index,
341+ url: t.url ?? "",
342+ title: t.title ?? "",
343+ status: t.status,
344+ description: s.tabs[t.id!]?.description,
345+ lastActive: t.lastAccessed ?? 0,
346+});
347+
348+/** Carry out a plan; returns the tabs whose move failed, which stay as they were. */
231349export async function applyPlan(
232350 rt: Runtime,
233351 plan: Plan,
@@ -235,11 +353,15 @@ export async function applyPlan(
235353 members: Member[],
236354 previous: Record<number, number>,
237355 createdIds: number[],
238−): Promise<void> {
356+): Promise<Set<number>> {
239357 const settings = rt.settings();
240358 const byId = new Map(members.map((m) => [m.id, m]));
359+ const failed = new Set<number>();
241360 for (const j of plan.joins) {
242− if (!(await addToGroup(rt, j.tabIds, j.groupId))) continue;
361+ if (!(await addToGroup(rt, j.tabIds, j.groupId))) {
362+ for (const id of j.tabIds) failed.add(id);
363+ continue;
364+ }
243365 for (const id of j.tabIds) previous[id] = -1;
244366 const g = rt.state().groups[j.groupId];
245367 if (g) {
@@ -260,7 +382,10 @@ export async function applyPlan(
260382 lastNamedAt: named ? rt.now() : undefined,
261383 dirty: false,
262384 });
263− if (groupId === undefined) continue;
385+ if (groupId === undefined) {
386+ for (const id of c.tabIds) failed.add(id);
387+ continue;
388+ }
264389 createdIds.push(groupId);
265390 for (const id of c.tabIds) previous[id] = -1;
266391 const g = rt.state().groups[groupId];
@@ -269,6 +394,7 @@ export async function applyPlan(
269394 rt.naming.touch(groupId);
270395 }
271396 }
397+ return failed;
272398}
273399
274400/** Undo within the hour: every tab goes back to the group it was in (organize only takes loose tabs). */
@@ -296,7 +422,7 @@ export async function undoOrganize(rt: Runtime): Promise<number> {
296422}
297423
298424/** What auto-organize remembers about a tab: it looks again only when the page or title changes. */
299−export const organizedKeyOf = (t: { url?: string; title?: string }): string => `${pathKey(t.url)}\n${t.title ?? ""}`;
425+export const organizedKeyOf = (t: { url?: string; title?: string }): string => `${pathKey(t.url)}\n${titleKey(t.title)}`;
300426
301427export const AUTO_RETRY_MS = 300_000;
302428export const AUTO_CONTINUE_MS = 1_500;
@@ -372,60 +498,27 @@ export class AutoOrganizer {
372498 if (pausedUntil && pausedUntil > rt.now()) return; // the host-retry ping sweeps again when it recovers
373499 const tabs = (await chrome.tabs.query({ windowId }).catch(() => [])).sort((a, b) => a.index - b.index);
374500 const s = rt.state();
375− // Still-loading tabs are skipped here; finishing the load touches the window again.
376− const loose = tabs.filter((t) => organizable(t) && t.status === "complete" && !s.tabs[t.id!]?.keepLoose);
501+ // Still-loading tabs are skipped here; finishing the load touches the window again. Discarded tabs keep
502+ // their title and address, so they're sorted like loaded ones.
503+ const loose = tabs.filter((t) => {
504+ const rec = s.tabs[t.id!];
505+ return organizable(t) && settled(t) && !rec?.keepLoose && rec?.organizeRefused !== organizedKeyOf(t);
506+ });
377507 const fresh = loose.filter((t) => s.tabs[t.id!]?.organizedKey !== organizedKeyOf(t));
378508 if (!fresh.length) return;
379− const { ids: existingIds, list: existingGroups } = existingGroupsFor(s, windowId);
380− if (loose.length < 2 && !existingIds.length) return; // one loose tab and nothing to join: wait for company
381− const toMember = (t: chrome.tabs.Tab): Member => ({
382− id: t.id!,
383− index: t.index,
384− url: t.url ?? "",
385− title: t.title ?? "",
386− status: t.status,
387− description: s.tabs[t.id!]?.description,
388− lastActive: t.lastAccessed ?? 0,
389− });
390− const ordered = [...fresh, ...loose.filter((t) => !fresh.includes(t))].map(toMember);
509+ if (loose.length < 2 && !existingGroupsFor(s, windowId).ids.length) return; // one loose tab and nothing to join: wait for company
510+ const ordered = [...fresh, ...loose.filter((t) => !fresh.includes(t))].map((t) => toMember(s, t));
391511 const batch = makeBatches(ordered)[0];
392− const maxGroups = maxGroupsFor(batch.length);
393− const payload: OrganizePayload = {
394− items: toItems(batch, settings),
395− existingGroups,
396− allowNew: batch.length >= 2,
397− maxGroups,
398− minGroupSize: settings.organizeMinGroupSize,
399− };
400− let reply = await rt.host<OrganizeResult>("organize", payload);
401− if (!reply.ok && reply.error.code === "OVER_BUDGET" && batch.length > 1) {
402− // Rare with the budgeted batch; try once with half, the rest follows on the next run.
403− const half = batch.slice(0, Math.ceil(batch.length / 2));
404− payload.items = toItems(half, settings);
405− payload.maxGroups = maxGroupsFor(half.length);
406− payload.allowNew = half.length >= 2;
407− batch.length = half.length;
408− reply = await rt.host<OrganizeResult>("organize", payload);
409− }
410− if (!reply.ok && reply.error.code === "BAD_MODEL_OUTPUT") reply = await rt.host<OrganizeResult>("organize", payload, { strict: true });
411− if (!reply.ok) {
412− rt.log("auto-organize", reply.error.code);
512+ const run = newRun(windowId, true);
513+ await sortPart(rt, batch, run);
514+ rt.commit();
515+ if (run.stopped) {
516+ rt.log("auto-organize", run.stopped.code);
413517 this.deps.scheduleFallback(rt.now() + AUTO_RETRY_MS);
414518 return;
415519 }
416− const batchIds = batch.map((m) => m.id);
417− const plan = planFromResult(reply.result, batchIds, existingIds, settings.organizeMinGroupSize, payload.maxGroups, (ids) =>
418− colorFor(batch.find((m) => m.id === ids[0])?.url ?? ""),
419− );
420− await applyPlan(rt, plan, windowId, batch, {}, []);
421− const live = rt.state();
422− for (const m of batch) {
423− const rec = live.tabs[m.id];
424− if (rec) rec.organizedKey = organizedKeyOf(m);
425− }
426− rt.commit();
427− const remaining = fresh.filter((t) => !batchIds.includes(t.id!));
428− if (remaining.length) this.touch(windowId, AUTO_CONTINUE_MS);
520+ const sent = new Set(batch.map((m) => m.id));
521+ if (fresh.some((t) => !sent.has(t.id!))) this.touch(windowId, AUTO_CONTINUE_MS);
429522 }
430523}
431524