d947711 · parseable-macos

git.iko.soy/parseable-macos

on master help sign in

Fix live tail ordering, clear, dedup, and column state

d947711Claude1 week ago

ParseableViewer/ViewModels/LiveTailViewModel.swift+87−43

compare with master

unifiedsplit

@@ -27,14 +27,26 @@ final class LiveTailViewModel {

27 /// column hidden last session would reappear visible when it shows up in a

28 /// later poll. This retains the intent so it stays hidden when it appears.

29 private var savedHiddenColumns: Set<String> = []

30+ /// The full persisted column order, including columns not yet seen this

31+ /// session — the order counterpart of `savedHiddenColumns`. A column that

32+ /// first shows up in a later poll is slotted back into its saved position

33+ /// (instead of being appended), and saving keeps unseen columns' positions

34+ /// rather than dropping them from the stored order.

35+ private var savedColumnOrder: [String] = []

30 private var currentStream: String?

36 private var currentStream: String?

32 // nonisolated(unsafe) so deinit can invalidate the timer.

38 // nonisolated(unsafe) so deinit can invalidate the timer.

33 // Only mutated from @MainActor methods (start/stop).

39 // Only mutated from @MainActor methods (start/stop).

34 nonisolated(unsafe) private var timer: Timer?

40 nonisolated(unsafe) private var timer: Timer?

35 private var lastTimestamp: Date?

41 private var lastTimestamp: Date?

42+ /// How many records with each content fingerprint have been delivered. A

43+ /// count rather than a set: Parseable stamps every event of one ingest request

44+ /// with the same millisecond `p_timestamp`, so genuinely repeated events (the

45+ /// same log line emitted three times) fingerprint identically. A set collapsed

46+ /// them into one row; counting keeps each occurrence while still suppressing

47+ /// the re-fetch of the same rows by the next overlapping poll window.

36− private var seenFingerprints: Set<String> = []

48+ private var seenFingerprints: [String: Int] = [:]

37− /// Insertion-ordered mirror of `seenFingerprints` (oldest first), used to

49+ /// Insertion-ordered mirror of `seenFingerprints` keys (oldest first), used to

38 /// evict the least-recently-seen fingerprints when the set is trimmed.

50 /// evict the least-recently-seen fingerprints when the set is trimmed.

39 private var seenFingerprintOrder: [String] = []

51 private var seenFingerprintOrder: [String] = []

40 /// Upper bound on the dedup fingerprint set. Kept well above `maxEntries` so

52 /// Upper bound on the dedup fingerprint set. Kept well above `maxEntries` so

@@ -209,7 +221,7 @@ final class LiveTailViewModel {

209 isPolling = false

221 isPolling = false

210 errorMessage = nil

222 errorMessage = nil

211 entries = []

223 entries = []

212− seenFingerprints = []

224+ seenFingerprints = [:]

213 seenFingerprintOrder = []

225 seenFingerprintOrder = []

214 allKnownKeys = []

226 allKnownKeys = []

215 droppedCount = 0

227 droppedCount = 0

@@ -220,6 +232,7 @@ final class LiveTailViewModel {

220 columnOrder = []

232 columnOrder = []

221 hiddenColumns = []

233 hiddenColumns = []

222 savedHiddenColumns = []

234 savedHiddenColumns = []

235+ savedColumnOrder = []

223 columnFilters = []

236 columnFilters = []

224 // Clear the text filter too. Leaving it set while clearing the column-filter

237 // Clear the text filter too. Leaving it set while clearing the column-filter

225 // chips is inconsistent and would let a stale search from a previous stream

238 // chips is inconsistent and would let a stale search from a previous stream

@@ -262,12 +275,12 @@ final class LiveTailViewModel {

262 // next timer tick skip itself via `guard !isPolling`.

275 // next timer tick skip itself via `guard !isPolling`.

263 isPolling = false

276 isPolling = false

264 entries = []

277 entries = []

265− seenFingerprints = []

278+ // Keep the dedup fingerprints (unlike start(), which begins a new session):

266− seenFingerprintOrder = []

279+ // every poll looks back at least 90 s regardless of `lastTimestamp` (see

267− // Advance the look-back floor to now: without this, the next poll queries

280+ // `safeStart` in poll()), so the records just cleared are re-fetched by the

268− // from the old `lastTimestamp` with an empty fingerprint set and re-inserts

281+ // next poll. With an emptied fingerprint set they all passed dedup again and

269− // every just-cleared record still inside the ~90 s window, repopulating the

282+ // the tail refilled with exactly what the user cleared. The set stays

270− // view the user just emptied. Matches start()'s reset.

283+ // bounded by the eviction in poll().

271 lastTimestamp = Date()

284 lastTimestamp = Date()

272 allKnownKeys = []

285 allKnownKeys = []

273 droppedCount = 0

286 droppedCount = 0

@@ -282,6 +295,7 @@ final class LiveTailViewModel {

282 columnOrder = []

295 columnOrder = []

283 hiddenColumns = []

296 hiddenColumns = []

284 savedHiddenColumns = []

297 savedHiddenColumns = []

298+ savedColumnOrder = []

285 columnFilters = []

299 columnFilters = []

286 // Match start(): clear the text filter alongside the column-filter chips so

300 // Match start(): clear the text filter alongside the column-filter chips so

287 // a clear()ed tail isn't immediately re-hidden by a lingering search string.

301 // a clear()ed tail isn't immediately re-hidden by a lingering search string.

@@ -337,6 +351,7 @@ final class LiveTailViewModel {

337

351

338 func resetColumnConfig() {

352 func resetColumnConfig() {

339 columnOrder = columns

353 columnOrder = columns

354+ savedColumnOrder = []

340 hiddenColumns.removeAll()

355 hiddenColumns.removeAll()

341 savedHiddenColumns.removeAll()

356 savedHiddenColumns.removeAll()

342 saveColumnConfig()

357 saveColumnConfig()

@@ -350,9 +365,16 @@ final class LiveTailViewModel {

350

365

351 private func saveColumnConfig() {

366 private func saveColumnConfig() {

352 guard let stream = currentStream else { return }

367 guard let stream = currentStream else { return }

353− // Persist the full intended hidden set (including columns not yet seen

368+ // Persist the full intended hidden set and order (including columns not

354− // this session) so it survives across restarts and incremental discovery.

369+ // yet seen this session) so they survive restarts and incremental discovery.

370+ let seen = Set(columnOrder)

371+ let order = QueryViewModel.mergedColumnOrder(

372+ columnOrder,

373+ inserting: savedColumnOrder.filter { !seen.contains($0) },

374+ reference: savedColumnOrder

375+ )

376+ savedColumnOrder = order

355− let config = QueryViewModel.ColumnConfiguration(order: columnOrder, hidden: savedHiddenColumns)

377+ let config = QueryViewModel.ColumnConfiguration(order: order, hidden: savedHiddenColumns)

356 if let data = try? JSONEncoder().encode(config) {

378 if let data = try? JSONEncoder().encode(config) {

357 UserDefaults.standard.set(data, forKey: Self.columnConfigKey(for: stream))

379 UserDefaults.standard.set(data, forKey: Self.columnConfigKey(for: stream))

358 }

380 }

@@ -394,10 +416,12 @@ final class LiveTailViewModel {

394 merged.append(col)

416 merged.append(col)

395 }

417 }

396 columnOrder = merged

418 columnOrder = merged

419+ savedColumnOrder = config.order

397 savedHiddenColumns = config.hidden

420 savedHiddenColumns = config.hidden

398 hiddenColumns = config.hidden.intersection(extractedSet)

421 hiddenColumns = config.hidden.intersection(extractedSet)

399 } else {

422 } else {

400 columnOrder = extracted

423 columnOrder = extracted

424+ savedColumnOrder = []

401 savedHiddenColumns = []

425 savedHiddenColumns = []

402 hiddenColumns = []

426 hiddenColumns = []

403 }

427 }

@@ -410,7 +434,7 @@ final class LiveTailViewModel {

410 }

434 }

411 if !newCols.isEmpty {

435 if !newCols.isEmpty {

412 columns.append(contentsOf: newCols)

436 columns.append(contentsOf: newCols)

413− columnOrder.append(contentsOf: newCols)

437+ columnOrder = QueryViewModel.mergedColumnOrder(columnOrder, inserting: newCols, reference: savedColumnOrder)

414 // Re-hide any newly appearing column that was hidden in a prior

438 // Re-hide any newly appearing column that was hidden in a prior

415 // session but hadn't been seen yet this run.

439 // session but hadn't been seen yet this run.

416 hiddenColumns.formUnion(newCols.filter { savedHiddenColumns.contains($0) })

440 hiddenColumns.formUnion(newCols.filter { savedHiddenColumns.contains($0) })

@@ -468,22 +492,27 @@ final class LiveTailViewModel {

468 // Process records off the main actor using a snapshot of seen fingerprints

492 // Process records off the main actor using a snapshot of seen fingerprints

469 let seenSnapshot = seenFingerprints

493 let seenSnapshot = seenFingerprints

470 let pollTime = now

494 let pollTime = now

471− let candidateEntries = await Task.detached(priority: .userInitiated) {

495+ let (candidateEntries, batchCounts) = await Task.detached(priority: .userInitiated) {

472− var results: [LiveTailEntry] = []

496+ var results: [(entry: LiveTailEntry, occurrence: Int)] = []

497+ var counts: [String: Int] = [:]

473 for record in records {

498 for record in records {

474 let fp = Self.fingerprint(for: record)

499 let fp = Self.fingerprint(for: record)

500+ let occurrence = (counts[fp] ?? 0) + 1

501+ counts[fp] = occurrence

502+ // The first `seen` occurrences of this content were delivered

503+ // by earlier polls; only later ones are new.

475− guard !seenSnapshot.contains(fp) else { continue }

504+ guard occurrence > (seenSnapshot[fp] ?? 0) else { continue }

476 let timestamp = Self.parseTimestamp(from: record) ?? pollTime

505 let timestamp = Self.parseTimestamp(from: record) ?? pollTime

477 let summary = Self.buildSummary(from: record)

506 let summary = Self.buildSummary(from: record)

478− results.append(LiveTailEntry(

507+ results.append((LiveTailEntry(

479 timestamp: timestamp,

508 timestamp: timestamp,

480 record: record,

509 record: record,

481 displayTimestamp: Self.displayFormatter.string(from: timestamp),

510 displayTimestamp: Self.displayFormatter.string(from: timestamp),

482 summary: summary,

511 summary: summary,

483 fingerprint: fp

512 fingerprint: fp

484− ))

513+ ), occurrence))

485 }

514 }

486− return results

515+ return (results, counts)

487 }.value

516 }.value

488

517

489 // Re-check after the detached processing hop: a stop/clear could have

518 // Re-check after the detached processing hop: a stop/clear could have

@@ -492,11 +521,13 @@ final class LiveTailViewModel {

492

521

493 // Authoritative dedup on main actor

522 // Authoritative dedup on main actor

494 var newEntries: [LiveTailEntry] = []

523 var newEntries: [LiveTailEntry] = []

495− for entry in candidateEntries {

524+ for candidate in candidateEntries

496− if seenFingerprints.insert(entry.fingerprint).inserted {

525+ where candidate.occurrence > (seenFingerprints[candidate.entry.fingerprint] ?? 0) {

526+ newEntries.append(candidate.entry)

527+ }

528+ for (fp, count) in batchCounts where count > (seenFingerprints[fp] ?? 0) {

497− seenFingerprintOrder.append(entry.fingerprint)

529+ if seenFingerprints[fp] == nil { seenFingerprintOrder.append(fp) }

498− newEntries.append(entry)

530+ seenFingerprints[fp] = count

499− }

500 }

531 }

501

532

502 if !newEntries.isEmpty {

533 if !newEntries.isEmpty {

@@ -518,23 +549,27 @@ final class LiveTailViewModel {

518 // would let it pass the dedup check and be re-appended as a

549 // would let it pass the dedup check and be re-appended as a

519 // duplicate. Keep the full seen-set so dedup stays correct;

550 // duplicate. Keep the full seen-set so dedup stays correct;

520 // it is bounded below to avoid unbounded growth.

551 // it is bounded below to avoid unbounded growth.

552+ }

553+

554+ // Bound the seen-set on every poll, not only when the buffer

555+ // overflows: clear() keeps fingerprints while emptying `entries`,

556+ // so the set can outgrow the cap without the buffer ever filling.

521− if seenFingerprints.count > maxSeenFingerprints {

557+ if seenFingerprints.count > maxSeenFingerprints {

522− // Evict the OLDEST surplus fingerprints by insertion order,

558+ // Evict the OLDEST surplus fingerprints by insertion order,

523− // not by what's currently buffered. Collapsing to only the

559+ // not by what's currently buffered. Collapsing to only the

524− // buffered set would drop recently-evicted fingerprints that

560+ // buffered set would drop recently-evicted fingerprints that

525− // still fall inside the next poll's ~90 s look-back window,

561+ // still fall inside the next poll's ~90 s look-back window,

526− // re-admitting those records as duplicates. Keeping the

562+ // re-admitting those records as duplicates. Keeping the

527− // most-recently-seen tail preserves a full window of evicted

563+ // most-recently-seen tail preserves a full window of evicted

528− // fingerprints. Buffered fingerprints are always retained so

564+ // fingerprints. Buffered fingerprints are always retained so

529− // a re-fetch of a buffered record stays deduplicated.

565+ // a re-fetch of a buffered record stays deduplicated.

530− let overflow = seenFingerprints.count - maxSeenFingerprints

566+ let overflow = seenFingerprints.count - maxSeenFingerprints

531− let buffered = Set(entries.map { $0.fingerprint })

567+ let buffered = Set(entries.map { $0.fingerprint })

532− let toDrop = Set(seenFingerprintOrder.prefix(overflow))

568+ let toDrop = Set(seenFingerprintOrder.prefix(overflow))

533− .subtracting(buffered)

569+ .subtracting(buffered)

534− if !toDrop.isEmpty {

570+ if !toDrop.isEmpty {

535− seenFingerprints.subtract(toDrop)

571+ for fp in toDrop { seenFingerprints.removeValue(forKey: fp) }

536− seenFingerprintOrder.removeAll { toDrop.contains($0) }

572+ seenFingerprintOrder.removeAll { toDrop.contains($0) }

537− }

538 }

573 }

539 }

574 }

540

575

@@ -689,10 +724,19 @@ final class LiveTailViewModel {

689 return nil

724 return nil

690 }

725 }

691 if case .string(let str) = value {

726 if case .string(let str) = value {

692− if let date = isoFractional.date(from: str) {

727+ if let date = isoFractional.date(from: str) ?? isoBasic.date(from: str) {

693 return date

728 return date

694 }

729 }

730+ // Parseable stores `p_timestamp` as an Arrow timestamp without a

731+ // timezone, and the query API serializes it as a naive UTC string with

732+ // no offset (e.g. "2024-01-11T09:08:34.290"). `.withInternetDateTime`

733+ // requires a zone designator, so such values never parsed: every entry

734+ // fell back to the poll time, which also scrambled the tail's order

735+ // (each poll's DESC-ordered batch was appended newest-first). Retry as

736+ // UTC; a string that already carries an offset won't parse with an

737+ // extra "Z", so this can't reinterpret zoned timestamps.

738+ let utc = str + "Z"

695− return isoBasic.date(from: str)

739+ return isoFractional.date(from: utc) ?? isoBasic.date(from: utc)

696 }

740 }

697 return nil

741 return nil

698 }

742 }

ParseableViewer/Views/LiveTailView.swift+12−2

@@ -123,7 +123,10 @@ struct LiveTailView: View {

123 Image(systemName: "trash")

124 }

125 .help("Clear entries")

126+ // ⌘K, not ⌘⌫: command-key equivalents are dispatched before the

127+ // focused text field sees the keystroke, so ⌘⌫ (delete to start

128+ // of line) in the Filter field would wipe the whole buffer.

126− .keyboardShortcut(.delete, modifiers: .command)

129+ .keyboardShortcut("k", modifiers: .command)

127 .accessibilityLabel("Clear all entries")

130 .accessibilityLabel("Clear all entries")

128 }

131 }

129 .padding(8)

132 .padding(8)

@@ -284,7 +287,14 @@ struct LiveTailView: View {

284 columnWidths = [:]

287 columnWidths = [:]

285 cachedSorted = []

288 cachedSorted = []

286 }

289 }

287− .onChange(of: viewModel.visibleColumns) { _, _ in

290+ .onChange(of: viewModel.visibleColumns) { _, newColumns in

291+ // Hiding the sort column leaves no header to show or change the sort,

292+ // while rows stay ordered by the invisible value (and auto-scroll stays

293+ // off). Drop it like LogTableView does; the sortColumn change rebuilds.

294+ if let sortColumn, !newColumns.contains(sortColumn) {

295+ self.sortColumn = nil

296+ return

297+ }

288 // Showing/hiding a column changes the severity column set, so rebuild

298 // Showing/hiding a column changes the severity column set, so rebuild

289 // (which also fills in widths for any newly visible columns). A

299 // (which also fills in widths for any newly visible columns). A

290 // width-only update would leave row severity tinting stale until the

300 // width-only update would leave row severity tinting stale until the

ParseableViewerTests/LiveTailViewModelTests.swift+25

@@ -461,4 +461,29 @@ final class LiveTailViewModelTests: XCTestCase {

461 let filter = LiveTailViewModel.ColumnFilter(column: "field", value: nil, exclude: false)

462 XCTAssertEqual(filter.displayLabel, "field = null")

463 }

464+

465+ // MARK: - Zone-less timestamps / saved column order

466+

467+ func testParseTimestampTreatsZonelessAsUTC() {

468+ // Parseable serializes p_timestamp without a zone designator.

469+ let naive = LiveTailViewModel.parseTimestamp(from: ["p_timestamp": .string("2024-06-15T10:30:45.123")])

470+ let zoned = LiveTailViewModel.parseTimestamp(from: ["p_timestamp": .string("2024-06-15T10:30:45.123Z")])

471+ XCTAssertNotNil(naive)

472+ XCTAssertEqual(naive, zoned)

473+ let naiveWhole = LiveTailViewModel.parseTimestamp(from: ["p_timestamp": .string("2024-06-15T10:30:45")])

474+ XCTAssertEqual(naiveWhole, LiveTailViewModel.parseTimestamp(from: ["p_timestamp": .string("2024-06-15T10:30:45Z")]))

475+ }

476+

477+ func testMergedColumnOrderRestoresSavedPosition() {

478+ let merged = QueryViewModel.mergedColumnOrder(

479+ ["p_timestamp", "level", "message"],

480+ inserting: ["trace_id"],

481+ reference: ["p_timestamp", "trace_id", "level", "message"]

482+ )

483+ XCTAssertEqual(merged, ["p_timestamp", "trace_id", "level", "message"])

484+ }

485+

486+ func testMergedColumnOrderAppendsColumnsWithoutSavedPosition() {

487+ XCTAssertEqual(QueryViewModel.mergedColumnOrder(["a"], inserting: ["b"], reference: []), ["a", "b"])

488+ }

464}

489}

README.md+1−1

@@ -143,7 +143,7 @@ ParseableViewer/

143| `Cmd+R` | Refresh streams |

144| `Cmd+Enter` | Run query |

145| `Cmd+P` | Pause/resume live tail |

146−| `Cmd+Delete` | Clear live tail entries |

146+| `Cmd+K` | Clear live tail entries |

147| `Cmd+Ctrl+S` | Toggle sidebar |

148

149## License