2c66fc1 · j · git.iko.soy

git.iko.soy/j

Make commit file lists lazy; answer conflict/empty from tree ids

Two changes that cut large-repo commands roughly in half again: Lazy files (Value::Thunk). Building the Repo value materialized every commit's file list even when nothing looked at it — about 70% of load time on a 1625-commit repository. A commit's files field is now a thunk that runs tree_to_files on first access. Value::field forces thunks transparently (every files access goes through field), value_eq forces them at the boundary, and the few pattern matches that can see a Value directly handle it. Most commits' files are now never built; the focus's are forced on demand by status/changed/diff/review and by persistence. O(1) conflict and emptiness (Backend::has_conflict / is_empty). The tree renderer derived both by walking each commit's file list; jj can answer from tree ids alone (a commit is conflicted iff its tree id is an unresolved merge, empty iff it equals the first parent's). The renderer uses these and only materializes a commit's files when the diff or the files data column needs them. The in-memory test backend reports "unknown", preserving the file-walking fallback. On the same repository: load 3.8s -> 1.4s, tree 5.3s -> 4.1s, status 8.2s -> 5.5s. Co-authored-by: vix <290354907+vix-agent@users.noreply.github.com>

commit2c66fc1ccd986ad8dc96f284a8f24e2aaaa39650 authoriko <mail@iko.soy> dateSat 19 Sep 2026, 12:03 (3 weeks ago) parent9585b3f

5 files changed+150−20

compare with master

unifiedsplit

src/domain.rs+10 src/jj.rs+33−7 src/render.rs+30−12 src/show.rs+4 src/value.rs+73−1

src/domain.rs+10

@@ -42,6 +42,16 @@ pub trait Backend {

4242 fn meta(&self, id: &str) -> Result<MetaInfo, Crash>;

4343 /// True if the change id denotes a merge commit (2+ parents).

4444 fn is_merge(&self, id: &str) -> bool;

45+ /// True if the commit's tree contains a conflict, if the backend can say

46+ /// without reading files (jj: O(1)); None means "unknown, compare files".

47+ fn has_conflict(&self, _id: &str) -> Option<bool> {

48+ None

49+ }

50+ /// True if the commit's tree equals its first parent's tree, if the

51+ /// backend can say without reading files; None means "compare files".

52+ fn is_empty(&self, _id: &str) -> Option<bool> {

53+ None

54+ }

4555 /// All ancestors (inclusive) of the given change ids.

4656 fn ancestors_closed(&self, ids: &BTreeSet<String>) -> BTreeSet<String>;

4757

src/jj.rs+33−7

@@ -1763,6 +1763,22 @@ impl Backend for JjBackend {

17631763 .unwrap_or(false)

17641764 }

17651765

1766+ fn has_conflict(&self, id: &str) -> Option<bool> {

1767+ let vis = self.inner.visible.lock().unwrap().clone();

1768+ // O(1): a commit is conflicted iff its tree id is an unresolved merge

1769+ vis.and_then(|v| v.commits.get(id).map(|r| r.commit.has_conflict()))

1770+ }

1771+

1772+ fn is_empty(&self, id: &str) -> Option<bool> {

1773+ let vis = self.inner.visible.lock().unwrap().clone();

1774+ // O(1): empty iff the tree id equals the first parent's tree id

1775+ vis.and_then(|v| {

1776+ let rec = v.commits.get(id)?;

1777+ let parent = v.commits.get(&rec.first_parent)?;

1778+ Some(rec.commit.tree_ids() == parent.commit.tree_ids())

1779+ })

1780+ }

1781+

17661782 fn ancestors_closed(&self, ids: &BTreeSet<String>) -> BTreeSet<String> {

17671783 let vis = self.inner.visible.lock().unwrap().clone();

17681784 let Some(vis) = vis else {

@@ -1941,8 +1957,8 @@ fn to_repo_path(path: &[String]) -> Result<RepoPathBuf, Crash> {

19411957/// files are unchanged between commits, so inflating each blob once (instead

19421958/// of once per commit that references it) keeps loading linear in the

19431959/// repository's unique data rather than commits × files

1944−#[derive(Default)]

1945−struct BlobCache(RefCell<HashMap<FileId, Rc<Vec<u8>>>>);

1960+#[derive(Clone, Default)]

1961+struct BlobCache(Rc<RefCell<HashMap<FileId, Rc<Vec<u8>>>>>);

19461962

19471963impl BlobCache {

19481964 fn get(&self, id: &FileId) -> Option<Rc<Vec<u8>>> {

@@ -1956,8 +1972,8 @@ impl BlobCache {

19561972/// whole file *entries* ({path, content}) keyed by (path, blob identity),

19571973/// shared across the whole repo load: an unchanged file is the same Value in

19581974/// every commit, so it is built once and cloned (an Rc bump) thereafter

1959−#[derive(Default)]

1960−struct EntryCache(RefCell<HashMap<String, Value>>);

1975+#[derive(Clone, Default)]

1976+struct EntryCache(Rc<RefCell<HashMap<String, Value>>>);

19611977

19621978impl EntryCache {

19631979 fn get(&self, key: &str) -> Option<Value> {

@@ -2191,8 +2207,18 @@ fn build_subtree(

21912207 cache: &BlobCache,

21922208 entries: &EntryCache,

21932209) -> Result<Value, OpenError> {

2194− let files =

2195− block_on(tree_to_files(store, &rec.commit.tree(), cache, entries)).map_err(|c| (2, c.msg))?;

2210+ // the files list is lazy (§7.2): most commits are never inspected, so

2211+ // their file entries are materialized only on first `field("files")`

2212+ let files_thunk = {

2213+ let store = store.clone();

2214+ let tree = rec.commit.tree();

2215+ let cache = cache.clone();

2216+ let entries = entries.clone();

2217+ Value::Thunk(Rc::new(crate::value::ThunkVal::new(move || {

2218+ let files = block_on(tree_to_files(&store, &tree, &cache, &entries))?;

2219+ Ok(Value::list(files))

2220+ })))

2221+ };

21962222 let labels: Vec<Value> = vis

21972223 .labels

21982224 .get(&rec.change_id)

@@ -2202,7 +2228,7 @@ fn build_subtree(

22022228 .map(Value::text)

22032229 .collect();

22042230 let commit_v = Value::record(&[

2205− ("files", Value::list(files)),

2231+ ("files", files_thunk),

22062232 ("id", Value::Id(Rc::new(rec.change_id.clone()))),

22072233 ("labels", Value::list(labels)),

22082234 ("message", Value::text(rec.commit.description())),

src/render.rs+30−12

@@ -191,7 +191,9 @@ fn display_id(interp: &Interp, id: &str, pal: &Palette) -> String {

191191}

192192

193193fn display_line(interp: &Interp, v: &Value, pal: &Palette) -> Result<String, Crash> {

194+ let v = &v.forced()?;

194195 match v {

196+ Value::Thunk(_) => unreachable!("forced never returns a thunk"),

195197 Value::Int(n) => Ok(n.to_string()),

196198 Value::Bool(b) => Ok(b.to_string()),

197199 Value::Text(t) => {

@@ -463,7 +465,9 @@ fn display_block(

463465 out: &mut String,

464466) -> Result<(), Crash> {

465467 let pad = " ".repeat(indent);

468+ let v = &v.forced()?;

466469 match v {

470+ Value::Thunk(_) => unreachable!("forced never returns a thunk"),

467471 Value::Text(t) => {

468472 out.push_str(t);

469473 return Ok(());

@@ -1016,11 +1020,13 @@ fn build_info(

10161020 .iter()

10171021 .map(|l| l.as_text().map(|s| s.to_string()))

10181022 .collect::<Result<_, _>>()?;

1019− let conflict = has_conflict(commit)?;

1020− let files = commit.field("files")?;

1021− let nfiles = files.as_list()?.len();

1022− // `empty` (no change against the parent) is needed at every detail level;

1023− // compute it by id-based snapshot equality, which never forces blob bytes.

1023+ // conflict and empty are O(1) backend queries when the backend can answer

1024+ // them from tree ids (jj), without touching the file list at all; the

1025+ // in-memory backend cannot, so fall back to walking the files

1026+ let conflict = match interp.backend.has_conflict(&id) {

1027+ Some(c) => c,

1028+ None => has_conflict(commit)?,

1029+ };

10241030 // `size` (a line count) and `detail_marks` are only rendered for detail ≥

10251031 // 2 and for the focus and its neighbours, so skip the per-commit line

10261032 // counting elsewhere — it dominates render time on large histories.

@@ -1030,16 +1036,29 @@ fn build_info(

10301036 matches!(c.field("root").and_then(|r| r.field("id")), Ok(Value::Id(i)) if *i == focus_id)

10311037 });

10321038 let want_diff = opts.detail >= 2 || is_focus_pre || child_is_focus || parent_is_focus;

1039+ let backend_empty = interp.backend.is_empty(&id);

1040+ // the files list is only materialized when something needs it: the diff

1041+ // (detail/focus), the files data column, or the emptiness fallback

1042+ let need_files = want_diff || opts.files || backend_empty.is_none();

1043+ let files = if need_files {

1044+ Some(commit.field("files")?)

1045+ } else {

1046+ None

1047+ };

1048+ let nfiles = files.as_ref().map(|f| f.as_list().map(|l| l.len())).transpose()?.unwrap_or(0);

10331049 let (empty, size, detail_marks) = match parent_files {

10341050 Some(pf) => {

1035− // empty ⟺ the snapshot equals the parent's (id-based, no forcing)

1036− let empty = crate::value::value_eq(&files, pf)?;

1051+ let empty = match backend_empty {

1052+ Some(e) => e,

1053+ None => crate::value::value_eq(files.as_ref().expect("files loaded"), pf)?,

1054+ };

10371055 if want_diff {

1056+ let files = files.as_ref().expect("files loaded");

10381057 let change = Value::record(&[("from", pf.clone()), ("to", files.clone())]);

10391058 let marks = touched_paths(&change)?;

10401059 // size: lines added+removed against the parent

10411060 let from_map = snapshot_map_of(pf)?;

1042− let to_map = snapshot_map_of(&files)?;

1061+ let to_map = snapshot_map_of(files)?;

10431062 let mut lines = 0usize;

10441063 for (p, _, _) in &marks {

10451064 let key: Vec<String> = p.split('/').map(|s| s.to_string()).collect();

@@ -1065,7 +1084,7 @@ fn build_info(

10651084 }

10661085 }

10671086 None => {

1068− let n = files.as_list()?.len();

1087+ let n = nfiles;

10691088 (n == 0 && id != ROOT_ID, None, Vec::new())

10701089 }

10711090 };

@@ -1082,13 +1101,12 @@ fn build_info(

10821101 immutable,

10831102 focus_id,

10841103 anc,

1085− Some(&files),

1104+ files.as_ref(),

10861105 with_focus,

10871106 is_ancestor_of_focus,

10881107 is_focus_pre,

10891108 )?);

1090− }

1091− let immutable_flag = immutable.contains(&id);

1109+ } let immutable_flag = immutable.contains(&id);

10921110 Ok(CommitInfo {

10931111 id,

10941112 message,

src/show.rs+4

@@ -14,6 +14,10 @@ pub fn show(interp: &Interp, v: &Value) -> String {

1414

1515fn render(interp: &Interp, v: &Value) -> String {

1616 match v {

17+ Value::Thunk(t) => match t.force() {

18+ Ok(v) => render(interp, &v),

19+ Err(_) => "<lazy>".to_string(),

20+ },

1721 Value::Int(n) => {

1822 if n.sign() == num_bigint::Sign::Minus {

1923 format!("(0 - {})", n.magnitude())

src/value.rs+73−1

@@ -18,6 +18,66 @@ pub enum Value {

1818 Id(Rc<String>),

1919 Blob(Rc<BlobVal>),

2020 Shape(Rc<ShapeVal>),

21+ /// a value computed on first use (§2): currently used for a commit's

22+ /// `files` list, so loading a repo does not materialize every commit's

23+ /// file list. Forced transparently by `Value::field` and `value_eq`.

24+ Thunk(Rc<ThunkVal>),

25+}

26+

27+pub struct ThunkVal {

28+ state: std::cell::RefCell<ThunkState>,

29+}

30+

31+enum ThunkState {

32+ Pending(Box<dyn FnOnce() -> Result<Value, Crash>>),

33+ Ready(Value),

34+}

35+

36+impl std::fmt::Debug for ThunkVal {

37+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {

38+ match &*self.state.borrow() {

39+ ThunkState::Pending(_) => write!(f, "Thunk(pending)"),

40+ ThunkState::Ready(v) => write!(f, "Thunk({:?})", v),

41+ }

42+ }

43+}

44+

45+impl ThunkVal {

46+ pub fn new(f: impl FnOnce() -> Result<Value, Crash> + 'static) -> Self {

47+ ThunkVal {

48+ state: std::cell::RefCell::new(ThunkState::Pending(Box::new(f))),

49+ }

50+ }

51+

52+ /// the value, computing it on first call and memoizing

53+ pub fn force(&self) -> Result<Value, Crash> {

54+ if let ThunkState::Ready(v) = &*self.state.borrow() {

55+ return Ok(v.clone());

56+ }

57+ let thunk = {

58+ let mut st = self.state.borrow_mut();

59+ match std::mem::replace(&mut *st, ThunkState::Ready(Value::Bool(false))) {

60+ ThunkState::Pending(t) => t,

61+ ThunkState::Ready(v) => {

62+ *st = ThunkState::Ready(v.clone());

63+ return Ok(v);

64+ }

65+ }

66+ };

67+ let v = thunk()?;

68+ *self.state.borrow_mut() = ThunkState::Ready(v.clone());

69+ Ok(v)

70+ }

71+}

72+

73+impl Value {

74+ /// force a thunk; every other value is returned unchanged

75+ pub fn forced(&self) -> Result<Value, Crash> {

76+ match self {

77+ Value::Thunk(t) => t.force(),

78+ _ => Ok(self.clone()),

79+ }

80+ }

2181}

2282

2383#[derive(Clone, Debug, PartialEq, Eq)]

@@ -266,6 +326,9 @@ impl Value {

266326 Value::Id(_) => "Id",

267327 Value::Blob(_) => "Blob",

268328 Value::Shape(_) => "Shape",

329+ // a thunk stands in for its eventual value (currently a list of

330+ // file entries); kind_name is used in error messages

331+ Value::Thunk(_) => "list",

269332 }

270333 }

271334

@@ -282,7 +345,8 @@ impl Value {

282345 Value::Record(m) => m

283346 .get(name)

284347 .cloned()

285− .ok_or_else(|| Crash::new(format!("record has no field `{}`", name))),

348+ .ok_or_else(|| Crash::new(format!("record has no field `{}`", name)))

349+ .and_then(|v| v.forced()),

286350 _ => Err(Crash::new(format!(

287351 "cannot select field `{}` from a {}",

288352 name,

@@ -533,6 +597,13 @@ impl Env {

533597// ----------------------------------------------------------------------

534598

535599pub fn value_eq(a: &Value, b: &Value) -> Result<bool, Crash> {

600+ // force thunks at the boundary so callers never see them

601+ if let Value::Thunk(_) = a {

602+ return value_eq(&a.forced()?, b);

603+ }

604+ if let Value::Thunk(_) = b {

605+ return value_eq(a, &b.forced()?);

606+ }

536607 Ok(match (a, b) {

537608 (Value::Int(x), Value::Int(y)) => x == y,

538609 (Value::Text(x), Value::Text(y)) => x == y,

@@ -596,6 +667,7 @@ impl std::fmt::Debug for Value {

596667 Value::Fun(_) => write!(f, "Fun(..)"),

597668 Value::Blob(b) => write!(f, "Blob({:?})", b),

598669 Value::Shape(s) => write!(f, "Shape({})", s.name),

670+ Value::Thunk(t) => write!(f, "{:?}", t),

599671 }

600672 }

601673}