on master help sign in
Share file-entry values across commits when loading a repo

Building the Repo value constructed a fresh {path, content} record for every file of every commit — 1.4M record allocations on a 1625-commit, ~866-file repository, the largest share of load time once blob contents were lazy. Most files are unchanged between commits, so an entry is now built once and shared (an Rc clone) across all commits that reference it, keyed by (path, FileId, executable). Symlinks, directories, and conflicts are cheap or rare and are not cached. On the same repository: tree 11s -> 5.6s, status 12.5s -> 9.4s. Co-authored-by: vix <290354907+vix-agent@users.noreply.github.com>

commit9585b3f3127b643ec065e1c63f0205dba475de9c authoriko <mail@iko.soy> dateSat 19 Sep 2026, 07:44 (3 weeks ago) parent46c0a5e
1 file changed+54−13
src/jj.rs+54−13
src/jj.rs+54−13
@@ -616,6 +616,7 @@ impl JjBackend {
616616 let mut written: BTreeMap<String, CommitId> = BTreeMap::new();
617617 // shared across the whole persist walk so unchanged blobs are inflated once
618618 let cache = BlobCache::default();
619+ let entries = EntryCache::default();
619620
620621 // the root commit must be the top and unchanged (§7.5 step 4)
621622 let top = crate::repo::by_id(new, ROOT_ID)?
@@ -634,7 +635,7 @@ impl JjBackend {
634635 if root_jj.description() != msg {
635636 return Err(Crash::new("persistence: the root commit cannot be changed"));
636637 }
637− let stored_files = block_on(tree_to_files(&store, &root_jj.tree(), &cache))
638+ let stored_files = block_on(tree_to_files(&store, &root_jj.tree(), &cache, &entries))
638639 .map_err(|e| Crash::new(e.msg))?;
639640 if !crate::value::value_eq(
640641 &Value::list(stored_files),
@@ -683,7 +684,7 @@ impl JjBackend {
683684 let parent_changed = hex_of(stored_commit.parent_ids().first()) != parent_jj.hex();
684685 let files_changed = {
685686 let stored_files =
686− block_on(tree_to_files(&store, &stored_commit.tree(), &cache))?;
687+ block_on(tree_to_files(&store, &stored_commit.tree(), &cache, &entries))?;
687688 !crate::value::value_eq(
688689 &Value::list(stored_files),
689690 &Value::list(files_v.as_list()?.to_vec()),
@@ -1814,7 +1815,7 @@ async fn jj_replay(
18141815 .map_err(|e| Crash::new(format!("replay: {}", e)))?;
18151816 // conflicts stay as unresolved blobs (§7.3)
18161817 let merged = MergedTree::new(store.clone(), merged, jj_lib::conflict_labels::ConflictLabels::unlabeled());
1817− tree_to_files(store, &merged, &BlobCache::default()).await
1818+ tree_to_files(store, &merged, &BlobCache::default(), &EntryCache::default()).await
18181819}
18191820
18201821// ----------------------------------------------------------------------
@@ -1952,6 +1953,21 @@ impl BlobCache {
19521953 }
19531954}
19541955
1956+/// whole file *entries* ({path, content}) keyed by (path, blob identity),
1957+/// shared across the whole repo load: an unchanged file is the same Value in
1958+/// every commit, so it is built once and cloned (an Rc bump) thereafter
1959+#[derive(Default)]
1960+struct EntryCache(RefCell<HashMap<String, Value>>);
1961+
1962+impl EntryCache {
1963+ fn get(&self, key: &str) -> Option<Value> {
1964+ self.0.borrow().get(key).cloned()
1965+ }
1966+ fn insert(&self, key: String, entry: Value) {
1967+ self.0.borrow_mut().insert(key, entry);
1968+ }
1969+}
1970+
19551971async fn read_file_bytes(
19561972 store: &Arc<Store>,
19571973 path: &RepoPath,
@@ -2078,17 +2094,40 @@ async fn tree_to_files(
20782094 store: &Arc<Store>,
20792095 tree: &MergedTree,
20802096 cache: &BlobCache,
2097+ entries: &EntryCache,
20812098) -> Result<Vec<Value>, Crash> {
20822099 let mut out = Vec::new();
20832100 for (path, value) in tree.entries() {
20842101 let value = value.map_err(|e| Crash::new(format!("cannot read a tree entry: {}", e)))?;
2085− let comps: Vec<String> = path
2086− .components()
2087− .map(|c| c.as_internal_str().to_string())
2088− .collect();
2089− let content = merged_value_to_blob(store, &path, &value, cache).await?;
2090− let p = Value::list(comps.into_iter().map(Value::text).collect());
2091− out.push(Value::record(&[("content", content), ("path", p)]));
2102+ // cache key: the entry is fully determined by its path and the
2103+ // identity of its (resolved) content; conflicts are not cached
2104+ let key = match value.as_resolved() {
2105+ Some(Some(TreeValue::File { id, executable, .. })) => Some(format!(
2106+ "F\u{0}{}\u{0}{}\u{0}{}",
2107+ path.as_internal_file_string(),
2108+ id.hex(),
2109+ executable
2110+ )),
2111+ // symlinks and directories are cheap; conflicts are rare — neither cached
2112+ _ => None,
2113+ };
2114+ let entry = match key.as_ref().and_then(|k| entries.get(k)) {
2115+ Some(e) => e,
2116+ None => {
2117+ let comps: Vec<String> = path
2118+ .components()
2119+ .map(|c| c.as_internal_str().to_string())
2120+ .collect();
2121+ let content = merged_value_to_blob(store, &path, &value, cache).await?;
2122+ let p = Value::list(comps.into_iter().map(Value::text).collect());
2123+ let e = Value::record(&[("content", content), ("path", p)]);
2124+ if let Some(k) = key {
2125+ entries.insert(k, e.clone());
2126+ }
2127+ e
2128+ }
2129+ };
2130+ out.push(entry);
20922131 }
20932132 Ok(out)
20942133}
@@ -2131,7 +2170,8 @@ fn build_repo_value(vis: &VisibleRepo, focus_change: &str) -> Result<Value, Open
21312170 .ok_or_else(|| (2, "no root commit".to_string()))?;
21322171 let store = root_rec.commit.store().clone();
21332172 let cache = BlobCache::default();
2134− let mut subtree = build_subtree(vis, root_rec, &store, &cache)?;
2173+ let entries = EntryCache::default();
2174+ let mut subtree = build_subtree(vis, root_rec, &store, &cache, &entries)?;
21352175 sort_subtree(&mut subtree, vis);
21362176 let top = Value::record(&[
21372177 ("children", subtree.field("children").map_err(|e| (2, e.msg))?),
@@ -2149,9 +2189,10 @@ fn build_subtree(
21492189 rec: &StoredCommit,
21502190 store: &Arc<Store>,
21512191 cache: &BlobCache,
2192+ entries: &EntryCache,
21522193) -> Result<Value, OpenError> {
21532194 let files =
2154− block_on(tree_to_files(store, &rec.commit.tree(), cache)).map_err(|c| (2, c.msg))?;
2195+ block_on(tree_to_files(store, &rec.commit.tree(), cache, entries)).map_err(|c| (2, c.msg))?;
21552196 let labels: Vec<Value> = vis
21562197 .labels
21572198 .get(&rec.change_id)
@@ -2170,7 +2211,7 @@ fn build_subtree(
21702211 if let Some(kids) = vis.children.get(&rec.change_id) {
21712212 for kid in kids {
21722213 if let Some(krec) = vis.commits.get(kid) {
2173− children.push(build_subtree(vis, krec, store, cache)?);
2214+ children.push(build_subtree(vis, krec, store, cache, entries)?);
21742215 }
21752216 }
21762217 }