diff --git a/eden/scm/edenscmnative/bindings/modules/pycopytrace/src/lib.rs b/eden/scm/edenscmnative/bindings/modules/pycopytrace/src/lib.rs index 4420a57227..ce6eca308d 100644 --- a/eden/scm/edenscmnative/bindings/modules/pycopytrace/src/lib.rs +++ b/eden/scm/edenscmnative/bindings/modules/pycopytrace/src/lib.rs @@ -102,7 +102,7 @@ py_class!(pub class dagcopytrace |py| { ) -> PyResult> { let old_tree = old_tree.get_underlying(py); let new_tree = new_tree.get_underlying(py); - let map = self.inner(py).find_renames(&old_tree.read(), &new_tree.read()).map_pyerr(py)?; + let map = block_on(self.inner(py).find_renames(&old_tree.read(), &new_tree.read())).map_pyerr(py)?; let map = map .into_iter() .map(|(k, v)| (k.to_string(), v.to_string())) diff --git a/eden/scm/edenscmnative/bindings/modules/pyrevisionstore/src/pythonfilescmstore.rs b/eden/scm/edenscmnative/bindings/modules/pyrevisionstore/src/pythonfilescmstore.rs index e09f6b0567..734c8e5bd2 100644 --- a/eden/scm/edenscmnative/bindings/modules/pyrevisionstore/src/pythonfilescmstore.rs +++ b/eden/scm/edenscmnative/bindings/modules/pyrevisionstore/src/pythonfilescmstore.rs @@ -63,10 +63,10 @@ impl ReadFileContents for PythonFileScmStore { futures::stream::iter(contents.into_iter()).boxed() } - fn read_rename_metadata( + async fn read_rename_metadata( &self, _keys: Vec, - ) -> Result)>, Self::Error> { - Ok(vec![]) + ) -> BoxStream), Self::Error>> { + futures::stream::empty().boxed() } } diff --git a/eden/scm/lib/checkout/src/lib.rs b/eden/scm/lib/checkout/src/lib.rs index cc4da8f6ee..28aaba2234 100644 --- a/eden/scm/lib/checkout/src/lib.rs +++ b/eden/scm/lib/checkout/src/lib.rs @@ -1050,11 +1050,11 @@ mod test { .boxed() } - fn read_rename_metadata( + async fn read_rename_metadata( &self, _keys: Vec, - ) -> Result)>, Self::Error> { - Ok(vec![]) + ) -> BoxStream), Self::Error>> { + stream::empty().boxed() } } diff --git a/eden/scm/lib/copytrace/src/copy_trace.rs b/eden/scm/lib/copytrace/src/copy_trace.rs index 9e130b86b4..d07ebd5ca3 100644 --- a/eden/scm/lib/copytrace/src/copy_trace.rs +++ b/eden/scm/lib/copytrace/src/copy_trace.rs @@ -49,7 +49,7 @@ pub trait CopyTrace { /// TODO: move this method into a separate trait. Practically the graph log and /// the find_renames can use different impls independently and form different /// combinations. - fn find_renames( + async fn find_renames( &self, old_tree: &TreeManifest, new_tree: &TreeManifest, diff --git a/eden/scm/lib/copytrace/src/dag_copy_trace.rs b/eden/scm/lib/copytrace/src/dag_copy_trace.rs index 525a6ca42d..f94613b282 100644 --- a/eden/scm/lib/copytrace/src/dag_copy_trace.rs +++ b/eden/scm/lib/copytrace/src/dag_copy_trace.rs @@ -17,6 +17,7 @@ use manifest_tree::TreeManifest; use manifest_tree::TreeStore; use pathhistory::RenameTracer; use pathmatcher::AlwaysMatcher; +use storemodel::futures::StreamExt; use storemodel::ReadFileContents; use storemodel::ReadRootTreeIds; use types::HgId; @@ -58,14 +59,21 @@ impl DagCopyTrace { Ok(dag_copy_trace) } - fn read_renamed_metadata(&self, keys: Vec) -> Result> { + async fn read_renamed_metadata( + &self, + keys: Vec, + ) -> Result> { // TODO: add metrics for the size of the result - let renames = self.file_reader.read_rename_metadata(keys)?; - let map: HashMap<_, _> = renames - .into_iter() - .filter(|(_, v)| v.is_some()) - .map(|(key, rename_from_key)| (key.path, rename_from_key.unwrap().path)) - .collect(); + let mut renames = self.file_reader.read_rename_metadata(keys).await; + + let mut map: HashMap = HashMap::new(); + while let Some(rename) = renames.next().await { + let (key, rename_from_key) = rename?; + if let Some(rename_from_key) = rename_from_key { + map.insert(key.path, rename_from_key.path); + } + } + Ok(map) } @@ -130,7 +138,7 @@ impl DagCopyTrace { // For simplicity, we only check p1. let p1 = &parents[0]; let (old_manifest, new_manifest) = self.vertex_to_tree_manifest(p1, &commit).await?; - let renames = self.find_renames(&old_manifest, &new_manifest)?; + let renames = self.find_renames(&old_manifest, &new_manifest).await?; let (renames, next_commit) = match direction { SearchDirection::Backward => (renames, p1.clone()), SearchDirection::Forward => { @@ -253,7 +261,7 @@ impl CopyTrace for DagCopyTrace { } } - fn find_renames( + async fn find_renames( &self, old_tree: &TreeManifest, new_tree: &TreeManifest, @@ -262,23 +270,29 @@ impl CopyTrace for DagCopyTrace { // * [x] parse file header and get mv info // * support content similarity for sl repo // * support content similarity for git repo - let matcher = AlwaysMatcher::new(); - let diff = Diff::new(old_tree, new_tree, &matcher)?; let mut new_files = Vec::new(); - for entry in diff { - let entry = entry?; - if let DiffType::RightOnly(file_metadata) = entry.diff_type { - let path = entry.path; - let key = Key { - path, - hgid: file_metadata.hgid, - }; - new_files.push(key); + { + // this block is for dropping matcher and diff at the end of the block, + // otherwise the compiler compilains variable might be used across 'await' + + let matcher = AlwaysMatcher::new(); + let diff = Diff::new(old_tree, new_tree, &matcher)?; + for entry in diff { + let entry = entry?; + + if let DiffType::RightOnly(file_metadata) = entry.diff_type { + let path = entry.path; + let key = Key { + path, + hgid: file_metadata.hgid, + }; + new_files.push(key); + } } } - self.read_renamed_metadata(new_files) + self.read_renamed_metadata(new_files).await } } diff --git a/eden/scm/lib/eagerepo/src/trait_impls.rs b/eden/scm/lib/eagerepo/src/trait_impls.rs index df8cff08ae..22443d036b 100644 --- a/eden/scm/lib/eagerepo/src/trait_impls.rs +++ b/eden/scm/lib/eagerepo/src/trait_impls.rs @@ -45,17 +45,19 @@ impl ReadFileContents for EagerRepoStore { futures::stream::iter(iter).boxed() } - fn read_rename_metadata(&self, keys: Vec) -> Result)>, Self::Error> { - keys.into_iter() - .map(|k| { - let id = k.hgid; - let copy_from = match self.get_content(id)? { - Some(data) => strip_metadata(&data)?.1, - None => anyhow::bail!("no such file: {:?}", &k), - }; - Ok((k, copy_from)) - }) - .collect() + async fn read_rename_metadata( + &self, + keys: Vec, + ) -> BoxStream), Self::Error>> { + let iter = keys.into_iter().map(|k| { + let id = k.hgid; + let copy_from = match self.get_content(id)? { + Some(data) => strip_metadata(&data)?.1, + None => anyhow::bail!("no such file: {:?}", &k), + }; + Ok((k, copy_from)) + }); + futures::stream::iter(iter).boxed() } } diff --git a/eden/scm/lib/gitstore/src/trait_impls.rs b/eden/scm/lib/gitstore/src/trait_impls.rs index fbe2abdcdf..7fcefdb4a7 100644 --- a/eden/scm/lib/gitstore/src/trait_impls.rs +++ b/eden/scm/lib/gitstore/src/trait_impls.rs @@ -37,11 +37,11 @@ impl ReadFileContents for GitStore { futures::stream::iter(iter).boxed() } - fn read_rename_metadata( + async fn read_rename_metadata( &self, _keys: Vec, - ) -> Result)>, Self::Error> { - Ok(vec![]) + ) -> BoxStream), Self::Error>> { + futures::stream::empty().boxed() } } diff --git a/eden/scm/lib/revisionstore/src/trait_impls.rs b/eden/scm/lib/revisionstore/src/trait_impls.rs index c4dd7a4e49..134162e721 100644 --- a/eden/scm/lib/revisionstore/src/trait_impls.rs +++ b/eden/scm/lib/revisionstore/src/trait_impls.rs @@ -12,7 +12,6 @@ use std::sync::Arc; use anyhow::anyhow; use anyhow::format_err; use anyhow::Result; -use async_runtime::block_on; use async_trait::async_trait; use futures::stream; use futures::stream::BoxStream; @@ -54,16 +53,16 @@ where .boxed() } - fn read_rename_metadata(&self, keys: Vec) -> Result)>> { - let items = block_on( - stream_data_from_remote_data_store(self.0.clone(), keys) - .map(|result| match result { - Ok((_data, key, copy_from)) => Ok((key, copy_from)), - Err(err) => Err(err), - }) - .collect::>(), - ); - items.into_iter().collect() + async fn read_rename_metadata( + &self, + keys: Vec, + ) -> BoxStream), Self::Error>> { + stream_data_from_remote_data_store(self.0.clone(), keys) + .map(|result| match result { + Ok((_data, key, copy_from)) => Ok((key, copy_from)), + Err(err) => Err(err), + }) + .boxed() } } @@ -80,16 +79,16 @@ impl ReadFileContents for ArcFileStore { .boxed() } - fn read_rename_metadata(&self, keys: Vec) -> Result)>> { - let items = block_on( - stream_data_from_scmstore(self.0.clone(), keys) - .map(|result| match result { - Ok((_data, key, copy_from)) => Ok((key, copy_from)), - Err(err) => Err(err), - }) - .collect::>(), - ); - items.into_iter().collect() + async fn read_rename_metadata( + &self, + keys: Vec, + ) -> BoxStream), Self::Error>> { + stream_data_from_scmstore(self.0.clone(), keys) + .map(|result| match result { + Ok((_data, key, copy_from)) => Ok((key, copy_from)), + Err(err) => Err(err), + }) + .boxed() } } diff --git a/eden/scm/lib/storemodel/src/lib.rs b/eden/scm/lib/storemodel/src/lib.rs index f7cc2ad4a5..4617ba426c 100644 --- a/eden/scm/lib/storemodel/src/lib.rs +++ b/eden/scm/lib/storemodel/src/lib.rs @@ -50,7 +50,10 @@ pub trait ReadFileContents { /// Read rename metadata of sepcified files. /// /// The result is a vector of (key, Option) pairs for success case. - fn read_rename_metadata(&self, keys: Vec) -> Result)>, Self::Error>; + async fn read_rename_metadata( + &self, + keys: Vec, + ) -> BoxStream), Self::Error>>; } pub trait RefreshableReadFileContents: ReadFileContents { diff --git a/eden/scm/lib/workingcopy/src/sparse.rs b/eden/scm/lib/workingcopy/src/sparse.rs index 482853cb9d..a6fb3795d9 100644 --- a/eden/scm/lib/workingcopy/src/sparse.rs +++ b/eden/scm/lib/workingcopy/src/sparse.rs @@ -537,11 +537,11 @@ inc .boxed() } - fn read_rename_metadata( + async fn read_rename_metadata( &self, _keys: Vec, - ) -> Result)>, Self::Error> { - Ok(vec![]) + ) -> BoxStream), Self::Error>> { + stream::empty().boxed() } } }