From 01fb3c0a770fc153560d3b0261d480c3a7d9ccc2 Mon Sep 17 00:00:00 2001 From: Xavier Deguillard Date: Fri, 13 Mar 2020 18:59:57 -0700 Subject: [PATCH] revisionstore: add a new StoreKey type Summary: This type can either be a Mercurial type key, or a content hash based key. Both the prefetch and get_missing now can handle these properly. This is essential for stores where data can either be fetched in both ways or when the data is split in 2. For LFS for instance, it is possible to have the LFS pointer (via getpackv2), but not the actual blob. In which case get_missing will simply return the content hash version of the StoreKey, to signify what it actually has missing. Reviewed By: quark-zju Differential Revision: D20445631 fbshipit-source-id: 06282f70214966cc96e805e9891f220b438c91a7 --- .../bindings/modules/pymanifest/src/lib.rs | 3 +- .../pyrevisionstore/src/datastorepyext.rs | 27 +++-- .../pyrevisionstore/src/historystorepyext.rs | 30 ++++-- .../modules/pyrevisionstore/src/lib.rs | 43 ++++---- .../pyrevisionstore/src/pythondatastore.rs | 36 ++++--- .../pyrevisionstore/src/pythonhistorystore.rs | 23 ++-- .../asyncrevisionstore/src/asyncdatastore.rs | 8 -- .../src/asynchistorystore.rs | 8 -- eden/scm/lib/backingstore/src/backingstore.rs | 2 +- .../scm/lib/revisionstore/src/contentstore.rs | 9 +- eden/scm/lib/revisionstore/src/datapack.rs | 32 +++--- eden/scm/lib/revisionstore/src/datastore.rs | 8 +- eden/scm/lib/revisionstore/src/edenapi.rs | 27 +++-- eden/scm/lib/revisionstore/src/historypack.rs | 32 +++--- .../scm/lib/revisionstore/src/historystore.rs | 8 +- .../revisionstore/src/indexedlogdatastore.rs | 16 +-- .../src/indexedloghistorystore.rs | 16 +-- eden/scm/lib/revisionstore/src/lfs.rs | 80 ++++++++------ eden/scm/lib/revisionstore/src/lib.rs | 4 +- eden/scm/lib/revisionstore/src/localstore.rs | 12 +-- eden/scm/lib/revisionstore/src/memcache.rs | 39 +++++-- .../lib/revisionstore/src/metadatastore.rs | 9 +- .../lib/revisionstore/src/multiplexstore.rs | 17 +-- .../lib/revisionstore/src/mutabledatapack.rs | 30 +++--- .../revisionstore/src/mutablehistorypack.rs | 34 +++--- eden/scm/lib/revisionstore/src/packstore.rs | 39 +++---- eden/scm/lib/revisionstore/src/repack.rs | 26 +++-- eden/scm/lib/revisionstore/src/testutil.rs | 50 +++++---- eden/scm/lib/revisionstore/src/types.rs | 102 ++++++++++++++++++ .../lib/revisionstore/src/uniondatastore.rs | 19 ++-- .../revisionstore/src/unionhistorystore.rs | 19 ++-- eden/scm/lib/revisionstore/src/unionstore.rs | 6 +- 32 files changed, 519 insertions(+), 295 deletions(-) create mode 100644 eden/scm/lib/revisionstore/src/types.rs diff --git a/eden/scm/edenscmnative/bindings/modules/pymanifest/src/lib.rs b/eden/scm/edenscmnative/bindings/modules/pymanifest/src/lib.rs index 8ae5cd46b7..d519ade261 100644 --- a/eden/scm/edenscmnative/bindings/modules/pymanifest/src/lib.rs +++ b/eden/scm/edenscmnative/bindings/modules/pymanifest/src/lib.rs @@ -20,7 +20,7 @@ use manifest_tree::TreeManifest; use pathmatcher::{AlwaysMatcher, Matcher, TreeMatcher}; use pypathmatcher::PythonMatcher; use pyrevisionstore::PythonHgIdDataStore; -use revisionstore::{HgIdDataStore, RemoteDataStore}; +use revisionstore::{HgIdDataStore, RemoteDataStore, StoreKey}; use types::{Key, Node, RepoPath, RepoPathBuf}; type Result = std::result::Result; @@ -51,6 +51,7 @@ impl manifest_tree::TreeStore for ManifestSt } fn prefetch(&self, keys: Vec) -> Result<()> { + let keys = keys.iter().map(|k| StoreKey::from(k)).collect::>(); self.underlying.prefetch(&keys) } } diff --git a/eden/scm/edenscmnative/bindings/modules/pyrevisionstore/src/datastorepyext.rs b/eden/scm/edenscmnative/bindings/modules/pyrevisionstore/src/datastorepyext.rs index bb49392cf8..f9c58b04c1 100644 --- a/eden/scm/edenscmnative/bindings/modules/pyrevisionstore/src/datastorepyext.rs +++ b/eden/scm/edenscmnative/bindings/modules/pyrevisionstore/src/datastorepyext.rs @@ -7,15 +7,15 @@ use std::convert::TryInto; -use anyhow::Result; +use anyhow::{format_err, Result}; use cpython::{ PyBytes, PyDict, PyIterator, PyList, PyObject, PyResult, PyTuple, Python, PythonObject, ToPyObject, }; use cpython_ext::{PyPath, PyPathBuf, ResultPyErrExt}; -use revisionstore::{HgIdDataStore, HgIdMutableDeltaStore, RemoteDataStore, ToKeys}; -use types::{Key, Node}; +use revisionstore::{HgIdDataStore, HgIdMutableDeltaStore, RemoteDataStore, StoreKey, ToKeys}; +use types::Node; use crate::pythonutil::{ from_base, from_delta_to_tuple, from_key, from_key_to_tuple, from_tuple_to_key, key_error, @@ -127,16 +127,23 @@ impl HgIdDataStorePyExt for T { // This lets us get a Vector of Keys without copying the strings. let keys = keys .map(|k| match k { - Ok(k) => from_tuple_to_key(py, &k), + Ok(k) => Ok(StoreKey::from(from_tuple_to_key(py, &k)?)), Err(e) => Err(e), }) - .collect::>>()?; - let missing = self.get_missing(&keys[..]).map_pyerr(py)?; + .collect::>>()?; + let missing = self.get_missing(&keys).map_pyerr(py)?; let results = PyList::new(py, &[]); for key in missing { - let key_tuple = from_key_to_tuple(py, &key); - results.append(py, key_tuple.into_object()); + match key { + StoreKey::HgId(key) => { + let key_tuple = from_key_to_tuple(py, &key); + results.append(py, key_tuple.into_object()); + } + StoreKey::Content(_) => { + return Err(format_err!("Unsupported key: {:?}", key)).map_pyerr(py) + } + } } Ok(results) @@ -195,8 +202,8 @@ impl RemoteDataStorePyExt for T { fn prefetch_py(&self, py: Python, keys: PyList) -> PyResult { let keys = keys .iter(py) - .map(|tuple| from_tuple_to_key(py, &tuple)) - .collect::>>()?; + .map(|tuple| Ok(StoreKey::from(from_tuple_to_key(py, &tuple)?))) + .collect::>>()?; self.prefetch(&keys).map_pyerr(py)?; Ok(Python::None(py)) } diff --git a/eden/scm/edenscmnative/bindings/modules/pyrevisionstore/src/historystorepyext.rs b/eden/scm/edenscmnative/bindings/modules/pyrevisionstore/src/historystorepyext.rs index 8b67882692..c5435d3de5 100644 --- a/eden/scm/edenscmnative/bindings/modules/pyrevisionstore/src/historystorepyext.rs +++ b/eden/scm/edenscmnative/bindings/modules/pyrevisionstore/src/historystorepyext.rs @@ -7,13 +7,15 @@ use std::convert::TryInto; -use anyhow::Result; +use anyhow::{format_err, Result}; use cpython::{ PyBytes, PyIterator, PyList, PyObject, PyResult, PyTuple, Python, PythonObject, ToPyObject, }; use cpython_ext::{PyPathBuf, ResultPyErrExt}; -use revisionstore::{HgIdHistoryStore, HgIdMutableHistoryStore, RemoteHistoryStore, ToKeys}; +use revisionstore::{ + HgIdHistoryStore, HgIdMutableHistoryStore, RemoteHistoryStore, StoreKey, ToKeys, +}; use types::{Key, NodeInfo}; use crate::pythonutil::{ @@ -52,14 +54,24 @@ impl HgIdHistoryStorePyExt for T { // Copy the PyObjects into a vector so we can get a reference iterator. // This lets us get a Vector of Keys without copying the strings. let keys = keys - .map(|k| k.and_then(|k| from_tuple_to_key(py, &k))) - .collect::>>()?; - let missing = self.get_missing(&keys[..]).map_pyerr(py)?; + .map(|k| match k { + Ok(k) => Ok(StoreKey::from(from_tuple_to_key(py, &k)?)), + Err(e) => Err(e), + }) + .collect::>>()?; + let missing = self.get_missing(&keys).map_pyerr(py)?; let results = PyList::new(py, &[]); for key in missing { - let key_tuple = from_key_to_tuple(py, &key); - results.append(py, key_tuple.into_object()); + match key { + StoreKey::HgId(key) => { + let key_tuple = from_key_to_tuple(py, &key); + results.append(py, key_tuple.into_object()); + } + StoreKey::Content(_) => { + return Err(format_err!("Unsupported key: {:?}", key)).map_pyerr(py) + } + } } Ok(results) @@ -189,8 +201,8 @@ impl RemoteHistoryStorePyExt for T { fn prefetch_py(&self, py: Python, keys: PyList) -> PyResult { let keys = keys .iter(py) - .map(|tuple| from_tuple_to_key(py, &tuple)) - .collect::>>()?; + .map(|tuple| Ok(StoreKey::from(from_tuple_to_key(py, &tuple)?))) + .collect::>>()?; self.prefetch(&keys).map_pyerr(py)?; Ok(Python::None(py)) } diff --git a/eden/scm/edenscmnative/bindings/modules/pyrevisionstore/src/lib.rs b/eden/scm/edenscmnative/bindings/modules/pyrevisionstore/src/lib.rs index 72d0960a6a..525d4d8130 100644 --- a/eden/scm/edenscmnative/bindings/modules/pyrevisionstore/src/lib.rs +++ b/eden/scm/edenscmnative/bindings/modules/pyrevisionstore/src/lib.rs @@ -25,11 +25,11 @@ use pyconfigparser::config; use revisionstore::{ repack::{filter_incrementalpacks, list_packs, repack_datapacks, repack_historypacks}, ContentStore, ContentStoreBuilder, CorruptionPolicy, DataPack, DataPackStore, DataPackVersion, - Delta, HgIdDataStore, HgIdHistoryStore, HgIdLocalStore, HgIdMutableDeltaStore, - HgIdMutableHistoryStore, HgIdRemoteStore, HistoryPack, HistoryPackStore, HistoryPackVersion, - IndexedLogHgIdDataStore, IndexedLogHgIdHistoryStore, IndexedlogRepair, MemcacheStore, Metadata, + Delta, HgIdDataStore, HgIdHistoryStore, HgIdMutableDeltaStore, HgIdMutableHistoryStore, + HgIdRemoteStore, HistoryPack, HistoryPackStore, HistoryPackVersion, IndexedLogHgIdDataStore, + IndexedLogHgIdHistoryStore, IndexedlogRepair, LocalStore, MemcacheStore, Metadata, MetadataStore, MetadataStoreBuilder, MutableDataPack, MutableHistoryPack, RemoteDataStore, - RemoteHistoryStore, + RemoteHistoryStore, StoreKey, }; use types::{Key, NodeInfo}; @@ -545,8 +545,8 @@ impl HgIdDataStore for mutabledeltastore { } } -impl HgIdLocalStore for mutabledeltastore { - fn get_missing(&self, keys: &[Key]) -> Result> { +impl LocalStore for mutabledeltastore { + fn get_missing(&self, keys: &[StoreKey]) -> Result> { let gil = Python::acquire_gil(); let py = gil.python(); @@ -623,8 +623,8 @@ impl HgIdHistoryStore for mutablehistorystore { } } -impl HgIdLocalStore for mutablehistorystore { - fn get_missing(&self, keys: &[Key]) -> Result> { +impl LocalStore for mutablehistorystore { + fn get_missing(&self, keys: &[StoreKey]) -> Result> { let gil = Python::acquire_gil(); let py = gil.python(); @@ -660,13 +660,16 @@ pub struct PyHgIdRemoteStore { } impl PyHgIdRemoteStore { - fn prefetch(&self, keys: &[Key]) -> Result<()> { + fn prefetch(&self, keys: &[StoreKey]) -> Result<()> { let gil = Python::acquire_gil(); let py = gil.python(); let keys = keys .into_iter() - .map(|key| from_key(py, &key)) + .filter_map(|key| match key { + StoreKey::HgId(key) => Some(from_key(py, &key)), + StoreKey::Content(_) => None, + }) .collect::>(); let inner = self.inner.read(); @@ -713,7 +716,7 @@ impl HgIdRemoteStore for PyHgIdRemoteStore { } impl RemoteDataStore for PyRemoteDataStore { - fn prefetch(&self, keys: &[Key]) -> Result<()> { + fn prefetch(&self, keys: &[StoreKey]) -> Result<()> { self.0.prefetch(keys) } } @@ -724,7 +727,7 @@ impl HgIdDataStore for PyRemoteDataStore { } fn get_delta(&self, key: &Key) -> Result> { - match self.prefetch(&[key.clone()]) { + match self.prefetch(&[StoreKey::from(key)]) { Ok(()) => self .0 .inner @@ -738,7 +741,7 @@ impl HgIdDataStore for PyRemoteDataStore { } fn get_delta_chain(&self, key: &Key) -> Result>> { - match self.prefetch(&[key.clone()]) { + match self.prefetch(&[StoreKey::from(key)]) { Ok(()) => self .0 .inner @@ -752,7 +755,7 @@ impl HgIdDataStore for PyRemoteDataStore { } fn get_meta(&self, key: &Key) -> Result> { - match self.prefetch(&[key.clone()]) { + match self.prefetch(&[StoreKey::from(key)]) { Ok(()) => self .0 .inner @@ -766,21 +769,21 @@ impl HgIdDataStore for PyRemoteDataStore { } } -impl HgIdLocalStore for PyRemoteDataStore { - fn get_missing(&self, keys: &[Key]) -> Result> { +impl LocalStore for PyRemoteDataStore { + fn get_missing(&self, keys: &[StoreKey]) -> Result> { Ok(keys.to_vec()) } } impl RemoteHistoryStore for PyRemoteHistoryStore { - fn prefetch(&self, keys: &[Key]) -> Result<()> { + fn prefetch(&self, keys: &[StoreKey]) -> Result<()> { self.0.prefetch(keys) } } impl HgIdHistoryStore for PyRemoteHistoryStore { fn get_node_info(&self, key: &Key) -> Result> { - match self.prefetch(&[key.clone()]) { + match self.prefetch(&[StoreKey::from(key)]) { Ok(()) => self .0 .inner @@ -794,8 +797,8 @@ impl HgIdHistoryStore for PyRemoteHistoryStore { } } -impl HgIdLocalStore for PyRemoteHistoryStore { - fn get_missing(&self, keys: &[Key]) -> Result> { +impl LocalStore for PyRemoteHistoryStore { + fn get_missing(&self, keys: &[StoreKey]) -> Result> { Ok(keys.to_vec()) } } diff --git a/eden/scm/edenscmnative/bindings/modules/pyrevisionstore/src/pythondatastore.rs b/eden/scm/edenscmnative/bindings/modules/pyrevisionstore/src/pythondatastore.rs index 0d41b1c88c..6c7e412c6c 100644 --- a/eden/scm/edenscmnative/bindings/modules/pyrevisionstore/src/pythondatastore.rs +++ b/eden/scm/edenscmnative/bindings/modules/pyrevisionstore/src/pythondatastore.rs @@ -12,7 +12,7 @@ use cpython::{ }; use cpython_ext::{PyErr, PyPathBuf}; -use revisionstore::{Delta, HgIdDataStore, HgIdLocalStore, Metadata, RemoteDataStore}; +use revisionstore::{Delta, HgIdDataStore, LocalStore, Metadata, RemoteDataStore, StoreKey}; use types::Key; use crate::pythonutil::{ @@ -147,15 +147,18 @@ impl HgIdDataStore for PythonHgIdDataStore { } impl RemoteDataStore for PythonHgIdDataStore { - fn prefetch(&self, keys: &[Key]) -> Result<()> { + fn prefetch(&self, keys: &[StoreKey]) -> Result<()> { let gil = Python::acquire_gil(); let py = gil.python(); let keys = keys .into_iter() - .map(|key| { - let py_name = PyPathBuf::from(key.path.as_repo_path()); - let py_node = PyBytes::new(py, key.hgid.as_ref()); - (py_name, py_node) + .filter_map(|key| match key { + StoreKey::HgId(key) => { + let py_name = PyPathBuf::from(key.path.as_repo_path()); + let py_node = PyBytes::new(py, key.hgid.as_ref()); + Some((py_name, py_node)) + } + StoreKey::Content(_) => None, }) .collect::>(); @@ -167,15 +170,20 @@ impl RemoteDataStore for PythonHgIdDataStore { } } -impl HgIdLocalStore for PythonHgIdDataStore { - fn get_missing(&self, keys: &[Key]) -> Result> { +impl LocalStore for PythonHgIdDataStore { + fn get_missing(&self, keys: &[StoreKey]) -> Result> { let gil = Python::acquire_gil(); let py = gil.python(); let py_missing = PyList::new(py, &[]); for key in keys.iter() { - let py_key = from_key_to_tuple(py, &key); - py_missing.append(py, py_key.into_object()); + match key { + StoreKey::HgId(key) => { + let py_key = from_key_to_tuple(py, &key); + py_missing.insert(py, py_missing.len(py), py_key.into_object()); + } + StoreKey::Content(_) => continue, + } } let py_missing = self @@ -185,8 +193,12 @@ impl HgIdLocalStore for PythonHgIdDataStore { let py_list = PyList::extract(py, &py_missing).map_err(|e| PyErr::from(e))?; let missing = py_list .iter(py) - .map(|k| from_tuple_to_key(py, &k).map_err(|e| PyErr::from(e).into())) - .collect::>>()?; + .map(|k| { + Ok(StoreKey::from( + from_tuple_to_key(py, &k).map_err(|e| PyErr::from(e))?, + )) + }) + .collect::>>()?; Ok(missing) } } diff --git a/eden/scm/edenscmnative/bindings/modules/pyrevisionstore/src/pythonhistorystore.rs b/eden/scm/edenscmnative/bindings/modules/pyrevisionstore/src/pythonhistorystore.rs index 7d0161b4cd..a57f37f606 100644 --- a/eden/scm/edenscmnative/bindings/modules/pyrevisionstore/src/pythonhistorystore.rs +++ b/eden/scm/edenscmnative/bindings/modules/pyrevisionstore/src/pythonhistorystore.rs @@ -12,7 +12,7 @@ use cpython::{ }; use cpython_ext::{PyErr, PyPathBuf}; -use revisionstore::{HgIdHistoryStore, HgIdLocalStore}; +use revisionstore::{HgIdHistoryStore, LocalStore, StoreKey}; use types::{Key, NodeInfo}; use crate::pythonutil::{bytes_from_tuple, from_key_to_tuple, from_tuple_to_key, to_node_info}; @@ -65,15 +65,20 @@ impl HgIdHistoryStore for PythonHgIdHistoryStore { } } -impl HgIdLocalStore for PythonHgIdHistoryStore { - fn get_missing(&self, keys: &[Key]) -> Result> { +impl LocalStore for PythonHgIdHistoryStore { + fn get_missing(&self, keys: &[StoreKey]) -> Result> { let gil = Python::acquire_gil(); let py = gil.python(); let py_missing = PyList::new(py, &[]); for key in keys.iter() { - let py_key = from_key_to_tuple(py, &key); - py_missing.insert_item(py, py_missing.len(py), py_key.into_object()); + match key { + StoreKey::HgId(key) => { + let py_key = from_key_to_tuple(py, &key); + py_missing.insert(py, py_missing.len(py), py_key.into_object()); + } + StoreKey::Content(_) => continue, + } } let py_missing = self @@ -83,8 +88,12 @@ impl HgIdLocalStore for PythonHgIdHistoryStore { let py_list = PyList::extract(py, &py_missing).map_err(|e| PyErr::from(e))?; let missing = py_list .iter(py) - .map(|k| from_tuple_to_key(py, &k).map_err(|e| PyErr::from(e).into())) - .collect::>>()?; + .map(|k| { + Ok(StoreKey::from( + from_tuple_to_key(py, &k).map_err(|e| PyErr::from(e))?, + )) + }) + .collect::>>()?; Ok(missing) } } diff --git a/eden/scm/lib/asyncrevisionstore/src/asyncdatastore.rs b/eden/scm/lib/asyncrevisionstore/src/asyncdatastore.rs index 5182ddb991..d7bcf415fc 100644 --- a/eden/scm/lib/asyncrevisionstore/src/asyncdatastore.rs +++ b/eden/scm/lib/asyncrevisionstore/src/asyncdatastore.rs @@ -56,14 +56,6 @@ impl AsyncHgIdDataStore { cloned!(key); self.data.block(move |store| store.get_meta(&key)) } - - /// Asynchronously call the HgIdDataStore::get_missing method. - pub fn get_missing( - &self, - keys: &'static [Key], - ) -> impl Future, Error = Error> + Send { - self.data.block(move |store| store.get_missing(keys)) - } } impl AsyncHgIdDataStore { diff --git a/eden/scm/lib/asyncrevisionstore/src/asynchistorystore.rs b/eden/scm/lib/asyncrevisionstore/src/asynchistorystore.rs index 48b5ae81a8..55fb95d642 100644 --- a/eden/scm/lib/asyncrevisionstore/src/asynchistorystore.rs +++ b/eden/scm/lib/asyncrevisionstore/src/asynchistorystore.rs @@ -27,14 +27,6 @@ impl AsyncHgIdHistoryStore { } } - /// Asynchronously call the HgIdHistoryStore::get_missing method. - pub fn get_missing( - &self, - keys: Vec, - ) -> impl Future, Error = Error> + Send { - self.history.block(move |store| store.get_missing(&keys)) - } - /// Asynchronously call the HgIdHistoryStore::get_node_info method. pub fn get_node_info( &self, diff --git a/eden/scm/lib/backingstore/src/backingstore.rs b/eden/scm/lib/backingstore/src/backingstore.rs index 15ce6e16fc..a736b824d6 100644 --- a/eden/scm/lib/backingstore/src/backingstore.rs +++ b/eden/scm/lib/backingstore/src/backingstore.rs @@ -14,7 +14,7 @@ use log::warn; use manifest::{List, Manifest}; use manifest_tree::TreeManifest; use revisionstore::{ - ContentStore, ContentStoreBuilder, EdenApiHgIdRemoteStore, HgIdDataStore, HgIdLocalStore, + ContentStore, ContentStoreBuilder, EdenApiHgIdRemoteStore, HgIdDataStore, LocalStore, MemcacheStore, }; use std::path::Path; diff --git a/eden/scm/lib/revisionstore/src/contentstore.rs b/eden/scm/lib/revisionstore/src/contentstore.rs index 2f2e09ae77..3aee9d5ba3 100644 --- a/eden/scm/lib/revisionstore/src/contentstore.rs +++ b/eden/scm/lib/revisionstore/src/contentstore.rs @@ -25,11 +25,12 @@ use crate::{ }, indexedlogdatastore::IndexedLogHgIdDataStore, lfs::{LfsMultiplexer, LfsStore}, - localstore::HgIdLocalStore, + localstore::LocalStore, memcache::MemcacheStore, multiplexstore::MultiplexDeltaStore, packstore::{CorruptionPolicy, MutableDataPackStore}, remotestore::HgIdRemoteStore, + types::StoreKey, uniondatastore::UnionHgIdDataStore, util::{ get_cache_packs_path, get_cache_path, get_indexedlogdatastore_path, get_local_path, @@ -90,7 +91,7 @@ impl HgIdDataStore for ContentStore { } impl RemoteDataStore for ContentStore { - fn prefetch(&self, keys: &[Key]) -> Result<()> { + fn prefetch(&self, keys: &[StoreKey]) -> Result<()> { if let Some(remote_store) = self.remote_store.as_ref() { let missing = self.get_missing(keys)?; if missing == vec![] { @@ -105,8 +106,8 @@ impl RemoteDataStore for ContentStore { } } -impl HgIdLocalStore for ContentStore { - fn get_missing(&self, keys: &[Key]) -> Result> { +impl LocalStore for ContentStore { + fn get_missing(&self, keys: &[StoreKey]) -> Result> { self.datastore.get_missing(keys) } } diff --git a/eden/scm/lib/revisionstore/src/datapack.rs b/eden/scm/lib/revisionstore/src/datapack.rs index 10f02eaa64..dfef79f060 100644 --- a/eden/scm/lib/revisionstore/src/datapack.rs +++ b/eden/scm/lib/revisionstore/src/datapack.rs @@ -99,11 +99,14 @@ use lz4_pyframe::decompress; use types::{HgId, Key, RepoPath}; use util::path::remove_file; -use crate::dataindex::{DataIndex, DeltaBaseOffset}; -use crate::datastore::{Delta, HgIdDataStore, Metadata}; -use crate::localstore::HgIdLocalStore; -use crate::repack::{Repackable, ToKeys}; -use crate::sliceext::SliceExt; +use crate::{ + dataindex::{DataIndex, DeltaBaseOffset}, + datastore::{Delta, HgIdDataStore, Metadata}, + localstore::LocalStore, + repack::{Repackable, ToKeys}, + sliceext::SliceExt, + types::StoreKey, +}; #[derive(Debug, Error)] #[error("Datapack Error: {0:?}")] @@ -382,17 +385,20 @@ impl HgIdDataStore for DataPack { } } -impl HgIdLocalStore for DataPack { +impl LocalStore for DataPack { fn from_path(path: &Path) -> Result { DataPack::new(path) } - fn get_missing(&self, keys: &[Key]) -> Result> { + fn get_missing(&self, keys: &[StoreKey]) -> Result> { Ok(keys .iter() - .filter(|k| match self.index.get_entry(&k.hgid) { - Ok(None) | Err(_) => true, - Ok(Some(_)) => false, + .filter(|k| match k { + StoreKey::HgId(k) => match self.index.get_entry(&k.hgid) { + Ok(None) | Err(_) => true, + Ok(Some(_)) => false, + }, + StoreKey::Content(_) => true, }) .map(|k| k.clone()) .collect()) @@ -499,13 +505,13 @@ pub mod tests { )]; let pack = make_datapack(&tempdir, &revisions); for &(ref delta, ref _metadata) in revisions.iter() { - let missing = pack.get_missing(&[delta.key.clone()]).unwrap(); + let missing = pack.get_missing(&[StoreKey::from(&delta.key)]).unwrap(); assert_eq!(missing.len(), 0); } let not = key("b", "3"); - let missing = pack.get_missing(&vec![not.clone()]).unwrap(); - assert_eq!(missing, vec![not.clone()]); + let missing = pack.get_missing(&vec![StoreKey::from(¬)]).unwrap(); + assert_eq!(missing, vec![StoreKey::from(not)]); } #[test] diff --git a/eden/scm/lib/revisionstore/src/datastore.rs b/eden/scm/lib/revisionstore/src/datastore.rs index 40b799a27d..6d409defaf 100644 --- a/eden/scm/lib/revisionstore/src/datastore.rs +++ b/eden/scm/lib/revisionstore/src/datastore.rs @@ -19,7 +19,7 @@ use serde_derive::{Deserialize, Serialize}; use types::{HgId, Key, RepoPath}; -use crate::localstore::HgIdLocalStore; +use crate::{localstore::LocalStore, types::StoreKey}; #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub struct Delta { @@ -34,7 +34,7 @@ pub struct Metadata { pub flags: Option, } -pub trait HgIdDataStore: HgIdLocalStore + Send + Sync { +pub trait HgIdDataStore: LocalStore + Send + Sync { fn get(&self, key: &Key) -> Result>>; fn get_delta(&self, key: &Key) -> Result>; fn get_delta_chain(&self, key: &Key) -> Result>>; @@ -50,7 +50,7 @@ pub trait RemoteDataStore: HgIdDataStore + Send + Sync { /// When implemented on a pure remote store, like the `EdenApi`, the method will always fetch /// everything that was asked. On a higher level store, such as the `ContentStore`, this will /// avoid fetching data that is already present locally. - fn prefetch(&self, keys: &[Key]) -> Result<()>; + fn prefetch(&self, keys: &[StoreKey]) -> Result<()>; } pub trait HgIdMutableDeltaStore: HgIdDataStore + Send + Sync { @@ -78,7 +78,7 @@ impl + Send + Sync> HgIdDataStor /// Implement `RemoteDataStore` for all types that can be `Deref` into a `RemoteDataStore`. This /// includes all the smart pointers like `Box`, `Rc`, `Arc`. impl + Send + Sync> RemoteDataStore for U { - fn prefetch(&self, keys: &[Key]) -> Result<()> { + fn prefetch(&self, keys: &[StoreKey]) -> Result<()> { T::prefetch(self, keys) } } diff --git a/eden/scm/lib/revisionstore/src/edenapi.rs b/eden/scm/lib/revisionstore/src/edenapi.rs index 1ec508f120..61350f9540 100644 --- a/eden/scm/lib/revisionstore/src/edenapi.rs +++ b/eden/scm/lib/revisionstore/src/edenapi.rs @@ -15,8 +15,9 @@ use types::Key; use crate::{ datastore::{Delta, HgIdDataStore, HgIdMutableDeltaStore, Metadata, RemoteDataStore}, historystore::{HgIdMutableHistoryStore, RemoteHistoryStore}, - localstore::HgIdLocalStore, + localstore::LocalStore, remotestore::HgIdRemoteStore, + types::StoreKey, }; #[derive(Clone)] @@ -77,11 +78,19 @@ struct EdenApiRemoteDataStore { } impl RemoteDataStore for EdenApiRemoteDataStore { - fn prefetch(&self, keys: &[Key]) -> Result<()> { + fn prefetch(&self, keys: &[StoreKey]) -> Result<()> { let edenapi = &self.inner.edenapi; + + let keys = keys + .iter() + .filter_map(|k| match k { + StoreKey::HgId(k) => Some(k.clone()), + StoreKey::Content(_) => None, + }) + .collect::>(); let (entries, _) = match edenapi.kind { - EdenApiHgIdRemoteStoreKind::File => edenapi.edenapi.get_files(keys.to_vec(), None)?, - EdenApiHgIdRemoteStoreKind::Tree => edenapi.edenapi.get_trees(keys.to_vec(), None)?, + EdenApiHgIdRemoteStoreKind::File => edenapi.edenapi.get_files(keys, None)?, + EdenApiHgIdRemoteStoreKind::Tree => edenapi.edenapi.get_trees(keys, None)?, }; for entry in entries { let key = entry.0.clone(); @@ -107,29 +116,29 @@ impl HgIdDataStore for EdenApiRemoteDataStore { } fn get_delta(&self, key: &Key) -> Result> { - match self.prefetch(&[key.clone()]) { + match self.prefetch(&[StoreKey::hgid(key.clone())]) { Ok(()) => self.inner.store.get_delta(key), Err(_) => Ok(None), } } fn get_delta_chain(&self, key: &Key) -> Result>> { - match self.prefetch(&[key.clone()]) { + match self.prefetch(&[StoreKey::hgid(key.clone())]) { Ok(()) => self.inner.store.get_delta_chain(key), Err(_) => Ok(None), } } fn get_meta(&self, key: &Key) -> Result> { - match self.prefetch(&[key.clone()]) { + match self.prefetch(&[StoreKey::hgid(key.clone())]) { Ok(()) => self.inner.store.get_meta(key), Err(_) => Ok(None), } } } -impl HgIdLocalStore for EdenApiRemoteDataStore { - fn get_missing(&self, keys: &[Key]) -> Result> { +impl LocalStore for EdenApiRemoteDataStore { + fn get_missing(&self, keys: &[StoreKey]) -> Result> { Ok(keys.to_vec()) } } diff --git a/eden/scm/lib/revisionstore/src/historypack.rs b/eden/scm/lib/revisionstore/src/historypack.rs index d72e7f0e9e..e175f74c30 100644 --- a/eden/scm/lib/revisionstore/src/historypack.rs +++ b/eden/scm/lib/revisionstore/src/historypack.rs @@ -98,11 +98,14 @@ use thiserror::Error; use types::{HgId, Key, NodeInfo, RepoPath, RepoPathBuf}; use util::path::remove_file; -use crate::historyindex::HistoryIndex; -use crate::historystore::HgIdHistoryStore; -use crate::localstore::HgIdLocalStore; -use crate::repack::{Repackable, ToKeys}; -use crate::sliceext::SliceExt; +use crate::{ + historyindex::HistoryIndex, + historystore::HgIdHistoryStore, + localstore::LocalStore, + repack::{Repackable, ToKeys}, + sliceext::SliceExt, + types::StoreKey, +}; #[derive(Debug, Error)] #[error("Historypack Error: {0:?}")] @@ -339,17 +342,20 @@ impl HgIdHistoryStore for HistoryPack { } } -impl HgIdLocalStore for HistoryPack { +impl LocalStore for HistoryPack { fn from_path(path: &Path) -> Result { HistoryPack::new(path) } - fn get_missing(&self, keys: &[Key]) -> Result> { + fn get_missing(&self, keys: &[StoreKey]) -> Result> { Ok(keys .iter() - .filter(|k| match self.index.get_hgid_entry(&k) { - Ok(None) | Err(_) => true, - Ok(Some(_)) => false, + .filter(|k| match k { + StoreKey::HgId(k) => match self.index.get_hgid_entry(k) { + Ok(None) | Err(_) => true, + Ok(Some(_)) => false, + }, + StoreKey::Content(_) => true, }) .map(|k| k.clone()) .collect()) @@ -542,12 +548,12 @@ pub mod tests { let pack = make_historypack(&tempdir, &nodes); - let mut test_keys: Vec = nodes.keys().map(|k| k.clone()).collect(); + let mut test_keys: Vec = nodes.keys().map(|k| StoreKey::from(k)).collect(); let missing_key = key("missing", "f0f0f0"); - test_keys.push(missing_key.clone()); + test_keys.push(StoreKey::from(&missing_key)); let missing = pack.get_missing(&test_keys[..]).unwrap(); - assert_eq!(vec![missing_key], missing); + assert_eq!(vec![StoreKey::from(missing_key)], missing); } #[test] diff --git a/eden/scm/lib/revisionstore/src/historystore.rs b/eden/scm/lib/revisionstore/src/historystore.rs index d9b709a4da..3ac02616fc 100644 --- a/eden/scm/lib/revisionstore/src/historystore.rs +++ b/eden/scm/lib/revisionstore/src/historystore.rs @@ -11,9 +11,9 @@ use anyhow::Result; use types::{HistoryEntry, Key, NodeInfo}; -use crate::localstore::HgIdLocalStore; +use crate::{localstore::LocalStore, types::StoreKey}; -pub trait HgIdHistoryStore: HgIdLocalStore + Send + Sync { +pub trait HgIdHistoryStore: LocalStore + Send + Sync { fn get_node_info(&self, key: &Key) -> Result>; } @@ -35,7 +35,7 @@ pub trait RemoteHistoryStore: HgIdHistoryStore + Send + Sync { /// When implemented on a pure remote store, like the `EdenApi`, the method will always fetch /// everything that was asked. On a higher level store, such as the `MetadataStore`, this will /// avoid fetching data that is already present locally. - fn prefetch(&self, keys: &[Key]) -> Result<()>; + fn prefetch(&self, keys: &[StoreKey]) -> Result<()>; } /// Implement `HgIdHistoryStore` for all types that can be `Deref` into a `HgIdHistoryStore`. @@ -58,7 +58,7 @@ impl + Send + Sync> } impl + Send + Sync> RemoteHistoryStore for U { - fn prefetch(&self, keys: &[Key]) -> Result<()> { + fn prefetch(&self, keys: &[StoreKey]) -> Result<()> { T::prefetch(self, keys) } } diff --git a/eden/scm/lib/revisionstore/src/indexedlogdatastore.rs b/eden/scm/lib/revisionstore/src/indexedlogdatastore.rs index 2416c683fd..0d6dcdd810 100644 --- a/eden/scm/lib/revisionstore/src/indexedlogdatastore.rs +++ b/eden/scm/lib/revisionstore/src/indexedlogdatastore.rs @@ -25,9 +25,10 @@ use types::{hgid::ReadHgIdExt, HgId, Key, RepoPath}; use crate::{ datastore::{Delta, HgIdDataStore, HgIdMutableDeltaStore, Metadata}, - localstore::HgIdLocalStore, + localstore::LocalStore, repack::ToKeys, sliceext::SliceExt, + types::StoreKey, }; struct IndexedLogHgIdDataStoreInner { @@ -190,18 +191,21 @@ impl HgIdMutableDeltaStore for IndexedLogHgIdDataStore { } } -impl HgIdLocalStore for IndexedLogHgIdDataStore { +impl LocalStore for IndexedLogHgIdDataStore { fn from_path(path: &Path) -> Result { IndexedLogHgIdDataStore::new(path) } - fn get_missing(&self, keys: &[Key]) -> Result> { + fn get_missing(&self, keys: &[StoreKey]) -> Result> { let inner = self.inner.read(); Ok(keys .iter() - .filter(|k| match Entry::from_log(k, &inner.log) { - Ok(None) | Err(_) => true, - Ok(Some(_)) => false, + .filter(|k| match k { + StoreKey::HgId(k) => match Entry::from_log(k, &inner.log) { + Ok(None) | Err(_) => true, + Ok(Some(_)) => false, + }, + StoreKey::Content(_) => true, }) .map(|k| k.clone()) .collect()) diff --git a/eden/scm/lib/revisionstore/src/indexedloghistorystore.rs b/eden/scm/lib/revisionstore/src/indexedloghistorystore.rs index 38c706ac90..182a540b74 100644 --- a/eden/scm/lib/revisionstore/src/indexedloghistorystore.rs +++ b/eden/scm/lib/revisionstore/src/indexedloghistorystore.rs @@ -27,9 +27,10 @@ use types::{ use crate::{ historystore::{HgIdHistoryStore, HgIdMutableHistoryStore}, - localstore::HgIdLocalStore, + localstore::LocalStore, repack::ToKeys, sliceext::SliceExt, + types::StoreKey, }; struct IndexedLogHgIdHistoryStoreInner { @@ -208,18 +209,21 @@ impl DefaultOpenOptions for IndexedLogHgIdHistoryStore { } } -impl HgIdLocalStore for IndexedLogHgIdHistoryStore { +impl LocalStore for IndexedLogHgIdHistoryStore { fn from_path(path: &Path) -> Result { IndexedLogHgIdHistoryStore::new(path) } - fn get_missing(&self, keys: &[Key]) -> Result> { + fn get_missing(&self, keys: &[StoreKey]) -> Result> { let inner = self.inner.read().unwrap(); Ok(keys .iter() - .filter(|k| match Entry::from_log(k, &inner.log) { - Ok(None) | Err(_) => true, - Ok(Some(_)) => false, + .filter(|k| match k { + StoreKey::HgId(k) => match Entry::from_log(k, &inner.log) { + Ok(None) | Err(_) => true, + Ok(Some(_)) => false, + }, + StoreKey::Content(_) => true, }) .map(|k| k.clone()) .collect()) diff --git a/eden/scm/lib/revisionstore/src/lfs.rs b/eden/scm/lib/revisionstore/src/lfs.rs index cc55e25286..cf2c9d71d1 100644 --- a/eden/scm/lib/revisionstore/src/lfs.rs +++ b/eden/scm/lib/revisionstore/src/lfs.rs @@ -16,7 +16,6 @@ use std::{ use anyhow::{bail, ensure, Result}; use bytes::{Bytes, BytesMut}; -use crypto::{digest::Digest, sha2::Sha256 as CryptoSha256}; use parking_lot::RwLock; use serde_derive::{Deserialize, Serialize}; @@ -28,7 +27,8 @@ use util::path::create_dir; use crate::{ datastore::{strip_metadata, Delta, HgIdDataStore, HgIdMutableDeltaStore, Metadata}, indexedlogutil::{Store, StoreOpenOptions}, - localstore::HgIdLocalStore, + localstore::LocalStore, + types::{ContentHash, StoreKey}, uniondatastore::UnionHgIdDataStore, util::{get_lfs_blobs_path, get_lfs_pointers_path}, }; @@ -80,24 +80,6 @@ struct LfsPointersEntry { content_hash: ContentHash, } -/// Kind of content hash stored in the LFS pointer. Adding new types is acceptable, re-ordering or -/// removal is forbidden. -#[derive(Serialize, Deserialize, PartialEq, Eq, Debug)] -enum ContentHash { - Sha256(Sha256), -} - -impl ContentHash { - fn sha256(data: &Bytes) -> Result { - let mut hash = CryptoSha256::new(); - hash.input(data); - - let mut bytes = [0; Sha256::len()]; - hash.result(&mut bytes); - Ok(ContentHash::Sha256(Sha256::from_slice(&bytes)?)) - } -} - impl LfsPointersStore { fn open_options() -> StoreOpenOptions { StoreOpenOptions::new() @@ -231,18 +213,34 @@ impl LfsStore { } } -impl HgIdLocalStore for LfsStore { - fn get_missing(&self, keys: &[Key]) -> Result> { +impl LocalStore for LfsStore { + fn get_missing(&self, keys: &[StoreKey]) -> Result> { let inner = self.inner.read(); Ok(keys .iter() - .filter(|k| match inner.pointers.get(k) { - Ok(None) | Err(_) => true, - Ok(Some(entry)) => match entry.content_hash { - ContentHash::Sha256(hash) => !inner.blobs.contains(&hash), + .filter_map(|k| match k { + StoreKey::HgId(key) => match inner.pointers.get(key) { + Ok(None) | Err(_) => Some(k.clone()), + Ok(Some(entry)) => match entry.content_hash { + ContentHash::Sha256(hash) => { + if inner.blobs.contains(&hash) { + None + } else { + Some(StoreKey::Content(entry.content_hash)) + } + } + }, + }, + StoreKey::Content(content_hash) => match content_hash { + ContentHash::Sha256(hash) => { + if inner.blobs.contains(&hash) { + None + } else { + Some(k.clone()) + } + } }, }) - .map(|k| k.clone()) .collect()) } } @@ -386,8 +384,8 @@ impl HgIdDataStore for LfsMultiplexer { } } -impl HgIdLocalStore for LfsMultiplexer { - fn get_missing(&self, keys: &[Key]) -> Result> { +impl LocalStore for LfsMultiplexer { + fn get_missing(&self, keys: &[StoreKey]) -> Result> { self.union.get_missing(keys) } } @@ -583,9 +581,12 @@ mod tests { key: k1.clone(), }; - assert_eq!(store.get_missing(&[k1.clone()])?, vec![k1.clone()]); + assert_eq!( + store.get_missing(&[StoreKey::from(&k1)])?, + vec![StoreKey::from(&k1)] + ); store.add(&delta, &Default::default())?; - assert_eq!(store.get_missing(&[k1.clone()])?, vec![]); + assert_eq!(store.get_missing(&[StoreKey::from(k1)])?, vec![]); Ok(()) } @@ -671,7 +672,7 @@ mod tests { multiplexer.add(&delta, &Default::default())?; assert_eq!(multiplexer.get_delta(&k1)?, Some(delta)); - assert_eq!(indexedlog.get_missing(&[k1.clone()])?, vec![]); + assert_eq!(indexedlog.get_missing(&[k1.into()])?, vec![]); Ok(()) } @@ -695,7 +696,10 @@ mod tests { multiplexer.add(&delta, &Default::default())?; assert_eq!(multiplexer.get_delta(&k1)?, Some(delta)); - assert_eq!(indexedlog.get_missing(&[k1.clone()])?, vec![k1.clone()]); + assert_eq!( + indexedlog.get_missing(&[StoreKey::from(&k1)])?, + vec![StoreKey::from(&k1)] + ); Ok(()) } @@ -734,7 +738,10 @@ mod tests { flags: Some(0x2000), }, )?; - assert_eq!(indexedlog.get_missing(&[k1.clone()])?, vec![k1.clone()]); + assert_eq!( + indexedlog.get_missing(&[StoreKey::from(&k1)])?, + vec![StoreKey::from(&k1)] + ); // The blob isn't present, so we cannot get it. assert_eq!(multiplexer.get(&k1)?, None); @@ -793,7 +800,10 @@ mod tests { flags: Some(0x2000), }, )?; - assert_eq!(indexedlog.get_missing(&[k1.clone()])?, vec![k1.clone()]); + assert_eq!( + indexedlog.get_missing(&[StoreKey::from(&k1)])?, + vec![StoreKey::from(&k1)] + ); // The blob isn't present, so we cannot get it. assert_eq!(multiplexer.get(&k1)?, None); diff --git a/eden/scm/lib/revisionstore/src/lib.rs b/eden/scm/lib/revisionstore/src/lib.rs index c4744256a9..21d85f6a1b 100644 --- a/eden/scm/lib/revisionstore/src/lib.rs +++ b/eden/scm/lib/revisionstore/src/lib.rs @@ -22,6 +22,7 @@ mod memcache; mod metadatastore; mod remotestore; mod sliceext; +mod types; mod unionstore; mod util; @@ -53,7 +54,7 @@ pub use crate::historypack::{HistoryEntry, HistoryPack, HistoryPackVersion}; pub use crate::historystore::{HgIdHistoryStore, HgIdMutableHistoryStore, RemoteHistoryStore}; pub use crate::indexedlogdatastore::IndexedLogHgIdDataStore; pub use crate::indexedloghistorystore::IndexedLogHgIdHistoryStore; -pub use crate::localstore::HgIdLocalStore; +pub use crate::localstore::LocalStore; pub use crate::memcache::MemcacheStore; pub use crate::metadatastore::{MetadataStore, MetadataStoreBuilder}; pub use crate::multiplexstore::{MultiplexDeltaStore, MultiplexHgIdHistoryStore}; @@ -65,6 +66,7 @@ pub use crate::packstore::{ }; pub use crate::remotestore::HgIdRemoteStore; pub use crate::repack::ToKeys; +pub use crate::types::StoreKey; pub use crate::uniondatastore::UnionHgIdDataStore; pub use crate::util::Error; diff --git a/eden/scm/lib/revisionstore/src/localstore.rs b/eden/scm/lib/revisionstore/src/localstore.rs index 2bba535fe5..a488d4f70a 100644 --- a/eden/scm/lib/revisionstore/src/localstore.rs +++ b/eden/scm/lib/revisionstore/src/localstore.rs @@ -11,9 +11,9 @@ use std::{ops::Deref, path::Path}; use anyhow::Result; -use types::Key; +use crate::types::StoreKey; -pub trait HgIdLocalStore: Send + Sync { +pub trait LocalStore: Send + Sync { /// Builds a Store from a filepath. The default implementation panics. fn from_path(_path: &Path) -> Result where @@ -23,17 +23,17 @@ pub trait HgIdLocalStore: Send + Sync { } /// Returns all the keys that aren't present in this `Store`. - fn get_missing(&self, keys: &[Key]) -> Result>; + fn get_missing(&self, keys: &[StoreKey]) -> Result>; /// Test whether this `Store` contains a specific key. - fn contains(&self, key: &Key) -> Result { + fn contains(&self, key: &StoreKey) -> Result { Ok(self.get_missing(&[key.clone()])?.is_empty()) } } /// All the types that can `Deref` into a `Store` implements `Store`. -impl + Send + Sync> HgIdLocalStore for U { - fn get_missing(&self, keys: &[Key]) -> Result> { +impl + Send + Sync> LocalStore for U { + fn get_missing(&self, keys: &[StoreKey]) -> Result> { T::get_missing(self, keys) } } diff --git a/eden/scm/lib/revisionstore/src/memcache.rs b/eden/scm/lib/revisionstore/src/memcache.rs index 214e4966f8..e92bd99394 100644 --- a/eden/scm/lib/revisionstore/src/memcache.rs +++ b/eden/scm/lib/revisionstore/src/memcache.rs @@ -19,8 +19,9 @@ use types::{Key, NodeInfo}; use crate::{ datastore::{Delta, HgIdDataStore, HgIdMutableDeltaStore, Metadata, RemoteDataStore}, historystore::{HgIdHistoryStore, HgIdMutableHistoryStore, RemoteHistoryStore}, - localstore::HgIdLocalStore, + localstore::LocalStore, remotestore::HgIdRemoteStore, + types::StoreKey, }; /// Type of blobs stored in Memcache. @@ -144,8 +145,8 @@ impl HgIdMutableHistoryStore for MemcacheStore { } } -impl HgIdLocalStore for MemcacheStore { - fn get_missing(&self, keys: &[Key]) -> Result> { +impl LocalStore for MemcacheStore { + fn get_missing(&self, keys: &[StoreKey]) -> Result> { Ok(keys.to_vec()) } } @@ -189,14 +190,14 @@ impl HgIdDataStore for MemcacheHgIdDataStore { } } -impl HgIdLocalStore for MemcacheHgIdDataStore { - fn get_missing(&self, keys: &[Key]) -> Result> { +impl LocalStore for MemcacheHgIdDataStore { + fn get_missing(&self, keys: &[StoreKey]) -> Result> { self.store.get_missing(keys) } } impl RemoteDataStore for MemcacheHgIdDataStore { - fn prefetch(&self, keys: &[Key]) -> Result<()> { + fn prefetch(&self, keys: &[StoreKey]) -> Result<()> { let span = info_span!( "MemcacheHgIdDataStore::prefetch", key_count = keys.len(), @@ -208,7 +209,15 @@ impl RemoteDataStore for MemcacheHgIdDataStore { let mut hits = 0; let mut size = 0; - for mcdata in self.memcache.get_data_iter(keys) { + let keys = keys + .iter() + .filter_map(|k| match k { + StoreKey::HgId(k) => Some(k.clone()), + StoreKey::Content(_) => None, + }) + .collect::>(); + + for mcdata in self.memcache.get_data_iter(&keys) { if let Ok(mcdata) = mcdata { let metadata = mcdata.metadata; let delta = Delta { @@ -248,14 +257,14 @@ impl HgIdHistoryStore for MemcacheHgIdHistoryStore { } } -impl HgIdLocalStore for MemcacheHgIdHistoryStore { - fn get_missing(&self, keys: &[Key]) -> Result> { +impl LocalStore for MemcacheHgIdHistoryStore { + fn get_missing(&self, keys: &[StoreKey]) -> Result> { self.store.get_missing(keys) } } impl RemoteHistoryStore for MemcacheHgIdHistoryStore { - fn prefetch(&self, keys: &[Key]) -> Result<()> { + fn prefetch(&self, keys: &[StoreKey]) -> Result<()> { let span = info_span!( "MemcacheHgIdHistoryStore::prefetch", key_count = keys.len(), @@ -264,10 +273,18 @@ impl RemoteHistoryStore for MemcacheHgIdHistoryStore { ); let _guard = span.enter(); + let keys = keys + .iter() + .filter_map(|k| match k { + StoreKey::HgId(k) => Some(k.clone()), + StoreKey::Content(_) => None, + }) + .collect::>(); + let mut hits = 0; let mut size = 0; - for mchist in self.memcache.get_hist_iter(keys) { + for mchist in self.memcache.get_hist_iter(&keys) { if let Ok(mchist) = mchist { self.store.add(&mchist.key, &mchist.nodeinfo)?; diff --git a/eden/scm/lib/revisionstore/src/metadatastore.rs b/eden/scm/lib/revisionstore/src/metadatastore.rs index 735915d94b..516da5e16f 100644 --- a/eden/scm/lib/revisionstore/src/metadatastore.rs +++ b/eden/scm/lib/revisionstore/src/metadatastore.rs @@ -18,11 +18,12 @@ use types::{Key, NodeInfo}; use crate::{ historystore::{HgIdHistoryStore, HgIdMutableHistoryStore, RemoteHistoryStore}, indexedloghistorystore::IndexedLogHgIdHistoryStore, - localstore::HgIdLocalStore, + localstore::LocalStore, memcache::MemcacheStore, multiplexstore::MultiplexHgIdHistoryStore, packstore::{CorruptionPolicy, MutableHistoryPackStore}, remotestore::HgIdRemoteStore, + types::StoreKey, unionhistorystore::UnionHgIdHistoryStore, util::{ get_cache_packs_path, get_cache_path, get_indexedloghistorystore_path, get_local_path, @@ -56,7 +57,7 @@ impl HgIdHistoryStore for MetadataStore { } impl RemoteHistoryStore for MetadataStore { - fn prefetch(&self, keys: &[Key]) -> Result<()> { + fn prefetch(&self, keys: &[StoreKey]) -> Result<()> { if let Some(remote_store) = self.remote_store.as_ref() { let missing = self.get_missing(&keys)?; if missing == vec![] { @@ -71,8 +72,8 @@ impl RemoteHistoryStore for MetadataStore { } } -impl HgIdLocalStore for MetadataStore { - fn get_missing(&self, keys: &[Key]) -> Result> { +impl LocalStore for MetadataStore { + fn get_missing(&self, keys: &[StoreKey]) -> Result> { self.historystore.get_missing(keys) } } diff --git a/eden/scm/lib/revisionstore/src/multiplexstore.rs b/eden/scm/lib/revisionstore/src/multiplexstore.rs index cc462088d9..424eb768af 100644 --- a/eden/scm/lib/revisionstore/src/multiplexstore.rs +++ b/eden/scm/lib/revisionstore/src/multiplexstore.rs @@ -11,9 +11,12 @@ use anyhow::{format_err, Result}; use types::{Key, NodeInfo}; -use crate::datastore::{Delta, HgIdDataStore, HgIdMutableDeltaStore, Metadata}; -use crate::historystore::{HgIdHistoryStore, HgIdMutableHistoryStore}; -use crate::localstore::HgIdLocalStore; +use crate::{ + datastore::{Delta, HgIdDataStore, HgIdMutableDeltaStore, Metadata}, + historystore::{HgIdHistoryStore, HgIdMutableHistoryStore}, + localstore::LocalStore, + types::StoreKey, +}; /// A `MultiplexDeltaStore` is a store that will duplicate all the writes to all the /// delta stores that it is made of. @@ -102,8 +105,8 @@ impl HgIdDataStore for MultiplexDeltaStore { } } -impl HgIdLocalStore for MultiplexDeltaStore { - fn get_missing(&self, keys: &[Key]) -> Result> { +impl LocalStore for MultiplexDeltaStore { + fn get_missing(&self, keys: &[StoreKey]) -> Result> { let initial_keys = Ok(keys.iter().cloned().collect()); self.stores .iter() @@ -144,8 +147,8 @@ impl HgIdHistoryStore for MultiplexHgIdHistoryStore< } } -impl HgIdLocalStore for MultiplexHgIdHistoryStore { - fn get_missing(&self, keys: &[Key]) -> Result> { +impl LocalStore for MultiplexHgIdHistoryStore { + fn get_missing(&self, keys: &[StoreKey]) -> Result> { let initial_keys = Ok(keys.iter().cloned().collect()); self.stores .iter() diff --git a/eden/scm/lib/revisionstore/src/mutabledatapack.rs b/eden/scm/lib/revisionstore/src/mutabledatapack.rs index eecab60c24..e502eb123c 100644 --- a/eden/scm/lib/revisionstore/src/mutabledatapack.rs +++ b/eden/scm/lib/revisionstore/src/mutabledatapack.rs @@ -24,13 +24,16 @@ use thiserror::Error; use lz4_pyframe::compress; use types::{HgId, Key}; -use crate::dataindex::{DataIndex, DeltaLocation}; -use crate::datapack::{DataEntry, DataPackVersion}; -use crate::datastore::{Delta, HgIdDataStore, HgIdMutableDeltaStore, Metadata}; -use crate::error::EmptyMutablePack; -use crate::localstore::HgIdLocalStore; -use crate::mutablepack::MutablePack; -use crate::packwriter::PackWriter; +use crate::{ + dataindex::{DataIndex, DeltaLocation}, + datapack::{DataEntry, DataPackVersion}, + datastore::{Delta, HgIdDataStore, HgIdMutableDeltaStore, Metadata}, + error::EmptyMutablePack, + localstore::LocalStore, + mutablepack::MutablePack, + packwriter::PackWriter, + types::StoreKey, +}; struct MutableDataPackInner { dir: PathBuf, @@ -267,12 +270,15 @@ impl HgIdDataStore for MutableDataPack { } } -impl HgIdLocalStore for MutableDataPack { - fn get_missing(&self, keys: &[Key]) -> Result> { +impl LocalStore for MutableDataPack { + fn get_missing(&self, keys: &[StoreKey]) -> Result> { let inner = self.inner.lock(); Ok(keys .iter() - .filter(|k| inner.mem_index.get(&k.hgid).is_none()) + .filter(|k| match k { + StoreKey::HgId(k) => inner.mem_index.get(&k.hgid).is_none(), + StoreKey::Content(_) => true, + }) .map(|k| k.clone()) .collect()) } @@ -432,9 +438,9 @@ mod tests { let not = key("not", "10000"); let missing = mutdatapack - .get_missing(&vec![delta.key.clone(), not.clone()]) + .get_missing(&vec![StoreKey::from(delta.key), StoreKey::from(¬)]) .unwrap(); - assert_eq!(missing, vec![not.clone()]); + assert_eq!(missing, vec![StoreKey::from(not)]); } #[test] diff --git a/eden/scm/lib/revisionstore/src/mutablehistorypack.rs b/eden/scm/lib/revisionstore/src/mutablehistorypack.rs index 91d0178328..6abdc0b91f 100644 --- a/eden/scm/lib/revisionstore/src/mutablehistorypack.rs +++ b/eden/scm/lib/revisionstore/src/mutablehistorypack.rs @@ -24,13 +24,16 @@ use thiserror::Error; use types::{Key, NodeInfo, RepoPath, RepoPathBuf}; -use crate::error::EmptyMutablePack; -use crate::historyindex::{FileSectionLocation, HistoryIndex, NodeLocation}; -use crate::historypack::{FileSectionHeader, HistoryEntry, HistoryPackVersion}; -use crate::historystore::{HgIdHistoryStore, HgIdMutableHistoryStore}; -use crate::localstore::HgIdLocalStore; -use crate::mutablepack::MutablePack; -use crate::packwriter::PackWriter; +use crate::{ + error::EmptyMutablePack, + historyindex::{FileSectionLocation, HistoryIndex, NodeLocation}, + historypack::{FileSectionHeader, HistoryEntry, HistoryPackVersion}, + historystore::{HgIdHistoryStore, HgIdMutableHistoryStore}, + localstore::LocalStore, + mutablepack::MutablePack, + packwriter::PackWriter, + types::StoreKey, +}; #[derive(Debug, Error)] #[error("Mutable History Pack Error: {0:?}")] @@ -306,14 +309,17 @@ impl HgIdHistoryStore for MutableHistoryPack { } } -impl HgIdLocalStore for MutableHistoryPack { - fn get_missing(&self, keys: &[Key]) -> Result> { +impl LocalStore for MutableHistoryPack { + fn get_missing(&self, keys: &[StoreKey]) -> Result> { let inner = self.inner.lock(); Ok(keys .iter() - .filter(|k| match inner.mem_index.get(&k.path) { - Some(e) => e.get(k).is_none(), - None => true, + .filter(|k| match k { + StoreKey::HgId(k) => match inner.mem_index.get(&k.path) { + Some(e) => e.get(k).is_none(), + None => true, + }, + StoreKey::Content(_) => true, }) .map(|k| k.clone()) .collect()) @@ -460,7 +466,7 @@ mod tests { true } - fn test_get_missing(insert: HashMap, notinsert: Vec) -> bool { + fn test_get_missing(insert: HashMap, notinsert: Vec) -> bool { let tempdir = tempdir().unwrap(); let muthistorypack = MutableHistoryPack::new(tempdir.path(), HistoryPackVersion::One).unwrap(); @@ -470,7 +476,7 @@ mod tests { } let mut lookup = notinsert.clone(); - lookup.extend(insert.keys().map(|k| k.clone())); + lookup.extend(insert.keys().map(|k| StoreKey::from(k))); let missing = muthistorypack.get_missing(&lookup).unwrap(); missing == notinsert diff --git a/eden/scm/lib/revisionstore/src/packstore.rs b/eden/scm/lib/revisionstore/src/packstore.rs index 87f76df37d..31b6251157 100644 --- a/eden/scm/lib/revisionstore/src/packstore.rs +++ b/eden/scm/lib/revisionstore/src/packstore.rs @@ -21,16 +21,19 @@ use parking_lot::Mutex; use types::{Key, NodeInfo}; -use crate::datapack::{DataPack, DataPackVersion}; -use crate::datastore::{Delta, HgIdDataStore, HgIdMutableDeltaStore, Metadata}; -use crate::historypack::{HistoryPack, HistoryPackVersion}; -use crate::historystore::{HgIdHistoryStore, HgIdMutableHistoryStore}; -use crate::localstore::HgIdLocalStore; -use crate::mutabledatapack::MutableDataPack; -use crate::mutablehistorypack::MutableHistoryPack; -use crate::repack::Repackable; -use crate::uniondatastore::UnionHgIdDataStore; -use crate::unionhistorystore::UnionHgIdHistoryStore; +use crate::{ + datapack::{DataPack, DataPackVersion}, + datastore::{Delta, HgIdDataStore, HgIdMutableDeltaStore, Metadata}, + historypack::{HistoryPack, HistoryPackVersion}, + historystore::{HgIdHistoryStore, HgIdMutableHistoryStore}, + localstore::LocalStore, + mutabledatapack::MutableDataPack, + mutablehistorypack::MutableHistoryPack, + repack::Repackable, + types::StoreKey, + uniondatastore::UnionHgIdDataStore, + unionhistorystore::UnionHgIdHistoryStore, +}; /// Naive implementation of a store that order its underlying stores based on how recently we found /// data in them. This helps in reducing the number of stores that are iterated on. @@ -224,7 +227,7 @@ impl HistoryPackStore { } } -impl PackStoreInner { +impl PackStoreInner { /// Open new on-disk packfiles, and close removed ones. fn rescan(&self) -> Result<()> { let mut new_packs = Vec::new(); @@ -323,8 +326,8 @@ impl PackStoreInner { } } -impl HgIdLocalStore for PackStore { - fn get_missing(&self, keys: &[Key]) -> Result> { +impl LocalStore for PackStore { + fn get_missing(&self, keys: &[StoreKey]) -> Result> { // Since the packfiles are loaded lazily, it's possible that `get_missing` is called before // any packfiles have been loaded. Let's tentatively scan the store before iterating over // all the known packs. @@ -413,8 +416,8 @@ impl HgIdDataStore for MutableDataPackStore { } } -impl HgIdLocalStore for MutableDataPackStore { - fn get_missing(&self, keys: &[Key]) -> Result> { +impl LocalStore for MutableDataPackStore { + fn get_missing(&self, keys: &[StoreKey]) -> Result> { self.inner.union_store.get_missing(keys) } } @@ -472,8 +475,8 @@ impl HgIdHistoryStore for MutableHistoryPackStore { } } -impl HgIdLocalStore for MutableHistoryPackStore { - fn get_missing(&self, keys: &[Key]) -> Result> { +impl LocalStore for MutableHistoryPackStore { + fn get_missing(&self, keys: &[StoreKey]) -> Result> { self.inner.union_store.get_missing(keys) } } @@ -548,7 +551,7 @@ mod tests { make_datapack(&tempdir, &vec![revision.clone()]); let store = DataPackStore::new(&tempdir, CorruptionPolicy::REMOVE); - let missing = store.get_missing(&vec![k])?; + let missing = store.get_missing(&vec![StoreKey::from(k)])?; assert_eq!(missing.len(), 0); Ok(()) } diff --git a/eden/scm/lib/revisionstore/src/repack.rs b/eden/scm/lib/revisionstore/src/repack.rs index 99ee54ebec..9baa14eb3b 100644 --- a/eden/scm/lib/revisionstore/src/repack.rs +++ b/eden/scm/lib/revisionstore/src/repack.rs @@ -15,14 +15,17 @@ use thiserror::Error; use types::Key; -use crate::datapack::{DataPack, DataPackVersion}; -use crate::datastore::{HgIdDataStore, HgIdMutableDeltaStore}; -use crate::historypack::{HistoryPack, HistoryPackVersion}; -use crate::historystore::{HgIdHistoryStore, HgIdMutableHistoryStore}; -use crate::localstore::HgIdLocalStore; -use crate::mutabledatapack::MutableDataPack; -use crate::mutablehistorypack::MutableHistoryPack; -use crate::mutablepack::MutablePack; +use crate::{ + datapack::{DataPack, DataPackVersion}, + datastore::{HgIdDataStore, HgIdMutableDeltaStore}, + historypack::{HistoryPack, HistoryPackVersion}, + historystore::{HgIdHistoryStore, HgIdMutableHistoryStore}, + localstore::LocalStore, + mutabledatapack::MutableDataPack, + mutablehistorypack::MutableHistoryPack, + mutablepack::MutablePack, + types::StoreKey, +}; pub trait ToKeys { fn to_keys(&self) -> Vec>; @@ -38,7 +41,7 @@ fn repack_datapack(data_pack: &DataPack, mut_pack: &mut MutableDataPack) -> Resu if let Some(chain) = data_pack.get_delta_chain(&key)? { for delta in chain.iter() { - if mut_pack.contains(&delta.key)? { + if mut_pack.contains(&StoreKey::hgid(delta.key.clone()))? { break; } @@ -63,7 +66,7 @@ enum RepackFailure { /// Repack all pack files in the paths iterator. Once repacked, the repacked packs will be removed /// from the filesystem. -fn repack_packs<'a, T: MutablePack, U: HgIdLocalStore + Repackable + ToKeys>( +fn repack_packs<'a, T: MutablePack, U: LocalStore + Repackable + ToKeys>( paths: impl IntoIterator + Clone, mut mut_pack: T, repack_pack: impl Fn(&U, &mut T) -> Result<()>, @@ -116,7 +119,8 @@ fn repack_packs<'a, T: MutablePack, U: HgIdLocalStore + Repackable + ToKeys>( .to_keys() .into_iter() .filter_map(|res| res.ok()) - .collect::>(); + .map(|k| StoreKey::hgid(k)) + .collect::>(); let missing = new_pack.get_missing(&keys)?; if missing.len() == 0 { diff --git a/eden/scm/lib/revisionstore/src/testutil.rs b/eden/scm/lib/revisionstore/src/testutil.rs index de90776b47..b02b06b25c 100644 --- a/eden/scm/lib/revisionstore/src/testutil.rs +++ b/eden/scm/lib/revisionstore/src/testutil.rs @@ -17,8 +17,9 @@ use types::{HgId, HistoryEntry, Key, NodeInfo, RepoPathBuf}; use crate::{ datastore::{Delta, HgIdDataStore, HgIdMutableDeltaStore, Metadata, RemoteDataStore}, historystore::{HgIdHistoryStore, HgIdMutableHistoryStore, RemoteHistoryStore}, - localstore::HgIdLocalStore, + localstore::LocalStore, remotestore::HgIdRemoteStore, + types::StoreKey, }; pub fn delta(data: &str, base: Option, key: Key) -> Delta { @@ -77,15 +78,20 @@ struct FakeRemoteDataStore { } impl RemoteDataStore for FakeRemoteDataStore { - fn prefetch(&self, keys: &[Key]) -> Result<()> { + fn prefetch(&self, keys: &[StoreKey]) -> Result<()> { for k in keys { - let data = self.map.get(&k).ok_or_else(|| Error::msg("Not found"))?; - let delta = Delta { - data: data.clone(), - base: None, - key: k.clone(), - }; - self.store.add(&delta, &Default::default())?; + match k { + StoreKey::HgId(k) => { + let data = self.map.get(&k).ok_or_else(|| Error::msg("Not found"))?; + let delta = Delta { + data: data.clone(), + base: None, + key: k.clone(), + }; + self.store.add(&delta, &Default::default())?; + } + StoreKey::Content(_) => continue, + } } Ok(()) @@ -98,29 +104,29 @@ impl HgIdDataStore for FakeRemoteDataStore { } fn get_delta(&self, key: &Key) -> Result> { - match self.prefetch(&[key.clone()]) { + match self.prefetch(&[StoreKey::hgid(key.clone())]) { Err(_) => Ok(None), Ok(()) => self.store.get_delta(key), } } fn get_delta_chain(&self, key: &Key) -> Result>> { - match self.prefetch(&[key.clone()]) { + match self.prefetch(&[StoreKey::hgid(key.clone())]) { Err(_) => Ok(None), Ok(()) => self.store.get_delta_chain(key), } } fn get_meta(&self, key: &Key) -> Result> { - match self.prefetch(&[key.clone()]) { + match self.prefetch(&[StoreKey::hgid(key.clone())]) { Err(_) => Ok(None), Ok(()) => self.store.get_meta(key), } } } -impl HgIdLocalStore for FakeRemoteDataStore { - fn get_missing(&self, keys: &[Key]) -> Result> { +impl LocalStore for FakeRemoteDataStore { + fn get_missing(&self, keys: &[StoreKey]) -> Result> { Ok(keys.to_vec()) } } @@ -131,10 +137,14 @@ struct FakeRemoteHistoryStore { } impl RemoteHistoryStore for FakeRemoteHistoryStore { - fn prefetch(&self, keys: &[Key]) -> Result<()> { + fn prefetch(&self, keys: &[StoreKey]) -> Result<()> { for k in keys { - self.store - .add(&k, self.map.get(&k).ok_or_else(|| Error::msg("Not found"))?)? + match k { + StoreKey::HgId(k) => self + .store + .add(&k, self.map.get(&k).ok_or_else(|| Error::msg("Not found"))?)?, + StoreKey::Content(_) => continue, + } } Ok(()) @@ -143,15 +153,15 @@ impl RemoteHistoryStore for FakeRemoteHistoryStore { impl HgIdHistoryStore for FakeRemoteHistoryStore { fn get_node_info(&self, key: &Key) -> Result> { - match self.prefetch(&[key.clone()]) { + match self.prefetch(&[StoreKey::hgid(key.clone())]) { Err(_) => Ok(None), Ok(()) => self.store.get_node_info(key), } } } -impl HgIdLocalStore for FakeRemoteHistoryStore { - fn get_missing(&self, keys: &[Key]) -> Result> { +impl LocalStore for FakeRemoteHistoryStore { + fn get_missing(&self, keys: &[StoreKey]) -> Result> { Ok(keys.to_vec()) } } diff --git a/eden/scm/lib/revisionstore/src/types.rs b/eden/scm/lib/revisionstore/src/types.rs new file mode 100644 index 0000000000..d0682b9bc9 --- /dev/null +++ b/eden/scm/lib/revisionstore/src/types.rs @@ -0,0 +1,102 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This software may be used and distributed according to the terms of the + * GNU General Public License version 2. + */ + +use anyhow::Result; +use bytes::Bytes; +use crypto::{digest::Digest, sha2::Sha256 as CryptoSha256}; +use serde_derive::{Deserialize, Serialize}; + +#[cfg(any(test, feature = "for-tests"))] +use rand::Rng; + +use types::{Key, Sha256}; + +/// Kind of content hash stored in the LFS pointer. Adding new types is acceptable, re-ordering or +/// removal is forbidden. +#[derive( + Serialize, + Deserialize, + PartialEq, + Eq, + Debug, + Hash, + Ord, + PartialOrd, + Clone +)] +pub enum ContentHash { + Sha256(Sha256), +} + +#[derive(Clone, PartialEq, Eq, Debug, Hash, Ord, PartialOrd)] +pub enum StoreKey { + HgId(Key), + Content(ContentHash), +} + +impl ContentHash { + pub fn sha256(data: &Bytes) -> Result { + let mut hash = CryptoSha256::new(); + hash.input(data); + + let mut bytes = [0; Sha256::len()]; + hash.result(&mut bytes); + Ok(ContentHash::Sha256(Sha256::from_slice(&bytes)?)) + } +} + +impl StoreKey { + pub fn hgid(key: Key) -> Self { + StoreKey::HgId(key) + } + + pub fn content(hash: ContentHash) -> Self { + StoreKey::Content(hash) + } +} + +impl From for StoreKey { + fn from(k: Key) -> Self { + StoreKey::HgId(k) + } +} + +impl<'a> From<&'a Key> for StoreKey { + fn from(k: &'a Key) -> Self { + StoreKey::HgId(k.clone()) + } +} + +impl From for StoreKey { + fn from(hash: ContentHash) -> Self { + StoreKey::Content(hash) + } +} + +impl<'a> From<&'a ContentHash> for StoreKey { + fn from(hash: &'a ContentHash) -> Self { + StoreKey::Content(hash.clone()) + } +} + +#[cfg(any(test, feature = "for-tests"))] +impl quickcheck::Arbitrary for ContentHash { + fn arbitrary(g: &mut G) -> Self { + ContentHash::Sha256(Sha256::arbitrary(g)) + } +} + +#[cfg(any(test, feature = "for-tests"))] +impl quickcheck::Arbitrary for StoreKey { + fn arbitrary(g: &mut G) -> Self { + if g.gen() { + StoreKey::HgId(Key::arbitrary(g)) + } else { + StoreKey::Content(ContentHash::arbitrary(g)) + } + } +} diff --git a/eden/scm/lib/revisionstore/src/uniondatastore.rs b/eden/scm/lib/revisionstore/src/uniondatastore.rs index 735dcb9375..332103eeff 100644 --- a/eden/scm/lib/revisionstore/src/uniondatastore.rs +++ b/eden/scm/lib/revisionstore/src/uniondatastore.rs @@ -14,6 +14,7 @@ use types::Key; use crate::{ datastore::{Delta, HgIdDataStore, Metadata, RemoteDataStore}, + types::StoreKey, unionstore::UnionStore, }; @@ -96,7 +97,7 @@ impl HgIdDataStore for UnionHgIdDataStore { } impl RemoteDataStore for UnionHgIdDataStore { - fn prefetch(&self, keys: &[Key]) -> Result<()> { + fn prefetch(&self, keys: &[StoreKey]) -> Result<()> { let initial_keys = Ok(keys.to_vec()); self.into_iter() .fold(initial_keys, |missing_keys, store| match missing_keys { @@ -122,7 +123,7 @@ mod tests { use quickcheck::quickcheck; use thiserror::Error; - use crate::localstore::HgIdLocalStore; + use crate::{localstore::LocalStore, types::StoreKey}; struct BadHgIdDataStore; @@ -150,8 +151,8 @@ mod tests { } } - impl HgIdLocalStore for EmptyHgIdDataStore { - fn get_missing(&self, keys: &[Key]) -> Result> { + impl LocalStore for EmptyHgIdDataStore { + fn get_missing(&self, keys: &[StoreKey]) -> Result> { Ok(keys.iter().cloned().collect()) } } @@ -174,8 +175,8 @@ mod tests { } } - impl HgIdLocalStore for BadHgIdDataStore { - fn get_missing(&self, _keys: &[Key]) -> Result> { + impl LocalStore for BadHgIdDataStore { + fn get_missing(&self, _keys: &[StoreKey]) -> Result> { Err(BadHgIdDataStoreError.into()) } } @@ -256,17 +257,17 @@ mod tests { } } - fn test_empty_unionstore_get_missing(keys: Vec) -> bool { + fn test_empty_unionstore_get_missing(keys: Vec) -> bool { keys == UnionHgIdDataStore::::new().get_missing(&keys).unwrap() } - fn test_empty_datastore_get_missing(keys: Vec) -> bool { + fn test_empty_datastore_get_missing(keys: Vec) -> bool { let mut unionstore = UnionHgIdDataStore::new(); unionstore.add(EmptyHgIdDataStore); keys == unionstore.get_missing(&keys).unwrap() } - fn test_bad_datastore_get_missing(keys: Vec) -> bool { + fn test_bad_datastore_get_missing(keys: Vec) -> bool { let mut unionstore = UnionHgIdDataStore::new(); unionstore.add(BadHgIdDataStore); match unionstore.get_missing(&keys) { diff --git a/eden/scm/lib/revisionstore/src/unionhistorystore.rs b/eden/scm/lib/revisionstore/src/unionhistorystore.rs index 5d772da679..fa460d7e83 100644 --- a/eden/scm/lib/revisionstore/src/unionhistorystore.rs +++ b/eden/scm/lib/revisionstore/src/unionhistorystore.rs @@ -12,6 +12,7 @@ use types::{Key, NodeInfo}; use crate::{ historystore::{HgIdHistoryStore, RemoteHistoryStore}, + types::StoreKey, unionstore::UnionStore, }; @@ -31,7 +32,7 @@ impl HgIdHistoryStore for UnionHgIdHistoryStore { } impl RemoteHistoryStore for UnionHgIdHistoryStore { - fn prefetch(&self, keys: &[Key]) -> Result<()> { + fn prefetch(&self, keys: &[StoreKey]) -> Result<()> { let initial_keys = Ok(keys.to_vec()); self.into_iter() .fold(initial_keys, |missing_keys, store| match missing_keys { @@ -57,7 +58,7 @@ mod tests { use quickcheck::quickcheck; use thiserror::Error; - use crate::localstore::HgIdLocalStore; + use crate::{localstore::LocalStore, types::StoreKey}; struct BadHgIdHistoryStore; @@ -73,8 +74,8 @@ mod tests { } } - impl HgIdLocalStore for EmptyHgIdHistoryStore { - fn get_missing(&self, keys: &[Key]) -> Result> { + impl LocalStore for EmptyHgIdHistoryStore { + fn get_missing(&self, keys: &[StoreKey]) -> Result> { Ok(keys.iter().cloned().collect()) } } @@ -85,8 +86,8 @@ mod tests { } } - impl HgIdLocalStore for BadHgIdHistoryStore { - fn get_missing(&self, _keys: &[Key]) -> Result> { + impl LocalStore for BadHgIdHistoryStore { + fn get_missing(&self, _keys: &[StoreKey]) -> Result> { Err(BadHgIdHistoryStoreError.into()) } } @@ -117,17 +118,17 @@ mod tests { } } - fn test_empty_unionstore_get_missing(keys: Vec) -> bool { + fn test_empty_unionstore_get_missing(keys: Vec) -> bool { keys == UnionHgIdHistoryStore::::new().get_missing(&keys).unwrap() } - fn test_empty_historystore_get_missing(keys: Vec) -> bool { + fn test_empty_historystore_get_missing(keys: Vec) -> bool { let mut unionstore = UnionHgIdHistoryStore::new(); unionstore.add(EmptyHgIdHistoryStore); keys == unionstore.get_missing(&keys).unwrap() } - fn test_bad_historystore_get_missing(keys: Vec) -> bool { + fn test_bad_historystore_get_missing(keys: Vec) -> bool { let mut unionstore = UnionHgIdHistoryStore::new(); unionstore.add(BadHgIdHistoryStore); match unionstore.get_missing(&keys) { diff --git a/eden/scm/lib/revisionstore/src/unionstore.rs b/eden/scm/lib/revisionstore/src/unionstore.rs index ba5d71bd15..9ee7ea225e 100644 --- a/eden/scm/lib/revisionstore/src/unionstore.rs +++ b/eden/scm/lib/revisionstore/src/unionstore.rs @@ -13,7 +13,7 @@ use anyhow::Result; use types::Key; -use crate::{localstore::HgIdLocalStore, repack::ToKeys}; +use crate::{localstore::LocalStore, repack::ToKeys, types::StoreKey}; pub struct UnionStore { stores: Vec, @@ -29,8 +29,8 @@ impl UnionStore { } } -impl HgIdLocalStore for UnionStore { - fn get_missing(&self, keys: &[Key]) -> Result> { +impl LocalStore for UnionStore { + fn get_missing(&self, keys: &[StoreKey]) -> Result> { let initial_keys = Ok(keys.iter().cloned().collect()); self.into_iter() .fold(initial_keys, |missing_keys, store| match missing_keys {