facebook/sapling
1
1
Fork 0
mirror of https://github.com/facebook/sapling.git synced 2024-10-06 14:58:03 +03:00
Code Issues Projects Releases Wiki Activity

revisionstore: add a new StoreKey type

Browse Source 
Summary:
This type can either be a Mercurial type key, or a content hash based key. Both
the prefetch and get_missing now can handle these properly. This is essential
for stores where data can either be fetched in both ways or when the data is
split in 2. For LFS for instance, it is possible to have the LFS pointer (via
getpackv2), but not the actual blob. In which case get_missing will simply
return the content hash version of the StoreKey, to signify what it actually
has missing.

Reviewed By: quark-zju

Differential Revision: D20445631

fbshipit-source-id: 06282f70214966cc96e805e9891f220b438c91a7
This commit is contained in:
Xavier Deguillard 2020-03-13 18:59:57 -07:00 committed by Facebook GitHub Bot
parent d900874401
commit 01fb3c0a77
32 changed files with 519 additions and 295 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
eden/scm/edenscmnative/bindings/modules/pymanifest/src/lib.rs
View File

@ -20,7 +20,7 @@ use manifest_tree::TreeManifest;
use pathmatcher::{AlwaysMatcher, Matcher, TreeMatcher};
use pypathmatcher::PythonMatcher;
use pyrevisionstore::PythonHgIdDataStore;
use revisionstore::{HgIdDataStore, RemoteDataStore};
use revisionstore::{HgIdDataStore, RemoteDataStore, StoreKey};
use types::{Key, Node, RepoPath, RepoPathBuf};
type Result<T, E = Error> = std::result::Result<T, E>;
@ -51,6 +51,7 @@ impl<T: HgIdDataStore + RemoteDataStore> manifest_tree::TreeStore for ManifestSt
}
fn prefetch(&self, keys: Vec<Key>) -> Result<()> {
let keys = keys.iter().map(|k| StoreKey::from(k)).collect::<Vec<_>>();
self.underlying.prefetch(&keys)
}
}

27
eden/scm/edenscmnative/bindings/modules/pyrevisionstore/src/datastorepyext.rs
View File

@ -7,15 +7,15 @@
use std::convert::TryInto;
use anyhow::Result;
use anyhow::{format_err, Result};
use cpython::{
PyBytes, PyDict, PyIterator, PyList, PyObject, PyResult, PyTuple, Python, PythonObject,
ToPyObject,
};
use cpython_ext::{PyPath, PyPathBuf, ResultPyErrExt};
use revisionstore::{HgIdDataStore, HgIdMutableDeltaStore, RemoteDataStore, ToKeys};
use types::{Key, Node};
use revisionstore::{HgIdDataStore, HgIdMutableDeltaStore, RemoteDataStore, StoreKey, ToKeys};
use types::Node;
use crate::pythonutil::{
from_base, from_delta_to_tuple, from_key, from_key_to_tuple, from_tuple_to_key, key_error,
@ -127,16 +127,23 @@ impl<T: HgIdDataStore + ?Sized> HgIdDataStorePyExt for T {
// This lets us get a Vector of Keys without copying the strings.
let keys = keys
.map(|k| match k {
Ok(k) => from_tuple_to_key(py, &k),
Ok(k) => Ok(StoreKey::from(from_tuple_to_key(py, &k)?)),
Err(e) => Err(e),
})
.collect::<PyResult<Vec<Key>>>()?;
let missing = self.get_missing(&keys[..]).map_pyerr(py)?;
.collect::<PyResult<Vec<StoreKey>>>()?;
let missing = self.get_missing(&keys).map_pyerr(py)?;
let results = PyList::new(py, &[]);
for key in missing {
let key_tuple = from_key_to_tuple(py, &key);
results.append(py, key_tuple.into_object());
match key {
StoreKey::HgId(key) => {
let key_tuple = from_key_to_tuple(py, &key);
results.append(py, key_tuple.into_object());
}
StoreKey::Content(_) => {
return Err(format_err!("Unsupported key: {:?}", key)).map_pyerr(py)
}
}
}
Ok(results)
@ -195,8 +202,8 @@ impl<T: RemoteDataStore + ?Sized> RemoteDataStorePyExt for T {
fn prefetch_py(&self, py: Python, keys: PyList) -> PyResult<PyObject> {
let keys = keys
.iter(py)
.map(|tuple| from_tuple_to_key(py, &tuple))
.collect::<PyResult<Vec<Key>>>()?;
.map(|tuple| Ok(StoreKey::from(from_tuple_to_key(py, &tuple)?)))
.collect::<PyResult<Vec<StoreKey>>>()?;
self.prefetch(&keys).map_pyerr(py)?;
Ok(Python::None(py))
}

30
eden/scm/edenscmnative/bindings/modules/pyrevisionstore/src/historystorepyext.rs
View File

@ -7,13 +7,15 @@
use std::convert::TryInto;
use anyhow::Result;
use anyhow::{format_err, Result};
use cpython::{
PyBytes, PyIterator, PyList, PyObject, PyResult, PyTuple, Python, PythonObject, ToPyObject,
};
use cpython_ext::{PyPathBuf, ResultPyErrExt};
use revisionstore::{HgIdHistoryStore, HgIdMutableHistoryStore, RemoteHistoryStore, ToKeys};
use revisionstore::{
HgIdHistoryStore, HgIdMutableHistoryStore, RemoteHistoryStore, StoreKey, ToKeys,
};
use types::{Key, NodeInfo};
use crate::pythonutil::{
@ -52,14 +54,24 @@ impl<T: HgIdHistoryStore + ?Sized> HgIdHistoryStorePyExt for T {
// Copy the PyObjects into a vector so we can get a reference iterator.
// This lets us get a Vector of Keys without copying the strings.
let keys = keys
.map(|k| k.and_then(|k| from_tuple_to_key(py, &k)))
.collect::<PyResult<Vec<Key>>>()?;
let missing = self.get_missing(&keys[..]).map_pyerr(py)?;
.map(|k| match k {
Ok(k) => Ok(StoreKey::from(from_tuple_to_key(py, &k)?)),
Err(e) => Err(e),
})
.collect::<PyResult<Vec<StoreKey>>>()?;
let missing = self.get_missing(&keys).map_pyerr(py)?;
let results = PyList::new(py, &[]);
for key in missing {
let key_tuple = from_key_to_tuple(py, &key);
results.append(py, key_tuple.into_object());
match key {
StoreKey::HgId(key) => {
let key_tuple = from_key_to_tuple(py, &key);
results.append(py, key_tuple.into_object());
}
StoreKey::Content(_) => {
return Err(format_err!("Unsupported key: {:?}", key)).map_pyerr(py)
}
}
}
Ok(results)
@ -189,8 +201,8 @@ impl<T: RemoteHistoryStore + ?Sized> RemoteHistoryStorePyExt for T {
fn prefetch_py(&self, py: Python, keys: PyList) -> PyResult<PyObject> {
let keys = keys
.iter(py)
.map(|tuple| from_tuple_to_key(py, &tuple))
.collect::<PyResult<Vec<Key>>>()?;
.map(|tuple| Ok(StoreKey::from(from_tuple_to_key(py, &tuple)?)))
.collect::<PyResult<Vec<StoreKey>>>()?;
self.prefetch(&keys).map_pyerr(py)?;
Ok(Python::None(py))
}

43
eden/scm/edenscmnative/bindings/modules/pyrevisionstore/src/lib.rs
View File

@ -25,11 +25,11 @@ use pyconfigparser::config;
use revisionstore::{
repack::{filter_incrementalpacks, list_packs, repack_datapacks, repack_historypacks},
ContentStore, ContentStoreBuilder, CorruptionPolicy, DataPack, DataPackStore, DataPackVersion,
Delta, HgIdDataStore, HgIdHistoryStore, HgIdLocalStore, HgIdMutableDeltaStore,
HgIdMutableHistoryStore, HgIdRemoteStore, HistoryPack, HistoryPackStore, HistoryPackVersion,
IndexedLogHgIdDataStore, IndexedLogHgIdHistoryStore, IndexedlogRepair, MemcacheStore, Metadata,
Delta, HgIdDataStore, HgIdHistoryStore, HgIdMutableDeltaStore, HgIdMutableHistoryStore,
HgIdRemoteStore, HistoryPack, HistoryPackStore, HistoryPackVersion, IndexedLogHgIdDataStore,
IndexedLogHgIdHistoryStore, IndexedlogRepair, LocalStore, MemcacheStore, Metadata,
MetadataStore, MetadataStoreBuilder, MutableDataPack, MutableHistoryPack, RemoteDataStore,
RemoteHistoryStore,
RemoteHistoryStore, StoreKey,
};
use types::{Key, NodeInfo};
@ -545,8 +545,8 @@ impl HgIdDataStore for mutabledeltastore {
}
}
impl HgIdLocalStore for mutabledeltastore {
fn get_missing(&self, keys: &[Key]) -> Result<Vec<Key>> {
impl LocalStore for mutabledeltastore {
fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
let gil = Python::acquire_gil();
let py = gil.python();
@ -623,8 +623,8 @@ impl HgIdHistoryStore for mutablehistorystore {
}
}
impl HgIdLocalStore for mutablehistorystore {
fn get_missing(&self, keys: &[Key]) -> Result<Vec<Key>> {
impl LocalStore for mutablehistorystore {
fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
let gil = Python::acquire_gil();
let py = gil.python();
@ -660,13 +660,16 @@ pub struct PyHgIdRemoteStore {
}
impl PyHgIdRemoteStore {
fn prefetch(&self, keys: &[Key]) -> Result<()> {
fn prefetch(&self, keys: &[StoreKey]) -> Result<()> {
let gil = Python::acquire_gil();
let py = gil.python();
let keys = keys
.into_iter()
.map(|key| from_key(py, &key))
.filter_map(|key| match key {
StoreKey::HgId(key) => Some(from_key(py, &key)),
StoreKey::Content(_) => None,
})
.collect::<Vec<_>>();
let inner = self.inner.read();
@ -713,7 +716,7 @@ impl HgIdRemoteStore for PyHgIdRemoteStore {
}
impl RemoteDataStore for PyRemoteDataStore {
fn prefetch(&self, keys: &[Key]) -> Result<()> {
fn prefetch(&self, keys: &[StoreKey]) -> Result<()> {
self.0.prefetch(keys)
}
}
@ -724,7 +727,7 @@ impl HgIdDataStore for PyRemoteDataStore {
}
fn get_delta(&self, key: &Key) -> Result<Option<Delta>> {
match self.prefetch(&[key.clone()]) {
match self.prefetch(&[StoreKey::from(key)]) {
Ok(()) => self
.0
.inner
@ -738,7 +741,7 @@ impl HgIdDataStore for PyRemoteDataStore {
}
fn get_delta_chain(&self, key: &Key) -> Result<Option<Vec<Delta>>> {
match self.prefetch(&[key.clone()]) {
match self.prefetch(&[StoreKey::from(key)]) {
Ok(()) => self
.0
.inner
@ -752,7 +755,7 @@ impl HgIdDataStore for PyRemoteDataStore {
}
fn get_meta(&self, key: &Key) -> Result<Option<Metadata>> {
match self.prefetch(&[key.clone()]) {
match self.prefetch(&[StoreKey::from(key)]) {
Ok(()) => self
.0
.inner
@ -766,21 +769,21 @@ impl HgIdDataStore for PyRemoteDataStore {
}
}
impl HgIdLocalStore for PyRemoteDataStore {
fn get_missing(&self, keys: &[Key]) -> Result<Vec<Key>> {
impl LocalStore for PyRemoteDataStore {
fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
Ok(keys.to_vec())
}
}
impl RemoteHistoryStore for PyRemoteHistoryStore {
fn prefetch(&self, keys: &[Key]) -> Result<()> {
fn prefetch(&self, keys: &[StoreKey]) -> Result<()> {
self.0.prefetch(keys)
}
}
impl HgIdHistoryStore for PyRemoteHistoryStore {
fn get_node_info(&self, key: &Key) -> Result<Option<NodeInfo>> {
match self.prefetch(&[key.clone()]) {
match self.prefetch(&[StoreKey::from(key)]) {
Ok(()) => self
.0
.inner
@ -794,8 +797,8 @@ impl HgIdHistoryStore for PyRemoteHistoryStore {
}
}
impl HgIdLocalStore for PyRemoteHistoryStore {
fn get_missing(&self, keys: &[Key]) -> Result<Vec<Key>> {
impl LocalStore for PyRemoteHistoryStore {
fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
Ok(keys.to_vec())
}
}

36
eden/scm/edenscmnative/bindings/modules/pyrevisionstore/src/pythondatastore.rs
View File

@ -12,7 +12,7 @@ use cpython::{
};
use cpython_ext::{PyErr, PyPathBuf};
use revisionstore::{Delta, HgIdDataStore, HgIdLocalStore, Metadata, RemoteDataStore};
use revisionstore::{Delta, HgIdDataStore, LocalStore, Metadata, RemoteDataStore, StoreKey};
use types::Key;
use crate::pythonutil::{
@ -147,15 +147,18 @@ impl HgIdDataStore for PythonHgIdDataStore {
}
impl RemoteDataStore for PythonHgIdDataStore {
fn prefetch(&self, keys: &[Key]) -> Result<()> {
fn prefetch(&self, keys: &[StoreKey]) -> Result<()> {
let gil = Python::acquire_gil();
let py = gil.python();
let keys = keys
.into_iter()
.map(|key| {
let py_name = PyPathBuf::from(key.path.as_repo_path());
let py_node = PyBytes::new(py, key.hgid.as_ref());
(py_name, py_node)
.filter_map(|key| match key {
StoreKey::HgId(key) => {
let py_name = PyPathBuf::from(key.path.as_repo_path());
let py_node = PyBytes::new(py, key.hgid.as_ref());
Some((py_name, py_node))
}
StoreKey::Content(_) => None,
})
.collect::<Vec<_>>();
@ -167,15 +170,20 @@ impl RemoteDataStore for PythonHgIdDataStore {
}
}
impl HgIdLocalStore for PythonHgIdDataStore {
fn get_missing(&self, keys: &[Key]) -> Result<Vec<Key>> {
impl LocalStore for PythonHgIdDataStore {
fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
let gil = Python::acquire_gil();
let py = gil.python();
let py_missing = PyList::new(py, &[]);
for key in keys.iter() {
let py_key = from_key_to_tuple(py, &key);
py_missing.append(py, py_key.into_object());
match key {
StoreKey::HgId(key) => {
let py_key = from_key_to_tuple(py, &key);
py_missing.insert(py, py_missing.len(py), py_key.into_object());
}
StoreKey::Content(_) => continue,
}
}
let py_missing = self
@ -185,8 +193,12 @@ impl HgIdLocalStore for PythonHgIdDataStore {
let py_list = PyList::extract(py, &py_missing).map_err(|e| PyErr::from(e))?;
let missing = py_list
.iter(py)
.map(|k| from_tuple_to_key(py, &k).map_err(|e| PyErr::from(e).into()))
.collect::<Result<Vec<Key>>>()?;
.map(|k| {
Ok(StoreKey::from(
from_tuple_to_key(py, &k).map_err(|e| PyErr::from(e))?,
))
})
.collect::<Result<Vec<StoreKey>>>()?;
Ok(missing)
}
}

23
eden/scm/edenscmnative/bindings/modules/pyrevisionstore/src/pythonhistorystore.rs
View File

@ -12,7 +12,7 @@ use cpython::{
};
use cpython_ext::{PyErr, PyPathBuf};
use revisionstore::{HgIdHistoryStore, HgIdLocalStore};
use revisionstore::{HgIdHistoryStore, LocalStore, StoreKey};
use types::{Key, NodeInfo};
use crate::pythonutil::{bytes_from_tuple, from_key_to_tuple, from_tuple_to_key, to_node_info};
@ -65,15 +65,20 @@ impl HgIdHistoryStore for PythonHgIdHistoryStore {
}
}
impl HgIdLocalStore for PythonHgIdHistoryStore {
fn get_missing(&self, keys: &[Key]) -> Result<Vec<Key>> {
impl LocalStore for PythonHgIdHistoryStore {
fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
let gil = Python::acquire_gil();
let py = gil.python();
let py_missing = PyList::new(py, &[]);
for key in keys.iter() {
let py_key = from_key_to_tuple(py, &key);
py_missing.insert_item(py, py_missing.len(py), py_key.into_object());
match key {
StoreKey::HgId(key) => {
let py_key = from_key_to_tuple(py, &key);
py_missing.insert(py, py_missing.len(py), py_key.into_object());
}
StoreKey::Content(_) => continue,
}
}
let py_missing = self
@ -83,8 +88,12 @@ impl HgIdLocalStore for PythonHgIdHistoryStore {
let py_list = PyList::extract(py, &py_missing).map_err(|e| PyErr::from(e))?;
let missing = py_list
.iter(py)
.map(|k| from_tuple_to_key(py, &k).map_err(|e| PyErr::from(e).into()))
.collect::<Result<Vec<Key>>>()?;
.map(|k| {
Ok(StoreKey::from(
from_tuple_to_key(py, &k).map_err(|e| PyErr::from(e))?,
))
})
.collect::<Result<Vec<StoreKey>>>()?;
Ok(missing)
}
}

8
eden/scm/lib/asyncrevisionstore/src/asyncdatastore.rs
View File

@ -56,14 +56,6 @@ impl<T: HgIdDataStore + Send + Sync> AsyncHgIdDataStore<T> {
cloned!(key);
self.data.block(move |store| store.get_meta(&key))
}
/// Asynchronously call the HgIdDataStore::get_missing method.
pub fn get_missing(
&self,
keys: &'static [Key],
) -> impl Future<Item = Vec<Key>, Error = Error> + Send {
self.data.block(move |store| store.get_missing(keys))
}
}
impl<T: HgIdDataStore + ToKeys + Send + Sync> AsyncHgIdDataStore<T> {

8
eden/scm/lib/asyncrevisionstore/src/asynchistorystore.rs
View File

@ -27,14 +27,6 @@ impl<T: HgIdHistoryStore + Send + Sync> AsyncHgIdHistoryStore<T> {
}
}
/// Asynchronously call the HgIdHistoryStore::get_missing method.
pub fn get_missing(
&self,
keys: Vec<Key>,
) -> impl Future<Item = Vec<Key>, Error = Error> + Send {
self.history.block(move |store| store.get_missing(&keys))
}
/// Asynchronously call the HgIdHistoryStore::get_node_info method.
pub fn get_node_info(
&self,

2
eden/scm/lib/backingstore/src/backingstore.rs
View File

@ -14,7 +14,7 @@ use log::warn;
use manifest::{List, Manifest};
use manifest_tree::TreeManifest;
use revisionstore::{
ContentStore, ContentStoreBuilder, EdenApiHgIdRemoteStore, HgIdDataStore, HgIdLocalStore,
ContentStore, ContentStoreBuilder, EdenApiHgIdRemoteStore, HgIdDataStore, LocalStore,
MemcacheStore,
};
use std::path::Path;

9
eden/scm/lib/revisionstore/src/contentstore.rs
View File

@ -25,11 +25,12 @@ use crate::{
},
indexedlogdatastore::IndexedLogHgIdDataStore,
lfs::{LfsMultiplexer, LfsStore},
localstore::HgIdLocalStore,
localstore::LocalStore,
memcache::MemcacheStore,
multiplexstore::MultiplexDeltaStore,
packstore::{CorruptionPolicy, MutableDataPackStore},
remotestore::HgIdRemoteStore,
types::StoreKey,
uniondatastore::UnionHgIdDataStore,
util::{
get_cache_packs_path, get_cache_path, get_indexedlogdatastore_path, get_local_path,
@ -90,7 +91,7 @@ impl HgIdDataStore for ContentStore {
}
impl RemoteDataStore for ContentStore {
fn prefetch(&self, keys: &[Key]) -> Result<()> {
fn prefetch(&self, keys: &[StoreKey]) -> Result<()> {
if let Some(remote_store) = self.remote_store.as_ref() {
let missing = self.get_missing(keys)?;
if missing == vec![] {
@ -105,8 +106,8 @@ impl RemoteDataStore for ContentStore {
}
}
impl HgIdLocalStore for ContentStore {
fn get_missing(&self, keys: &[Key]) -> Result<Vec<Key>> {
impl LocalStore for ContentStore {
fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
self.datastore.get_missing(keys)
}
}

32
eden/scm/lib/revisionstore/src/datapack.rs
View File

@ -99,11 +99,14 @@ use lz4_pyframe::decompress;
use types::{HgId, Key, RepoPath};
use util::path::remove_file;
use crate::dataindex::{DataIndex, DeltaBaseOffset};
use crate::datastore::{Delta, HgIdDataStore, Metadata};
use crate::localstore::HgIdLocalStore;
use crate::repack::{Repackable, ToKeys};
use crate::sliceext::SliceExt;
use crate::{
dataindex::{DataIndex, DeltaBaseOffset},
datastore::{Delta, HgIdDataStore, Metadata},
localstore::LocalStore,
repack::{Repackable, ToKeys},
sliceext::SliceExt,
types::StoreKey,
};
#[derive(Debug, Error)]
#[error("Datapack Error: {0:?}")]
@ -382,17 +385,20 @@ impl HgIdDataStore for DataPack {
}
}
impl HgIdLocalStore for DataPack {
impl LocalStore for DataPack {
fn from_path(path: &Path) -> Result<Self> {
DataPack::new(path)
}
fn get_missing(&self, keys: &[Key]) -> Result<Vec<Key>> {
fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
Ok(keys
.iter()
.filter(|k| match self.index.get_entry(&k.hgid) {
Ok(None) | Err(_) => true,
Ok(Some(_)) => false,
.filter(|k| match k {
StoreKey::HgId(k) => match self.index.get_entry(&k.hgid) {
Ok(None) | Err(_) => true,
Ok(Some(_)) => false,
},
StoreKey::Content(_) => true,
})
.map(|k| k.clone())
.collect())
@ -499,13 +505,13 @@ pub mod tests {
)];
let pack = make_datapack(&tempdir, &revisions);
for &(ref delta, ref _metadata) in revisions.iter() {
let missing = pack.get_missing(&[delta.key.clone()]).unwrap();
let missing = pack.get_missing(&[StoreKey::from(&delta.key)]).unwrap();
assert_eq!(missing.len(), 0);
}
let not = key("b", "3");
let missing = pack.get_missing(&vec![not.clone()]).unwrap();
assert_eq!(missing, vec![not.clone()]);
let missing = pack.get_missing(&vec![StoreKey::from(&not)]).unwrap();
assert_eq!(missing, vec![StoreKey::from(not)]);
}
#[test]

8
eden/scm/lib/revisionstore/src/datastore.rs
View File

@ -19,7 +19,7 @@ use serde_derive::{Deserialize, Serialize};
use types::{HgId, Key, RepoPath};
use crate::localstore::HgIdLocalStore;
use crate::{localstore::LocalStore, types::StoreKey};
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct Delta {
@ -34,7 +34,7 @@ pub struct Metadata {
pub flags: Option<u64>,
}
pub trait HgIdDataStore: HgIdLocalStore + Send + Sync {
pub trait HgIdDataStore: LocalStore + Send + Sync {
fn get(&self, key: &Key) -> Result<Option<Vec<u8>>>;
fn get_delta(&self, key: &Key) -> Result<Option<Delta>>;
fn get_delta_chain(&self, key: &Key) -> Result<Option<Vec<Delta>>>;
@ -50,7 +50,7 @@ pub trait RemoteDataStore: HgIdDataStore + Send + Sync {
/// When implemented on a pure remote store, like the `EdenApi`, the method will always fetch
/// everything that was asked. On a higher level store, such as the `ContentStore`, this will
/// avoid fetching data that is already present locally.
fn prefetch(&self, keys: &[Key]) -> Result<()>;
fn prefetch(&self, keys: &[StoreKey]) -> Result<()>;
}
pub trait HgIdMutableDeltaStore: HgIdDataStore + Send + Sync {
@ -78,7 +78,7 @@ impl<T: HgIdDataStore + ?Sized, U: Deref<Target = T> + Send + Sync> HgIdDataStor
/// Implement `RemoteDataStore` for all types that can be `Deref` into a `RemoteDataStore`. This
/// includes all the smart pointers like `Box`, `Rc`, `Arc`.
impl<T: RemoteDataStore + ?Sized, U: Deref<Target = T> + Send + Sync> RemoteDataStore for U {
fn prefetch(&self, keys: &[Key]) -> Result<()> {
fn prefetch(&self, keys: &[StoreKey]) -> Result<()> {
T::prefetch(self, keys)
}
}

27
eden/scm/lib/revisionstore/src/edenapi.rs
View File

@ -15,8 +15,9 @@ use types::Key;
use crate::{
datastore::{Delta, HgIdDataStore, HgIdMutableDeltaStore, Metadata, RemoteDataStore},
historystore::{HgIdMutableHistoryStore, RemoteHistoryStore},
localstore::HgIdLocalStore,
localstore::LocalStore,
remotestore::HgIdRemoteStore,
types::StoreKey,
};
#[derive(Clone)]
@ -77,11 +78,19 @@ struct EdenApiRemoteDataStore {
}
impl RemoteDataStore for EdenApiRemoteDataStore {
fn prefetch(&self, keys: &[Key]) -> Result<()> {
fn prefetch(&self, keys: &[StoreKey]) -> Result<()> {
let edenapi = &self.inner.edenapi;
let keys = keys
.iter()
.filter_map(|k| match k {
StoreKey::HgId(k) => Some(k.clone()),
StoreKey::Content(_) => None,
})
.collect::<Vec<_>>();
let (entries, _) = match edenapi.kind {
EdenApiHgIdRemoteStoreKind::File => edenapi.edenapi.get_files(keys.to_vec(), None)?,
EdenApiHgIdRemoteStoreKind::Tree => edenapi.edenapi.get_trees(keys.to_vec(), None)?,
EdenApiHgIdRemoteStoreKind::File => edenapi.edenapi.get_files(keys, None)?,
EdenApiHgIdRemoteStoreKind::Tree => edenapi.edenapi.get_trees(keys, None)?,
};
for entry in entries {
let key = entry.0.clone();
@ -107,29 +116,29 @@ impl HgIdDataStore for EdenApiRemoteDataStore {
}
fn get_delta(&self, key: &Key) -> Result<Option<Delta>> {
match self.prefetch(&[key.clone()]) {
match self.prefetch(&[StoreKey::hgid(key.clone())]) {
Ok(()) => self.inner.store.get_delta(key),
Err(_) => Ok(None),
}
}
fn get_delta_chain(&self, key: &Key) -> Result<Option<Vec<Delta>>> {
match self.prefetch(&[key.clone()]) {
match self.prefetch(&[StoreKey::hgid(key.clone())]) {
Ok(()) => self.inner.store.get_delta_chain(key),
Err(_) => Ok(None),
}
}
fn get_meta(&self, key: &Key) -> Result<Option<Metadata>> {
match self.prefetch(&[key.clone()]) {
match self.prefetch(&[StoreKey::hgid(key.clone())]) {
Ok(()) => self.inner.store.get_meta(key),
Err(_) => Ok(None),
}
}
}
impl HgIdLocalStore for EdenApiRemoteDataStore {
fn get_missing(&self, keys: &[Key]) -> Result<Vec<Key>> {
impl LocalStore for EdenApiRemoteDataStore {
fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
Ok(keys.to_vec())
}
}

32
eden/scm/lib/revisionstore/src/historypack.rs
View File

@ -98,11 +98,14 @@ use thiserror::Error;
use types::{HgId, Key, NodeInfo, RepoPath, RepoPathBuf};
use util::path::remove_file;
use crate::historyindex::HistoryIndex;
use crate::historystore::HgIdHistoryStore;
use crate::localstore::HgIdLocalStore;
use crate::repack::{Repackable, ToKeys};
use crate::sliceext::SliceExt;
use crate::{
historyindex::HistoryIndex,
historystore::HgIdHistoryStore,
localstore::LocalStore,
repack::{Repackable, ToKeys},
sliceext::SliceExt,
types::StoreKey,
};
#[derive(Debug, Error)]
#[error("Historypack Error: {0:?}")]
@ -339,17 +342,20 @@ impl HgIdHistoryStore for HistoryPack {
}
}
impl HgIdLocalStore for HistoryPack {
impl LocalStore for HistoryPack {
fn from_path(path: &Path) -> Result<Self> {
HistoryPack::new(path)
}
fn get_missing(&self, keys: &[Key]) -> Result<Vec<Key>> {
fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
Ok(keys
.iter()
.filter(|k| match self.index.get_hgid_entry(&k) {
Ok(None) | Err(_) => true,
Ok(Some(_)) => false,
.filter(|k| match k {
StoreKey::HgId(k) => match self.index.get_hgid_entry(k) {
Ok(None) | Err(_) => true,
Ok(Some(_)) => false,
},
StoreKey::Content(_) => true,
})
.map(|k| k.clone())
.collect())
@ -542,12 +548,12 @@ pub mod tests {
let pack = make_historypack(&tempdir, &nodes);
let mut test_keys: Vec<Key> = nodes.keys().map(|k| k.clone()).collect();
let mut test_keys: Vec<StoreKey> = nodes.keys().map(|k| StoreKey::from(k)).collect();
let missing_key = key("missing", "f0f0f0");
test_keys.push(missing_key.clone());
test_keys.push(StoreKey::from(&missing_key));
let missing = pack.get_missing(&test_keys[..]).unwrap();
assert_eq!(vec![missing_key], missing);
assert_eq!(vec![StoreKey::from(missing_key)], missing);
}
#[test]

8
eden/scm/lib/revisionstore/src/historystore.rs
View File

@ -11,9 +11,9 @@ use anyhow::Result;
use types::{HistoryEntry, Key, NodeInfo};
use crate::localstore::HgIdLocalStore;
use crate::{localstore::LocalStore, types::StoreKey};
pub trait HgIdHistoryStore: HgIdLocalStore + Send + Sync {
pub trait HgIdHistoryStore: LocalStore + Send + Sync {
fn get_node_info(&self, key: &Key) -> Result<Option<NodeInfo>>;
}
@ -35,7 +35,7 @@ pub trait RemoteHistoryStore: HgIdHistoryStore + Send + Sync {
/// When implemented on a pure remote store, like the `EdenApi`, the method will always fetch
/// everything that was asked. On a higher level store, such as the `MetadataStore`, this will
/// avoid fetching data that is already present locally.
fn prefetch(&self, keys: &[Key]) -> Result<()>;
fn prefetch(&self, keys: &[StoreKey]) -> Result<()>;
}
/// Implement `HgIdHistoryStore` for all types that can be `Deref` into a `HgIdHistoryStore`.
@ -58,7 +58,7 @@ impl<T: HgIdMutableHistoryStore + ?Sized, U: Deref<Target = T> + Send + Sync>
}
impl<T: RemoteHistoryStore + ?Sized, U: Deref<Target = T> + Send + Sync> RemoteHistoryStore for U {
fn prefetch(&self, keys: &[Key]) -> Result<()> {
fn prefetch(&self, keys: &[StoreKey]) -> Result<()> {
T::prefetch(self, keys)
}
}

16
eden/scm/lib/revisionstore/src/indexedlogdatastore.rs
View File

@ -25,9 +25,10 @@ use types::{hgid::ReadHgIdExt, HgId, Key, RepoPath};
use crate::{
datastore::{Delta, HgIdDataStore, HgIdMutableDeltaStore, Metadata},
localstore::HgIdLocalStore,
localstore::LocalStore,
repack::ToKeys,
sliceext::SliceExt,
types::StoreKey,
};
struct IndexedLogHgIdDataStoreInner {
@ -190,18 +191,21 @@ impl HgIdMutableDeltaStore for IndexedLogHgIdDataStore {
}
}
impl HgIdLocalStore for IndexedLogHgIdDataStore {
impl LocalStore for IndexedLogHgIdDataStore {
fn from_path(path: &Path) -> Result<Self> {
IndexedLogHgIdDataStore::new(path)
}
fn get_missing(&self, keys: &[Key]) -> Result<Vec<Key>> {
fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
let inner = self.inner.read();
Ok(keys
.iter()
.filter(|k| match Entry::from_log(k, &inner.log) {
Ok(None) | Err(_) => true,
Ok(Some(_)) => false,
.filter(|k| match k {
StoreKey::HgId(k) => match Entry::from_log(k, &inner.log) {
Ok(None) | Err(_) => true,
Ok(Some(_)) => false,
},
StoreKey::Content(_) => true,
})
.map(|k| k.clone())
.collect())

16
eden/scm/lib/revisionstore/src/indexedloghistorystore.rs
View File

@ -27,9 +27,10 @@ use types::{
use crate::{
historystore::{HgIdHistoryStore, HgIdMutableHistoryStore},
localstore::HgIdLocalStore,
localstore::LocalStore,
repack::ToKeys,
sliceext::SliceExt,
types::StoreKey,
};
struct IndexedLogHgIdHistoryStoreInner {
@ -208,18 +209,21 @@ impl DefaultOpenOptions<OpenOptions> for IndexedLogHgIdHistoryStore {
}
}
impl HgIdLocalStore for IndexedLogHgIdHistoryStore {
impl LocalStore for IndexedLogHgIdHistoryStore {
fn from_path(path: &Path) -> Result<Self> {
IndexedLogHgIdHistoryStore::new(path)
}
fn get_missing(&self, keys: &[Key]) -> Result<Vec<Key>> {
fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
let inner = self.inner.read().unwrap();
Ok(keys
.iter()
.filter(|k| match Entry::from_log(k, &inner.log) {
Ok(None) | Err(_) => true,
Ok(Some(_)) => false,
.filter(|k| match k {
StoreKey::HgId(k) => match Entry::from_log(k, &inner.log) {
Ok(None) | Err(_) => true,
Ok(Some(_)) => false,
},
StoreKey::Content(_) => true,
})
.map(|k| k.clone())
.collect())

80
eden/scm/lib/revisionstore/src/lfs.rs
View File

@ -16,7 +16,6 @@ use std::{
use anyhow::{bail, ensure, Result};
use bytes::{Bytes, BytesMut};
use crypto::{digest::Digest, sha2::Sha256 as CryptoSha256};
use parking_lot::RwLock;
use serde_derive::{Deserialize, Serialize};
@ -28,7 +27,8 @@ use util::path::create_dir;
use crate::{
datastore::{strip_metadata, Delta, HgIdDataStore, HgIdMutableDeltaStore, Metadata},
indexedlogutil::{Store, StoreOpenOptions},
localstore::HgIdLocalStore,
localstore::LocalStore,
types::{ContentHash, StoreKey},
uniondatastore::UnionHgIdDataStore,
util::{get_lfs_blobs_path, get_lfs_pointers_path},
};
@ -80,24 +80,6 @@ struct LfsPointersEntry {
content_hash: ContentHash,
}
/// Kind of content hash stored in the LFS pointer. Adding new types is acceptable, re-ordering or
/// removal is forbidden.
#[derive(Serialize, Deserialize, PartialEq, Eq, Debug)]
enum ContentHash {
Sha256(Sha256),
}
impl ContentHash {
fn sha256(data: &Bytes) -> Result<Self> {
let mut hash = CryptoSha256::new();
hash.input(data);
let mut bytes = [0; Sha256::len()];
hash.result(&mut bytes);
Ok(ContentHash::Sha256(Sha256::from_slice(&bytes)?))
}
}
impl LfsPointersStore {
fn open_options() -> StoreOpenOptions {
StoreOpenOptions::new()
@ -231,18 +213,34 @@ impl LfsStore {
}
}
impl HgIdLocalStore for LfsStore {
fn get_missing(&self, keys: &[Key]) -> Result<Vec<Key>> {
impl LocalStore for LfsStore {
fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
let inner = self.inner.read();
Ok(keys
.iter()
.filter(|k| match inner.pointers.get(k) {
Ok(None) | Err(_) => true,
Ok(Some(entry)) => match entry.content_hash {
ContentHash::Sha256(hash) => !inner.blobs.contains(&hash),
.filter_map(|k| match k {
StoreKey::HgId(key) => match inner.pointers.get(key) {
Ok(None) | Err(_) => Some(k.clone()),
Ok(Some(entry)) => match entry.content_hash {
ContentHash::Sha256(hash) => {
if inner.blobs.contains(&hash) {
None
} else {
Some(StoreKey::Content(entry.content_hash))
}
}
},
},
StoreKey::Content(content_hash) => match content_hash {
ContentHash::Sha256(hash) => {
if inner.blobs.contains(&hash) {
None
} else {
Some(k.clone())
}
}
},
})
.map(|k| k.clone())
.collect())
}
}
@ -386,8 +384,8 @@ impl HgIdDataStore for LfsMultiplexer {
}
}
impl HgIdLocalStore for LfsMultiplexer {
fn get_missing(&self, keys: &[Key]) -> Result<Vec<Key>> {
impl LocalStore for LfsMultiplexer {
fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
self.union.get_missing(keys)
}
}
@ -583,9 +581,12 @@ mod tests {
key: k1.clone(),
};
assert_eq!(store.get_missing(&[k1.clone()])?, vec![k1.clone()]);
assert_eq!(
store.get_missing(&[StoreKey::from(&k1)])?,
vec![StoreKey::from(&k1)]
);
store.add(&delta, &Default::default())?;
assert_eq!(store.get_missing(&[k1.clone()])?, vec![]);
assert_eq!(store.get_missing(&[StoreKey::from(k1)])?, vec![]);
Ok(())
}
@ -671,7 +672,7 @@ mod tests {
multiplexer.add(&delta, &Default::default())?;
assert_eq!(multiplexer.get_delta(&k1)?, Some(delta));
assert_eq!(indexedlog.get_missing(&[k1.clone()])?, vec![]);
assert_eq!(indexedlog.get_missing(&[k1.into()])?, vec![]);
Ok(())
}
@ -695,7 +696,10 @@ mod tests {
multiplexer.add(&delta, &Default::default())?;
assert_eq!(multiplexer.get_delta(&k1)?, Some(delta));
assert_eq!(indexedlog.get_missing(&[k1.clone()])?, vec![k1.clone()]);
assert_eq!(
indexedlog.get_missing(&[StoreKey::from(&k1)])?,
vec![StoreKey::from(&k1)]
);
Ok(())
}
@ -734,7 +738,10 @@ mod tests {
flags: Some(0x2000),
},
)?;
assert_eq!(indexedlog.get_missing(&[k1.clone()])?, vec![k1.clone()]);
assert_eq!(
indexedlog.get_missing(&[StoreKey::from(&k1)])?,
vec![StoreKey::from(&k1)]
);
// The blob isn't present, so we cannot get it.
assert_eq!(multiplexer.get(&k1)?, None);
@ -793,7 +800,10 @@ mod tests {
flags: Some(0x2000),
},
)?;
assert_eq!(indexedlog.get_missing(&[k1.clone()])?, vec![k1.clone()]);
assert_eq!(
indexedlog.get_missing(&[StoreKey::from(&k1)])?,
vec![StoreKey::from(&k1)]
);
// The blob isn't present, so we cannot get it.
assert_eq!(multiplexer.get(&k1)?, None);

4
eden/scm/lib/revisionstore/src/lib.rs
View File

@ -22,6 +22,7 @@ mod memcache;
mod metadatastore;
mod remotestore;
mod sliceext;
mod types;
mod unionstore;
mod util;
@ -53,7 +54,7 @@ pub use crate::historypack::{HistoryEntry, HistoryPack, HistoryPackVersion};
pub use crate::historystore::{HgIdHistoryStore, HgIdMutableHistoryStore, RemoteHistoryStore};
pub use crate::indexedlogdatastore::IndexedLogHgIdDataStore;
pub use crate::indexedloghistorystore::IndexedLogHgIdHistoryStore;
pub use crate::localstore::HgIdLocalStore;
pub use crate::localstore::LocalStore;
pub use crate::memcache::MemcacheStore;
pub use crate::metadatastore::{MetadataStore, MetadataStoreBuilder};
pub use crate::multiplexstore::{MultiplexDeltaStore, MultiplexHgIdHistoryStore};
@ -65,6 +66,7 @@ pub use crate::packstore::{
};
pub use crate::remotestore::HgIdRemoteStore;
pub use crate::repack::ToKeys;
pub use crate::types::StoreKey;
pub use crate::uniondatastore::UnionHgIdDataStore;
pub use crate::util::Error;

12
eden/scm/lib/revisionstore/src/localstore.rs
View File

@ -11,9 +11,9 @@ use std::{ops::Deref, path::Path};
use anyhow::Result;
use types::Key;
use crate::types::StoreKey;
pub trait HgIdLocalStore: Send + Sync {
pub trait LocalStore: Send + Sync {
/// Builds a Store from a filepath. The default implementation panics.
fn from_path(_path: &Path) -> Result<Self>
where
@ -23,17 +23,17 @@ pub trait HgIdLocalStore: Send + Sync {
}
/// Returns all the keys that aren't present in this `Store`.
fn get_missing(&self, keys: &[Key]) -> Result<Vec<Key>>;
fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>>;
/// Test whether this `Store` contains a specific key.
fn contains(&self, key: &Key) -> Result<bool> {
fn contains(&self, key: &StoreKey) -> Result<bool> {
Ok(self.get_missing(&[key.clone()])?.is_empty())
}
}
/// All the types that can `Deref` into a `Store` implements `Store`.
impl<T: HgIdLocalStore + ?Sized, U: Deref<Target = T> + Send + Sync> HgIdLocalStore for U {
fn get_missing(&self, keys: &[Key]) -> Result<Vec<Key>> {
impl<T: LocalStore + ?Sized, U: Deref<Target = T> + Send + Sync> LocalStore for U {
fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
T::get_missing(self, keys)
}
}

39
eden/scm/lib/revisionstore/src/memcache.rs
View File

@ -19,8 +19,9 @@ use types::{Key, NodeInfo};
use crate::{
datastore::{Delta, HgIdDataStore, HgIdMutableDeltaStore, Metadata, RemoteDataStore},
historystore::{HgIdHistoryStore, HgIdMutableHistoryStore, RemoteHistoryStore},
localstore::HgIdLocalStore,
localstore::LocalStore,
remotestore::HgIdRemoteStore,
types::StoreKey,
};
/// Type of blobs stored in Memcache.
@ -144,8 +145,8 @@ impl HgIdMutableHistoryStore for MemcacheStore {
}
}
impl HgIdLocalStore for MemcacheStore {
fn get_missing(&self, keys: &[Key]) -> Result<Vec<Key>> {
impl LocalStore for MemcacheStore {
fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
Ok(keys.to_vec())
}
}
@ -189,14 +190,14 @@ impl HgIdDataStore for MemcacheHgIdDataStore {
}
}
impl HgIdLocalStore for MemcacheHgIdDataStore {
fn get_missing(&self, keys: &[Key]) -> Result<Vec<Key>> {
impl LocalStore for MemcacheHgIdDataStore {
fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
self.store.get_missing(keys)
}
}
impl RemoteDataStore for MemcacheHgIdDataStore {
fn prefetch(&self, keys: &[Key]) -> Result<()> {
fn prefetch(&self, keys: &[StoreKey]) -> Result<()> {
let span = info_span!(
"MemcacheHgIdDataStore::prefetch",
key_count = keys.len(),
@ -208,7 +209,15 @@ impl RemoteDataStore for MemcacheHgIdDataStore {
let mut hits = 0;
let mut size = 0;
for mcdata in self.memcache.get_data_iter(keys) {
let keys = keys
.iter()
.filter_map(|k| match k {
StoreKey::HgId(k) => Some(k.clone()),
StoreKey::Content(_) => None,
})
.collect::<Vec<_>>();
for mcdata in self.memcache.get_data_iter(&keys) {
if let Ok(mcdata) = mcdata {
let metadata = mcdata.metadata;
let delta = Delta {
@ -248,14 +257,14 @@ impl HgIdHistoryStore for MemcacheHgIdHistoryStore {
}
}
impl HgIdLocalStore for MemcacheHgIdHistoryStore {
fn get_missing(&self, keys: &[Key]) -> Result<Vec<Key>> {
impl LocalStore for MemcacheHgIdHistoryStore {
fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
self.store.get_missing(keys)
}
}
impl RemoteHistoryStore for MemcacheHgIdHistoryStore {
fn prefetch(&self, keys: &[Key]) -> Result<()> {
fn prefetch(&self, keys: &[StoreKey]) -> Result<()> {
let span = info_span!(
"MemcacheHgIdHistoryStore::prefetch",
key_count = keys.len(),
@ -264,10 +273,18 @@ impl RemoteHistoryStore for MemcacheHgIdHistoryStore {
);
let _guard = span.enter();
let keys = keys
.iter()
.filter_map(|k| match k {
StoreKey::HgId(k) => Some(k.clone()),
StoreKey::Content(_) => None,
})
.collect::<Vec<_>>();
let mut hits = 0;
let mut size = 0;
for mchist in self.memcache.get_hist_iter(keys) {
for mchist in self.memcache.get_hist_iter(&keys) {
if let Ok(mchist) = mchist {
self.store.add(&mchist.key, &mchist.nodeinfo)?;

9
eden/scm/lib/revisionstore/src/metadatastore.rs
View File

@ -18,11 +18,12 @@ use types::{Key, NodeInfo};
use crate::{
historystore::{HgIdHistoryStore, HgIdMutableHistoryStore, RemoteHistoryStore},
indexedloghistorystore::IndexedLogHgIdHistoryStore,
localstore::HgIdLocalStore,
localstore::LocalStore,
memcache::MemcacheStore,
multiplexstore::MultiplexHgIdHistoryStore,
packstore::{CorruptionPolicy, MutableHistoryPackStore},
remotestore::HgIdRemoteStore,
types::StoreKey,
unionhistorystore::UnionHgIdHistoryStore,
util::{
get_cache_packs_path, get_cache_path, get_indexedloghistorystore_path, get_local_path,
@ -56,7 +57,7 @@ impl HgIdHistoryStore for MetadataStore {
}
impl RemoteHistoryStore for MetadataStore {
fn prefetch(&self, keys: &[Key]) -> Result<()> {
fn prefetch(&self, keys: &[StoreKey]) -> Result<()> {
if let Some(remote_store) = self.remote_store.as_ref() {
let missing = self.get_missing(&keys)?;
if missing == vec![] {
@ -71,8 +72,8 @@ impl RemoteHistoryStore for MetadataStore {
}
}
impl HgIdLocalStore for MetadataStore {
fn get_missing(&self, keys: &[Key]) -> Result<Vec<Key>> {
impl LocalStore for MetadataStore {
fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
self.historystore.get_missing(keys)
}
}

17
eden/scm/lib/revisionstore/src/multiplexstore.rs
View File

@ -11,9 +11,12 @@ use anyhow::{format_err, Result};
use types::{Key, NodeInfo};
use crate::datastore::{Delta, HgIdDataStore, HgIdMutableDeltaStore, Metadata};
use crate::historystore::{HgIdHistoryStore, HgIdMutableHistoryStore};
use crate::localstore::HgIdLocalStore;
use crate::{
datastore::{Delta, HgIdDataStore, HgIdMutableDeltaStore, Metadata},
historystore::{HgIdHistoryStore, HgIdMutableHistoryStore},
localstore::LocalStore,
types::StoreKey,
};
/// A `MultiplexDeltaStore` is a store that will duplicate all the writes to all the
/// delta stores that it is made of.
@ -102,8 +105,8 @@ impl<T: HgIdMutableDeltaStore> HgIdDataStore for MultiplexDeltaStore<T> {
}
}
impl<T: HgIdMutableDeltaStore> HgIdLocalStore for MultiplexDeltaStore<T> {
fn get_missing(&self, keys: &[Key]) -> Result<Vec<Key>> {
impl<T: HgIdMutableDeltaStore> LocalStore for MultiplexDeltaStore<T> {
fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
let initial_keys = Ok(keys.iter().cloned().collect());
self.stores
.iter()
@ -144,8 +147,8 @@ impl<T: HgIdMutableHistoryStore> HgIdHistoryStore for MultiplexHgIdHistoryStore<
}
}
impl<T: HgIdMutableHistoryStore> HgIdLocalStore for MultiplexHgIdHistoryStore<T> {
fn get_missing(&self, keys: &[Key]) -> Result<Vec<Key>> {
impl<T: HgIdMutableHistoryStore> LocalStore for MultiplexHgIdHistoryStore<T> {
fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
let initial_keys = Ok(keys.iter().cloned().collect());
self.stores
.iter()

30
eden/scm/lib/revisionstore/src/mutabledatapack.rs
View File

@ -24,13 +24,16 @@ use thiserror::Error;
use lz4_pyframe::compress;
use types::{HgId, Key};
use crate::dataindex::{DataIndex, DeltaLocation};
use crate::datapack::{DataEntry, DataPackVersion};
use crate::datastore::{Delta, HgIdDataStore, HgIdMutableDeltaStore, Metadata};
use crate::error::EmptyMutablePack;
use crate::localstore::HgIdLocalStore;
use crate::mutablepack::MutablePack;
use crate::packwriter::PackWriter;
use crate::{
dataindex::{DataIndex, DeltaLocation},
datapack::{DataEntry, DataPackVersion},
datastore::{Delta, HgIdDataStore, HgIdMutableDeltaStore, Metadata},
error::EmptyMutablePack,
localstore::LocalStore,
mutablepack::MutablePack,
packwriter::PackWriter,
types::StoreKey,
};
struct MutableDataPackInner {
dir: PathBuf,
@ -267,12 +270,15 @@ impl HgIdDataStore for MutableDataPack {
}
}
impl HgIdLocalStore for MutableDataPack {
fn get_missing(&self, keys: &[Key]) -> Result<Vec<Key>> {
impl LocalStore for MutableDataPack {
fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
let inner = self.inner.lock();
Ok(keys
.iter()
.filter(|k| inner.mem_index.get(&k.hgid).is_none())
.filter(|k| match k {
StoreKey::HgId(k) => inner.mem_index.get(&k.hgid).is_none(),
StoreKey::Content(_) => true,
})
.map(|k| k.clone())
.collect())
}
@ -432,9 +438,9 @@ mod tests {
let not = key("not", "10000");
let missing = mutdatapack
.get_missing(&vec![delta.key.clone(), not.clone()])
.get_missing(&vec![StoreKey::from(delta.key), StoreKey::from(&not)])
.unwrap();
assert_eq!(missing, vec![not.clone()]);
assert_eq!(missing, vec![StoreKey::from(not)]);
}
#[test]

34
eden/scm/lib/revisionstore/src/mutablehistorypack.rs
View File

@ -24,13 +24,16 @@ use thiserror::Error;
use types::{Key, NodeInfo, RepoPath, RepoPathBuf};
use crate::error::EmptyMutablePack;
use crate::historyindex::{FileSectionLocation, HistoryIndex, NodeLocation};
use crate::historypack::{FileSectionHeader, HistoryEntry, HistoryPackVersion};
use crate::historystore::{HgIdHistoryStore, HgIdMutableHistoryStore};
use crate::localstore::HgIdLocalStore;
use crate::mutablepack::MutablePack;
use crate::packwriter::PackWriter;
use crate::{
error::EmptyMutablePack,
historyindex::{FileSectionLocation, HistoryIndex, NodeLocation},
historypack::{FileSectionHeader, HistoryEntry, HistoryPackVersion},
historystore::{HgIdHistoryStore, HgIdMutableHistoryStore},
localstore::LocalStore,
mutablepack::MutablePack,
packwriter::PackWriter,
types::StoreKey,
};
#[derive(Debug, Error)]
#[error("Mutable History Pack Error: {0:?}")]
@ -306,14 +309,17 @@ impl HgIdHistoryStore for MutableHistoryPack {
}
}
impl HgIdLocalStore for MutableHistoryPack {
fn get_missing(&self, keys: &[Key]) -> Result<Vec<Key>> {
impl LocalStore for MutableHistoryPack {
fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
let inner = self.inner.lock();
Ok(keys
.iter()
.filter(|k| match inner.mem_index.get(&k.path) {
Some(e) => e.get(k).is_none(),
None => true,
.filter(|k| match k {
StoreKey::HgId(k) => match inner.mem_index.get(&k.path) {
Some(e) => e.get(k).is_none(),
None => true,
},
StoreKey::Content(_) => true,
})
.map(|k| k.clone())
.collect())
@ -460,7 +466,7 @@ mod tests {
true
}
fn test_get_missing(insert: HashMap<Key, NodeInfo>, notinsert: Vec<Key>) -> bool {
fn test_get_missing(insert: HashMap<Key, NodeInfo>, notinsert: Vec<StoreKey>) -> bool {
let tempdir = tempdir().unwrap();
let muthistorypack =
MutableHistoryPack::new(tempdir.path(), HistoryPackVersion::One).unwrap();
@ -470,7 +476,7 @@ mod tests {
}
let mut lookup = notinsert.clone();
lookup.extend(insert.keys().map(|k| k.clone()));
lookup.extend(insert.keys().map(|k| StoreKey::from(k)));
let missing = muthistorypack.get_missing(&lookup).unwrap();
missing == notinsert

39
eden/scm/lib/revisionstore/src/packstore.rs
View File

@ -21,16 +21,19 @@ use parking_lot::Mutex;
use types::{Key, NodeInfo};
use crate::datapack::{DataPack, DataPackVersion};
use crate::datastore::{Delta, HgIdDataStore, HgIdMutableDeltaStore, Metadata};
use crate::historypack::{HistoryPack, HistoryPackVersion};
use crate::historystore::{HgIdHistoryStore, HgIdMutableHistoryStore};
use crate::localstore::HgIdLocalStore;
use crate::mutabledatapack::MutableDataPack;
use crate::mutablehistorypack::MutableHistoryPack;
use crate::repack::Repackable;
use crate::uniondatastore::UnionHgIdDataStore;
use crate::unionhistorystore::UnionHgIdHistoryStore;
use crate::{
datapack::{DataPack, DataPackVersion},
datastore::{Delta, HgIdDataStore, HgIdMutableDeltaStore, Metadata},
historypack::{HistoryPack, HistoryPackVersion},
historystore::{HgIdHistoryStore, HgIdMutableHistoryStore},
localstore::LocalStore,
mutabledatapack::MutableDataPack,
mutablehistorypack::MutableHistoryPack,
repack::Repackable,
types::StoreKey,
uniondatastore::UnionHgIdDataStore,
unionhistorystore::UnionHgIdHistoryStore,
};
/// Naive implementation of a store that order its underlying stores based on how recently we found
/// data in them. This helps in reducing the number of stores that are iterated on.
@ -224,7 +227,7 @@ impl HistoryPackStore {
}
}
impl<T: HgIdLocalStore + Repackable> PackStoreInner<T> {
impl<T: LocalStore + Repackable> PackStoreInner<T> {
/// Open new on-disk packfiles, and close removed ones.
fn rescan(&self) -> Result<()> {
let mut new_packs = Vec::new();
@ -323,8 +326,8 @@ impl<T: HgIdLocalStore + Repackable> PackStoreInner<T> {
}
}
impl<T: HgIdLocalStore + Repackable> HgIdLocalStore for PackStore<T> {
fn get_missing(&self, keys: &[Key]) -> Result<Vec<Key>> {
impl<T: LocalStore + Repackable> LocalStore for PackStore<T> {
fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
// Since the packfiles are loaded lazily, it's possible that `get_missing` is called before
// any packfiles have been loaded. Let's tentatively scan the store before iterating over
// all the known packs.
@ -413,8 +416,8 @@ impl HgIdDataStore for MutableDataPackStore {
}
}
impl HgIdLocalStore for MutableDataPackStore {
fn get_missing(&self, keys: &[Key]) -> Result<Vec<Key>> {
impl LocalStore for MutableDataPackStore {
fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
self.inner.union_store.get_missing(keys)
}
}
@ -472,8 +475,8 @@ impl HgIdHistoryStore for MutableHistoryPackStore {
}
}
impl HgIdLocalStore for MutableHistoryPackStore {
fn get_missing(&self, keys: &[Key]) -> Result<Vec<Key>> {
impl LocalStore for MutableHistoryPackStore {
fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
self.inner.union_store.get_missing(keys)
}
}
@ -548,7 +551,7 @@ mod tests {
make_datapack(&tempdir, &vec![revision.clone()]);
let store = DataPackStore::new(&tempdir, CorruptionPolicy::REMOVE);
let missing = store.get_missing(&vec![k])?;
let missing = store.get_missing(&vec![StoreKey::from(k)])?;
assert_eq!(missing.len(), 0);
Ok(())
}

26
eden/scm/lib/revisionstore/src/repack.rs
View File

@ -15,14 +15,17 @@ use thiserror::Error;
use types::Key;
use crate::datapack::{DataPack, DataPackVersion};
use crate::datastore::{HgIdDataStore, HgIdMutableDeltaStore};
use crate::historypack::{HistoryPack, HistoryPackVersion};
use crate::historystore::{HgIdHistoryStore, HgIdMutableHistoryStore};
use crate::localstore::HgIdLocalStore;
use crate::mutabledatapack::MutableDataPack;
use crate::mutablehistorypack::MutableHistoryPack;
use crate::mutablepack::MutablePack;
use crate::{
datapack::{DataPack, DataPackVersion},
datastore::{HgIdDataStore, HgIdMutableDeltaStore},
historypack::{HistoryPack, HistoryPackVersion},
historystore::{HgIdHistoryStore, HgIdMutableHistoryStore},
localstore::LocalStore,
mutabledatapack::MutableDataPack,
mutablehistorypack::MutableHistoryPack,
mutablepack::MutablePack,
types::StoreKey,
};
pub trait ToKeys {
fn to_keys(&self) -> Vec<Result<Key>>;
@ -38,7 +41,7 @@ fn repack_datapack(data_pack: &DataPack, mut_pack: &mut MutableDataPack) -> Resu
if let Some(chain) = data_pack.get_delta_chain(&key)? {
for delta in chain.iter() {
if mut_pack.contains(&delta.key)? {
if mut_pack.contains(&StoreKey::hgid(delta.key.clone()))? {
break;
}
@ -63,7 +66,7 @@ enum RepackFailure {
/// Repack all pack files in the paths iterator. Once repacked, the repacked packs will be removed
/// from the filesystem.
fn repack_packs<'a, T: MutablePack, U: HgIdLocalStore + Repackable + ToKeys>(
fn repack_packs<'a, T: MutablePack, U: LocalStore + Repackable + ToKeys>(
paths: impl IntoIterator<Item = &'a PathBuf> + Clone,
mut mut_pack: T,
repack_pack: impl Fn(&U, &mut T) -> Result<()>,
@ -116,7 +119,8 @@ fn repack_packs<'a, T: MutablePack, U: HgIdLocalStore + Repackable + ToKeys>(
.to_keys()
.into_iter()
.filter_map(|res| res.ok())
.collect::<Vec<Key>>();
.map(|k| StoreKey::hgid(k))
.collect::<Vec<_>>();
let missing = new_pack.get_missing(&keys)?;
if missing.len() == 0 {

50
eden/scm/lib/revisionstore/src/testutil.rs
View File

@ -17,8 +17,9 @@ use types::{HgId, HistoryEntry, Key, NodeInfo, RepoPathBuf};
use crate::{
datastore::{Delta, HgIdDataStore, HgIdMutableDeltaStore, Metadata, RemoteDataStore},
historystore::{HgIdHistoryStore, HgIdMutableHistoryStore, RemoteHistoryStore},
localstore::HgIdLocalStore,
localstore::LocalStore,
remotestore::HgIdRemoteStore,
types::StoreKey,
};
pub fn delta(data: &str, base: Option<Key>, key: Key) -> Delta {
@ -77,15 +78,20 @@ struct FakeRemoteDataStore {
}
impl RemoteDataStore for FakeRemoteDataStore {
fn prefetch(&self, keys: &[Key]) -> Result<()> {
fn prefetch(&self, keys: &[StoreKey]) -> Result<()> {
for k in keys {
let data = self.map.get(&k).ok_or_else(|| Error::msg("Not found"))?;
let delta = Delta {
data: data.clone(),
base: None,
key: k.clone(),
};
self.store.add(&delta, &Default::default())?;
match k {
StoreKey::HgId(k) => {
let data = self.map.get(&k).ok_or_else(|| Error::msg("Not found"))?;
let delta = Delta {
data: data.clone(),
base: None,
key: k.clone(),
};
self.store.add(&delta, &Default::default())?;
}
StoreKey::Content(_) => continue,
}
}
Ok(())
@ -98,29 +104,29 @@ impl HgIdDataStore for FakeRemoteDataStore {
}
fn get_delta(&self, key: &Key) -> Result<Option<Delta>> {
match self.prefetch(&[key.clone()]) {
match self.prefetch(&[StoreKey::hgid(key.clone())]) {
Err(_) => Ok(None),
Ok(()) => self.store.get_delta(key),
}
}
fn get_delta_chain(&self, key: &Key) -> Result<Option<Vec<Delta>>> {
match self.prefetch(&[key.clone()]) {
match self.prefetch(&[StoreKey::hgid(key.clone())]) {
Err(_) => Ok(None),
Ok(()) => self.store.get_delta_chain(key),
}
}
fn get_meta(&self, key: &Key) -> Result<Option<Metadata>> {
match self.prefetch(&[key.clone()]) {
match self.prefetch(&[StoreKey::hgid(key.clone())]) {
Err(_) => Ok(None),
Ok(()) => self.store.get_meta(key),
}
}
}
impl HgIdLocalStore for FakeRemoteDataStore {
fn get_missing(&self, keys: &[Key]) -> Result<Vec<Key>> {
impl LocalStore for FakeRemoteDataStore {
fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
Ok(keys.to_vec())
}
}
@ -131,10 +137,14 @@ struct FakeRemoteHistoryStore {
}
impl RemoteHistoryStore for FakeRemoteHistoryStore {
fn prefetch(&self, keys: &[Key]) -> Result<()> {
fn prefetch(&self, keys: &[StoreKey]) -> Result<()> {
for k in keys {
self.store
.add(&k, self.map.get(&k).ok_or_else(|| Error::msg("Not found"))?)?
match k {
StoreKey::HgId(k) => self
.store
.add(&k, self.map.get(&k).ok_or_else(|| Error::msg("Not found"))?)?,
StoreKey::Content(_) => continue,
}
}
Ok(())
@ -143,15 +153,15 @@ impl RemoteHistoryStore for FakeRemoteHistoryStore {
impl HgIdHistoryStore for FakeRemoteHistoryStore {
fn get_node_info(&self, key: &Key) -> Result<Option<NodeInfo>> {
match self.prefetch(&[key.clone()]) {
match self.prefetch(&[StoreKey::hgid(key.clone())]) {
Err(_) => Ok(None),
Ok(()) => self.store.get_node_info(key),
}
}
}
impl HgIdLocalStore for FakeRemoteHistoryStore {
fn get_missing(&self, keys: &[Key]) -> Result<Vec<Key>> {
impl LocalStore for FakeRemoteHistoryStore {
fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
Ok(keys.to_vec())
}
}

102
eden/scm/lib/revisionstore/src/types.rs Normal file
View File

@ -0,0 +1,102 @@
/*
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This software may be used and distributed according to the terms of the
* GNU General Public License version 2.
*/
use anyhow::Result;
use bytes::Bytes;
use crypto::{digest::Digest, sha2::Sha256 as CryptoSha256};
use serde_derive::{Deserialize, Serialize};
#[cfg(any(test, feature = "for-tests"))]
use rand::Rng;
use types::{Key, Sha256};
/// Kind of content hash stored in the LFS pointer. Adding new types is acceptable, re-ordering or
/// removal is forbidden.
#[derive(
Serialize,
Deserialize,
PartialEq,
Eq,
Debug,
Hash,
Ord,
PartialOrd,
Clone
)]
pub enum ContentHash {
Sha256(Sha256),
}
#[derive(Clone, PartialEq, Eq, Debug, Hash, Ord, PartialOrd)]
pub enum StoreKey {
HgId(Key),
Content(ContentHash),
}
impl ContentHash {
pub fn sha256(data: &Bytes) -> Result<Self> {
let mut hash = CryptoSha256::new();
hash.input(data);
let mut bytes = [0; Sha256::len()];
hash.result(&mut bytes);
Ok(ContentHash::Sha256(Sha256::from_slice(&bytes)?))
}
}
impl StoreKey {
pub fn hgid(key: Key) -> Self {
StoreKey::HgId(key)
}
pub fn content(hash: ContentHash) -> Self {
StoreKey::Content(hash)
}
}
impl From<Key> for StoreKey {
fn from(k: Key) -> Self {
StoreKey::HgId(k)
}
}
impl<'a> From<&'a Key> for StoreKey {
fn from(k: &'a Key) -> Self {
StoreKey::HgId(k.clone())
}
}
impl From<ContentHash> for StoreKey {
fn from(hash: ContentHash) -> Self {
StoreKey::Content(hash)
}
}
impl<'a> From<&'a ContentHash> for StoreKey {
fn from(hash: &'a ContentHash) -> Self {
StoreKey::Content(hash.clone())
}
}
#[cfg(any(test, feature = "for-tests"))]
impl quickcheck::Arbitrary for ContentHash {
fn arbitrary<G: quickcheck::Gen>(g: &mut G) -> Self {
ContentHash::Sha256(Sha256::arbitrary(g))
}
}
#[cfg(any(test, feature = "for-tests"))]
impl quickcheck::Arbitrary for StoreKey {
fn arbitrary<G: quickcheck::Gen>(g: &mut G) -> Self {
if g.gen() {
StoreKey::HgId(Key::arbitrary(g))
} else {
StoreKey::Content(ContentHash::arbitrary(g))
}
}
}

19
eden/scm/lib/revisionstore/src/uniondatastore.rs
View File

@ -14,6 +14,7 @@ use types::Key;
use crate::{
datastore::{Delta, HgIdDataStore, Metadata, RemoteDataStore},
types::StoreKey,
unionstore::UnionStore,
};
@ -96,7 +97,7 @@ impl<T: HgIdDataStore> HgIdDataStore for UnionHgIdDataStore<T> {
}
impl<T: RemoteDataStore> RemoteDataStore for UnionHgIdDataStore<T> {
fn prefetch(&self, keys: &[Key]) -> Result<()> {
fn prefetch(&self, keys: &[StoreKey]) -> Result<()> {
let initial_keys = Ok(keys.to_vec());
self.into_iter()
.fold(initial_keys, |missing_keys, store| match missing_keys {
@ -122,7 +123,7 @@ mod tests {
use quickcheck::quickcheck;
use thiserror::Error;
use crate::localstore::HgIdLocalStore;
use crate::{localstore::LocalStore, types::StoreKey};
struct BadHgIdDataStore;
@ -150,8 +151,8 @@ mod tests {
}
}
impl HgIdLocalStore for EmptyHgIdDataStore {
fn get_missing(&self, keys: &[Key]) -> Result<Vec<Key>> {
impl LocalStore for EmptyHgIdDataStore {
fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
Ok(keys.iter().cloned().collect())
}
}
@ -174,8 +175,8 @@ mod tests {
}
}
impl HgIdLocalStore for BadHgIdDataStore {
fn get_missing(&self, _keys: &[Key]) -> Result<Vec<Key>> {
impl LocalStore for BadHgIdDataStore {
fn get_missing(&self, _keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
Err(BadHgIdDataStoreError.into())
}
}
@ -256,17 +257,17 @@ mod tests {
}
}
fn test_empty_unionstore_get_missing(keys: Vec<Key>) -> bool {
fn test_empty_unionstore_get_missing(keys: Vec<StoreKey>) -> bool {
keys == UnionHgIdDataStore::<EmptyHgIdDataStore>::new().get_missing(&keys).unwrap()
}
fn test_empty_datastore_get_missing(keys: Vec<Key>) -> bool {
fn test_empty_datastore_get_missing(keys: Vec<StoreKey>) -> bool {
let mut unionstore = UnionHgIdDataStore::new();
unionstore.add(EmptyHgIdDataStore);
keys == unionstore.get_missing(&keys).unwrap()
}
fn test_bad_datastore_get_missing(keys: Vec<Key>) -> bool {
fn test_bad_datastore_get_missing(keys: Vec<StoreKey>) -> bool {
let mut unionstore = UnionHgIdDataStore::new();
unionstore.add(BadHgIdDataStore);
match unionstore.get_missing(&keys) {

19
eden/scm/lib/revisionstore/src/unionhistorystore.rs
View File

@ -12,6 +12,7 @@ use types::{Key, NodeInfo};
use crate::{
historystore::{HgIdHistoryStore, RemoteHistoryStore},
types::StoreKey,
unionstore::UnionStore,
};
@ -31,7 +32,7 @@ impl<T: HgIdHistoryStore> HgIdHistoryStore for UnionHgIdHistoryStore<T> {
}
impl<T: RemoteHistoryStore> RemoteHistoryStore for UnionHgIdHistoryStore<T> {
fn prefetch(&self, keys: &[Key]) -> Result<()> {
fn prefetch(&self, keys: &[StoreKey]) -> Result<()> {
let initial_keys = Ok(keys.to_vec());
self.into_iter()
.fold(initial_keys, |missing_keys, store| match missing_keys {
@ -57,7 +58,7 @@ mod tests {
use quickcheck::quickcheck;
use thiserror::Error;
use crate::localstore::HgIdLocalStore;
use crate::{localstore::LocalStore, types::StoreKey};
struct BadHgIdHistoryStore;
@ -73,8 +74,8 @@ mod tests {
}
}
impl HgIdLocalStore for EmptyHgIdHistoryStore {
fn get_missing(&self, keys: &[Key]) -> Result<Vec<Key>> {
impl LocalStore for EmptyHgIdHistoryStore {
fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
Ok(keys.iter().cloned().collect())
}
}
@ -85,8 +86,8 @@ mod tests {
}
}
impl HgIdLocalStore for BadHgIdHistoryStore {
fn get_missing(&self, _keys: &[Key]) -> Result<Vec<Key>> {
impl LocalStore for BadHgIdHistoryStore {
fn get_missing(&self, _keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
Err(BadHgIdHistoryStoreError.into())
}
}
@ -117,17 +118,17 @@ mod tests {
}
}
fn test_empty_unionstore_get_missing(keys: Vec<Key>) -> bool {
fn test_empty_unionstore_get_missing(keys: Vec<StoreKey>) -> bool {
keys == UnionHgIdHistoryStore::<EmptyHgIdHistoryStore>::new().get_missing(&keys).unwrap()
}
fn test_empty_historystore_get_missing(keys: Vec<Key>) -> bool {
fn test_empty_historystore_get_missing(keys: Vec<StoreKey>) -> bool {
let mut unionstore = UnionHgIdHistoryStore::new();
unionstore.add(EmptyHgIdHistoryStore);
keys == unionstore.get_missing(&keys).unwrap()
}
fn test_bad_historystore_get_missing(keys: Vec<Key>) -> bool {
fn test_bad_historystore_get_missing(keys: Vec<StoreKey>) -> bool {
let mut unionstore = UnionHgIdHistoryStore::new();
unionstore.add(BadHgIdHistoryStore);
match unionstore.get_missing(&keys) {

6
eden/scm/lib/revisionstore/src/unionstore.rs
View File

@ -13,7 +13,7 @@ use anyhow::Result;
use types::Key;
use crate::{localstore::HgIdLocalStore, repack::ToKeys};
use crate::{localstore::LocalStore, repack::ToKeys, types::StoreKey};
pub struct UnionStore<T> {
stores: Vec<T>,
@ -29,8 +29,8 @@ impl<T> UnionStore<T> {
}
}
impl<T: HgIdLocalStore> HgIdLocalStore for UnionStore<T> {
fn get_missing(&self, keys: &[Key]) -> Result<Vec<Key>> {
impl<T: LocalStore> LocalStore for UnionStore<T> {
fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
let initial_keys = Ok(keys.iter().cloned().collect());
self.into_iter()
.fold(initial_keys, |missing_keys, store| match missing_keys {