facebook/sapling
1
1
Fork 0
mirror of https://github.com/facebook/sapling.git synced 2024-12-27 23:22:02 +03:00
Code Issues Projects Releases Wiki Activity

revisionstore: add a new StoreResult type

Browse Source 
Summary:
When using LFS, it's possible that a pointer may be present in the local
LfsStore, but the blob would only be in the shared one. Such scenario can
happen after an upload, when the blob is moved to the shared store for
instance. In this case, during a `get` call, the local LFS store won't be able
to find the blob and thus would return Ok(None), the shared LFS store woud not
be able to find the pointer itself and would thus return Ok(None) too. If the
server is not aware of the file node itself, the `ContentStore::get` would also
return Ok(None), even though all the information is present locally.

The main reason why this is happening is due to the `get` call operating
primarily on file node based keys, and for content-based stores (like LFS),
this means that the translation layer needs to be present in the same store,
which in some case may not be the case. By allowing stores to return a
`StoreKey` when progress was made in finding the key we can effectively solve
the problem described above, the local store would translate the file node key
onto a content key, and the shared store would read the blob properly.

Reviewed By: DurhamG

Differential Revision: D22565607

fbshipit-source-id: 94dd74a462526778f7a7e232a97b21211f95239f
This commit is contained in:
Xavier Deguillard 2020-07-24 10:44:00 -07:00 committed by Facebook GitHub Bot
parent 41cc264acd
commit 3a97764d70
27 changed files with 602 additions and 401 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
eden/scm/edenscmnative/bindings/modules/pymanifest/src/lib.rs
View File

@ -20,7 +20,7 @@ use manifest_tree::TreeManifest;
use pathmatcher::{AlwaysMatcher, Matcher, TreeMatcher};
use pypathmatcher::PythonMatcher;
use pyrevisionstore::PythonHgIdDataStore;
use revisionstore::{HgIdDataStore, RemoteDataStore, StoreKey};
use revisionstore::{HgIdDataStore, RemoteDataStore, StoreKey, StoreResult};
use types::{Key, Node, RepoPath, RepoPathBuf};
type Result<T, E = Error> = std::result::Result<T, E>;
@ -38,10 +38,10 @@ impl<T> ManifestStore<T> {
impl<T: HgIdDataStore + RemoteDataStore> manifest_tree::TreeStore for ManifestStore<T> {
fn get(&self, path: &RepoPath, node: Node) -> Result<Bytes> {
let key = Key::new(path.to_owned(), node);
self.underlying
.get(&key)?
.ok_or_else(|| format_err!("Key {:?} not found in manifest", key))
.map(|data| Bytes::from(data))
match self.underlying.get(StoreKey::hgid(key))? {
StoreResult::NotFound(key) => Err(format_err!("Key {:?} not found in manifest", key)),
StoreResult::Found(data) => Ok(Bytes::from(data)),
}
}
fn insert(&self, _path: &RepoPath, _node: Node, _data: Bytes) -> Result<()> {

81
eden/scm/edenscmnative/bindings/modules/pyrevisionstore/src/datastorepyext.rs
View File

@ -15,8 +15,9 @@ use cpython::{
use cpython_ext::{PyPath, PyPathBuf, ResultPyErrExt};
use revisionstore::{
datastore::Delta, ContentDataStore, ContentHash, HgIdDataStore, HgIdMutableDeltaStore,
RemoteDataStore, StoreKey, ToKeys,
datastore::{Delta, StoreResult},
ContentDataStore, ContentHash, HgIdDataStore, HgIdMutableDeltaStore, RemoteDataStore, StoreKey,
ToKeys,
};
use types::Node;
@ -62,22 +63,23 @@ pub trait RemoteDataStorePyExt: RemoteDataStore {
impl<T: HgIdDataStore + ?Sized> HgIdDataStorePyExt for T {
fn get_py(&self, py: Python, name: &PyPath, node: &PyBytes) -> PyResult<PyBytes> {
let key = to_key(py, name, node)?;
let result = py
.allow_threads(|| self.get(&key))
.map_pyerr(py)?
.ok_or_else(|| key_error(py, &key))?;
Ok(PyBytes::new(py, &result[..]))
let key = StoreKey::hgid(to_key(py, name, node)?);
let result = py.allow_threads(|| self.get(key)).map_pyerr(py)?;
match result {
StoreResult::Found(data) => Ok(PyBytes::new(py, &data[..])),
StoreResult::NotFound(key) => Err(key_error(py, &key)),
}
}
fn get_delta_py(&self, py: Python, name: &PyPath, node: &PyBytes) -> PyResult<PyObject> {
let key = to_key(py, name, node)?;
let storekey = StoreKey::hgid(key.clone());
let data = py
.allow_threads(|| self.get(&key))
.map_pyerr(py)?
.ok_or_else(|| key_error(py, &key))?;
let res = py.allow_threads(|| self.get(storekey)).map_pyerr(py)?;
let data = match res {
StoreResult::Found(data) => data,
StoreResult::NotFound(key) => return Err(key_error(py, &key)),
};
let delta = Delta {
data: data.into(),
@ -102,11 +104,14 @@ impl<T: HgIdDataStore + ?Sized> HgIdDataStorePyExt for T {
fn get_delta_chain_py(&self, py: Python, name: &PyPath, node: &PyBytes) -> PyResult<PyList> {
let key = to_key(py, name, node)?;
let storekey = StoreKey::hgid(key.clone());
let data = py
.allow_threads(|| self.get(&key))
.map_pyerr(py)?
.ok_or_else(|| key_error(py, &key))?;
let res = py.allow_threads(|| self.get(storekey)).map_pyerr(py)?;
let data = match res {
StoreResult::Found(data) => data,
StoreResult::NotFound(key) => return Err(key_error(py, &key)),
};
let delta = Delta {
data: data.into(),
@ -124,11 +129,13 @@ impl<T: HgIdDataStore + ?Sized> HgIdDataStorePyExt for T {
}
fn get_meta_py(&self, py: Python, name: &PyPath, node: &PyBytes) -> PyResult<PyDict> {
let key = to_key(py, name, node)?;
let metadata = py
.allow_threads(|| self.get_meta(&key))
.map_pyerr(py)?
.ok_or_else(|| key_error(py, &key))?;
let key = StoreKey::hgid(to_key(py, name, node)?);
let res = py.allow_threads(|| self.get_meta(key)).map_pyerr(py)?;
let metadata = match res {
StoreResult::Found(metadata) => metadata,
StoreResult::NotFound(key) => return Err(key_error(py, &key)),
};
let metadict = PyDict::new(py);
if let Some(size) = metadata.size {
@ -184,20 +191,22 @@ impl<T: HgIdDataStore + ?Sized> HgIdDataStorePyExt for T {
impl<T: ContentDataStore + ?Sized> ContentDataStorePyExt for T {
fn blob_py(&self, py: Python, name: &PyPath, node: &PyBytes) -> PyResult<PyBytes> {
let key = to_key(py, name, node)?;
let result = py
.allow_threads(|| self.blob(&StoreKey::from(&key)))
.map_pyerr(py)?
.ok_or_else(|| key_error(py, &key))?;
Ok(PyBytes::new(py, result.as_ref()))
let key = StoreKey::hgid(to_key(py, name, node)?);
let res = py.allow_threads(|| self.blob(key)).map_pyerr(py)?;
match res {
StoreResult::Found(blob) => Ok(PyBytes::new(py, blob.as_ref())),
StoreResult::NotFound(key) => Err(key_error(py, &key)),
}
}
fn metadata_py(&self, py: Python, name: &PyPath, node: &PyBytes) -> PyResult<PyDict> {
let key = to_key(py, name, node)?;
let meta = py
.allow_threads(|| self.metadata(&StoreKey::from(&key)))
.map_pyerr(py)?
.ok_or_else(|| key_error(py, &key))?;
let key = StoreKey::hgid(to_key(py, name, node)?);
let res = py.allow_threads(|| self.metadata(key)).map_pyerr(py)?;
let meta = match res {
StoreResult::Found(meta) => meta,
StoreResult::NotFound(key) => return Err(key_error(py, &key)),
};
let metadict = PyDict::new(py);
metadict.set_item(py, "size", meta.size)?;
@ -216,7 +225,11 @@ impl<T: ToKeys + HgIdDataStore + ?Sized> IterableHgIdDataStorePyExt for T {
fn iter_py(&self, py: Python) -> PyResult<Vec<PyTuple>> {
let iter = py.allow_threads(|| self.to_keys()).into_iter().map(|res| {
let key = res?;
let data = py.allow_threads(|| self.get(&key))?.unwrap();
let res = py.allow_threads(|| self.get(StoreKey::hgid(key.clone())))?;
let data = match res {
StoreResult::Found(data) => data,
StoreResult::NotFound(_) => return Err(format_err!("Key {:?} not found", key)),
};
let delta = Delta {
data: data.into(),
base: None,

2
eden/scm/edenscmnative/bindings/modules/pyrevisionstore/src/historystorepyext.rs
View File

@ -82,7 +82,7 @@ impl<T: HgIdHistoryStore + ?Sized> HgIdHistoryStorePyExt for T {
let info = py
.allow_threads(|| self.get_node_info(&key))
.map_pyerr(py)?
.ok_or_else(|| key_error(py, &key))?;
.ok_or_else(|| key_error(py, &StoreKey::hgid(key.clone())))?;
Ok(from_node_info(py, &key, &info))
}

19
eden/scm/edenscmnative/bindings/modules/pyrevisionstore/src/lib.rs
View File

@ -31,6 +31,7 @@ use revisionstore::{
HistoryPackVersion, IndexedLogHgIdDataStore, IndexedLogHgIdHistoryStore, IndexedlogRepair,
LocalStore, MemcacheStore, Metadata, MetadataStore, MetadataStoreBuilder, MutableDataPack,
MutableHistoryPack, RemoteDataStore, RemoteHistoryStore, RepackKind, RepackLocation, StoreKey,
StoreResult,
};
use types::{Key, NodeInfo};
@ -493,14 +494,14 @@ impl ExtractInnerRef for mutabledeltastore {
}
impl HgIdDataStore for mutabledeltastore {
fn get(&self, key: &Key) -> Result<Option<Vec<u8>>> {
fn get(&self, key: StoreKey) -> Result<StoreResult<Vec<u8>>> {
let gil = Python::acquire_gil();
let py = gil.python();
self.store(py).get(key)
}
fn get_meta(&self, key: &Key) -> Result<Option<Metadata>> {
fn get_meta(&self, key: StoreKey) -> Result<StoreResult<Metadata>> {
let gil = Python::acquire_gil();
let py = gil.python();
@ -703,17 +704,15 @@ impl RemoteDataStore for PyRemoteDataStore {
}
impl HgIdDataStore for PyRemoteDataStore {
fn get(&self, key: &Key) -> Result<Option<Vec<u8>>> {
let missing = self.translate_lfs_missing(&[StoreKey::hgid(key.clone())])?;
match self.prefetch(&missing) {
fn get(&self, key: StoreKey) -> Result<StoreResult<Vec<u8>>> {
match self.prefetch(&[key.clone()]) {
Ok(()) => self.0.inner.read().datastore.as_ref().unwrap().get(key),
Err(_) => Ok(None),
Err(_) => Ok(StoreResult::NotFound(key)),
}
}
fn get_meta(&self, key: &Key) -> Result<Option<Metadata>> {
let missing = self.translate_lfs_missing(&[StoreKey::hgid(key.clone())])?;
match self.prefetch(&missing) {
fn get_meta(&self, key: StoreKey) -> Result<StoreResult<Metadata>> {
match self.prefetch(&[key.clone()]) {
Ok(()) => self
.0
.inner
@ -722,7 +721,7 @@ impl HgIdDataStore for PyRemoteDataStore {
.as_ref()
.unwrap()
.get_meta(key),
Err(_) => Ok(None),
Err(_) => Ok(StoreResult::NotFound(key)),
}
}
}

25
eden/scm/edenscmnative/bindings/modules/pyrevisionstore/src/pythondatastore.rs
View File

@ -12,8 +12,7 @@ use cpython::{
};
use cpython_ext::{PyErr, PyPathBuf};
use revisionstore::{HgIdDataStore, LocalStore, Metadata, RemoteDataStore, StoreKey};
use types::Key;
use revisionstore::{HgIdDataStore, LocalStore, Metadata, RemoteDataStore, StoreKey, StoreResult};
use crate::pythonutil::{from_key_to_tuple, from_tuple_to_key, to_metadata};
@ -28,7 +27,12 @@ impl PythonHgIdDataStore {
}
impl HgIdDataStore for PythonHgIdDataStore {
fn get(&self, key: &Key) -> Result<Option<Vec<u8>>> {
fn get(&self, key: StoreKey) -> Result<StoreResult<Vec<u8>>> {
let key = match key {
StoreKey::HgId(key) => key,
contentkey => return Ok(StoreResult::NotFound(contentkey)),
};
let gil = Python::acquire_gil();
let py = gil.python();
let py_name = PyPathBuf::from(key.path.as_repo_path());
@ -41,7 +45,7 @@ impl HgIdDataStore for PythonHgIdDataStore {
Ok(data) => data,
Err(py_err) => {
if py_err.get_type(py) == exc::KeyError::type_object(py) {
return Ok(None);
return Ok(StoreResult::NotFound(StoreKey::hgid(key)));
} else {
return Err(PyErr::from(py_err).into());
}
@ -50,10 +54,15 @@ impl HgIdDataStore for PythonHgIdDataStore {
let py_bytes = PyBytes::extract(py, &py_data).map_err(|e| PyErr::from(e))?;
Ok(Some(py_bytes.data(py).to_vec()))
Ok(StoreResult::Found(py_bytes.data(py).to_vec()))
}
fn get_meta(&self, key: &Key) -> Result<Option<Metadata>> {
fn get_meta(&self, key: StoreKey) -> Result<StoreResult<Metadata>> {
let key = match key {
StoreKey::HgId(key) => key,
contentkey => return Ok(StoreResult::NotFound(contentkey)),
};
let gil = Python::acquire_gil();
let py = gil.python();
let py_name = PyPathBuf::from(key.path.as_repo_path());
@ -65,7 +74,7 @@ impl HgIdDataStore for PythonHgIdDataStore {
Ok(data) => data,
Err(py_err) => {
if py_err.get_type(py) == exc::KeyError::type_object(py) {
return Ok(None);
return Ok(StoreResult::NotFound(StoreKey::hgid(key)));
} else {
return Err(PyErr::from(py_err).into());
}
@ -74,7 +83,7 @@ impl HgIdDataStore for PythonHgIdDataStore {
let py_dict = PyDict::extract(py, &py_meta).map_err(|e| PyErr::from(e))?;
to_metadata(py, &py_dict)
.map_err(|e| PyErr::from(e).into())
.map(Some)
.map(StoreResult::Found)
}
}

7
eden/scm/edenscmnative/bindings/modules/pyrevisionstore/src/pythonutil.rs
View File

@ -11,7 +11,10 @@ use cpython::{
};
use cpython_ext::{PyPath, PyPathBuf, ResultPyErrExt};
use revisionstore::datastore::{Delta, Metadata};
use revisionstore::{
datastore::{Delta, Metadata},
StoreKey,
};
use types::{Key, Node, RepoPathBuf};
pub fn to_node(py: Python, node: &PyBytes) -> Node {
@ -116,6 +119,6 @@ pub fn to_metadata(py: Python, meta: &PyDict) -> PyResult<Metadata> {
})
}
pub fn key_error(py: Python, key: &Key) -> PyErr {
pub fn key_error(py: Python, key: &StoreKey) -> PyErr {
PyErr::new::<exc::KeyError, _>(py, format!("Key not found {:?}", key))
}

12
eden/scm/edenscmnative/bindings/modules/pyworker/src/lib.rs
View File

@ -23,7 +23,7 @@ use crossbeam::channel::{bounded, Receiver, Sender};
use cpython_ext::{ExtractInner, PyNone, PyPath, PyPathBuf, ResultPyErrExt, Str};
use pyrevisionstore::contentstore;
use revisionstore::{ContentStore, HgIdDataStore};
use revisionstore::{ContentStore, HgIdDataStore, StoreKey, StoreResult};
use types::{HgId, Key, RepoPath, RepoPathBuf};
use vfs::{UpdateFlag, VFS};
@ -154,10 +154,12 @@ fn update(
.get_file_content(&key)?
.ok_or_else(|| format_err!("Can't find key: {}", key))?;
let meta = state
.store
.get_meta(&key)?
.ok_or_else(|| format_err!("Can't find metadata for key: {}", key))?;
let meta = match state.store.get_meta(StoreKey::hgid(key))? {
StoreResult::NotFound(key) => {
return Err(format_err!("Can't find metadata for key: {:?}", key))
}
StoreResult::Found(meta) => meta,
};
if meta.is_lfs() {
bail!("LFS pointers cannot be deserialized properly yet");

6
eden/scm/lib/backingstore/src/backingstore.rs
View File

@ -17,7 +17,7 @@ use manifest::{List, Manifest};
use manifest_tree::TreeManifest;
use revisionstore::{
ContentStore, ContentStoreBuilder, EdenApiFileStore, EdenApiTreeStore, HgIdDataStore,
LocalStore, MemcacheStore, RemoteDataStore, StoreKey,
LocalStore, MemcacheStore, RemoteDataStore, StoreKey, StoreResult,
};
use std::path::Path;
use std::sync::Arc;
@ -76,7 +76,9 @@ impl BackingStore {
fn get_blob_impl(&self, key: Key) -> Result<Option<Vec<u8>>> {
// Return None for LFS blobs
// TODO: LFS support
if let Ok(Some(metadata)) = self.blobstore.get_meta(&key) {
if let Ok(StoreResult::Found(metadata)) =
self.blobstore.get_meta(StoreKey::hgid(key.clone()))
{
if metadata.is_lfs() {
return Ok(None);
}

11
eden/scm/lib/backingstore/src/remotestore.rs
View File

@ -8,10 +8,9 @@
use anyhow::Result;
use revisionstore::{
HgIdDataStore, HgIdMutableDeltaStore, HgIdMutableHistoryStore, HgIdRemoteStore, LocalStore,
Metadata, RemoteDataStore, RemoteHistoryStore, StoreKey,
Metadata, RemoteDataStore, RemoteHistoryStore, StoreKey, StoreResult,
};
use std::sync::Arc;
use types::Key;
// TODO: Once we have EdenAPI production ready, remove this.
pub struct FakeRemoteStore;
@ -45,11 +44,11 @@ impl RemoteDataStore for FakeRemoteDataStore {
}
impl HgIdDataStore for FakeRemoteDataStore {
fn get(&self, _key: &Key) -> Result<Option<Vec<u8>>> {
Ok(None)
fn get(&self, key: StoreKey) -> Result<StoreResult<Vec<u8>>> {
Ok(StoreResult::NotFound(key))
}
fn get_meta(&self, _key: &Key) -> Result<Option<Metadata>> {
Ok(None)
fn get_meta(&self, key: StoreKey) -> Result<StoreResult<Metadata>> {
Ok(StoreResult::NotFound(key))
}
}

14
eden/scm/lib/backingstore/src/treecontentstore.rs
View File

@ -8,7 +8,7 @@
use anyhow::{format_err, Result};
use bytes::Bytes;
use manifest_tree::TreeStore;
use revisionstore::{ContentStore, HgIdDataStore};
use revisionstore::{ContentStore, HgIdDataStore, StoreKey, StoreResult};
use types::{HgId, Key, RepoPath};
pub(crate) struct TreeContentStore {
@ -27,11 +27,15 @@ impl TreeContentStore {
impl TreeStore for TreeContentStore {
fn get(&self, path: &RepoPath, hgid: HgId) -> Result<Bytes> {
let key = Key::new(path.to_owned(), hgid);
let key = StoreKey::hgid(Key::new(path.to_owned(), hgid));
self.inner.get(&key).and_then(|opt| {
opt.ok_or_else(|| format_err!("hgid: {:?} path: {:?} is not found.", hgid, path))
.map(Into::into)
self.inner.get(key).and_then(|res| match res {
StoreResult::Found(data) => Ok(data.into()),
StoreResult::NotFound(_) => Err(format_err!(
"hgid: {:?} path: {:?} is not found.",
hgid,
path
)),
})
}

5
eden/scm/lib/hgcommands/src/commands.rs
View File

@ -21,7 +21,8 @@ use blackbox::{event::Event, json, SessionId};
use dynamicconfig::Generator;
use edenapi::EdenApiBlocking;
use revisionstore::{
CorruptionPolicy, DataPackStore, HgIdDataStore, IndexedLogHgIdDataStore, UnionHgIdDataStore,
CorruptionPolicy, DataPackStore, HgIdDataStore, IndexedLogHgIdDataStore, StoreKey, StoreResult,
UnionHgIdDataStore,
};
use types::{HgId, Key, RepoPathBuf};
@ -240,7 +241,7 @@ pub fn debugstore(opts: DebugstoreOpts, io: &mut IO, repo: Repo) -> Result<u8> {
unionstore.add(packstore);
unionstore.add(indexedstore);
let k = Key::new(path, hgid);
if let Some(content) = unionstore.get(&k)? {
if let StoreResult::Found(content) = unionstore.get(StoreKey::hgid(k))? {
io.write(content)?;
}
Ok(0)

14
eden/scm/lib/revisionstore/src/c_api.rs
View File

@ -24,7 +24,8 @@ use anyhow::Result;
use types::{HgId, Key, RepoPath};
use crate::datapack::DataPack;
use crate::datastore::HgIdDataStore;
use crate::datastore::{HgIdDataStore, StoreResult};
use crate::types::StoreKey;
use crate::uniondatastore::UnionHgIdDataStore;
pub struct DataPackUnion {
@ -111,10 +112,13 @@ impl DataPackUnion {
/// Lookup Key. If the key is missing, scan for changes in the pack files and try once more.
fn get(&mut self, key: &Key) -> Result<Option<Vec<u8>>> {
match self.store.get(key)? {
Some(data) => Ok(Some(data)),
None => match self.rescan_paths() {
ScanResult::ChangesDetected => self.store.get(key),
match self.store.get(StoreKey::hgid(key.clone()))? {
StoreResult::Found(data) => Ok(Some(data)),
StoreResult::NotFound(key) => match self.rescan_paths() {
ScanResult::ChangesDetected => match self.store.get(key)? {
StoreResult::Found(data) => Ok(Some(data)),
StoreResult::NotFound(_) => Ok(None),
},
ScanResult::NoChanges => Ok(None),
},
}

116
eden/scm/lib/revisionstore/src/contentstore.rs
View File

@ -23,7 +23,7 @@ use types::Key;
use crate::{
datastore::{
strip_metadata, ContentDataStore, ContentMetadata, Delta, HgIdDataStore,
HgIdMutableDeltaStore, Metadata, RemoteDataStore,
HgIdMutableDeltaStore, Metadata, RemoteDataStore, StoreResult,
},
indexedlogdatastore::IndexedLogHgIdDataStore,
lfs::{LfsMultiplexer, LfsRemote, LfsStore},
@ -67,7 +67,7 @@ impl ContentStore {
/// XXX: This should only be used on `ContentStore` that are storing actual
/// file content, tree stores should use the `get` method instead.
pub fn get_file_content(&self, key: &Key) -> Result<Option<Bytes>> {
if let Some(vec) = self.get(key)? {
if let StoreResult::Found(vec) = self.get(StoreKey::hgid(key.clone()))? {
let bytes = vec.into();
let (bytes, _) = strip_metadata(&bytes)?;
Ok(Some(bytes))
@ -107,11 +107,11 @@ impl ContentStore {
}
impl HgIdDataStore for ContentStore {
fn get(&self, key: &Key) -> Result<Option<Vec<u8>>> {
fn get(&self, key: StoreKey) -> Result<StoreResult<Vec<u8>>> {
self.datastore.get(key)
}
fn get_meta(&self, key: &Key) -> Result<Option<Metadata>> {
fn get_meta(&self, key: StoreKey) -> Result<StoreResult<Metadata>> {
self.datastore.get_meta(key)
}
}
@ -179,11 +179,11 @@ impl HgIdMutableDeltaStore for ContentStore {
impl ContentDataStore for ContentStore {
/// Fetch a raw blob from the LFS stores.
fn blob(&self, key: &StoreKey) -> Result<Option<Bytes>> {
fn blob(&self, key: StoreKey) -> Result<StoreResult<Bytes>> {
self.blob_stores.blob(key)
}
fn metadata(&self, key: &StoreKey) -> Result<Option<ContentMetadata>> {
fn metadata(&self, key: StoreKey) -> Result<StoreResult<ContentMetadata>> {
self.blob_stores.metadata(key)
}
}
@ -460,8 +460,8 @@ mod tests {
key: k1.clone(),
};
store.add(&delta, &Default::default())?;
let stored = store.get(&k1)?;
assert_eq!(stored.as_deref(), Some(delta.data.as_ref()));
let stored = store.get(StoreKey::hgid(k1))?;
assert_eq!(stored, StoreResult::Found(delta.data.as_ref().to_vec()));
Ok(())
}
@ -483,7 +483,8 @@ mod tests {
drop(store);
let store = ContentStore::new(&localdir, &config)?;
assert!(store.get(&k1)?.is_none());
let k1 = StoreKey::hgid(k1);
assert_eq!(store.get(k1.clone())?, StoreResult::NotFound(k1));
Ok(())
}
@ -503,8 +504,8 @@ mod tests {
};
store.add(&delta, &Default::default())?;
store.flush()?;
let stored = store.get(&k1)?;
assert_eq!(stored.as_deref(), Some(delta.data.as_ref()));
let stored = store.get(StoreKey::hgid(k1))?;
assert_eq!(stored, StoreResult::Found(delta.data.as_ref().to_vec()));
Ok(())
}
@ -527,8 +528,8 @@ mod tests {
drop(store);
let store = ContentStore::new(&localdir, &config)?;
let stored = store.get(&k1)?;
assert_eq!(stored.as_deref(), Some(delta.data.as_ref()));
let stored = store.get(StoreKey::hgid(k1))?;
assert_eq!(stored, StoreResult::Found(delta.data.as_ref().to_vec()));
Ok(())
}
@ -550,9 +551,9 @@ mod tests {
.local_path(&localdir)
.remotestore(Arc::new(remotestore))
.build()?;
let data_get = store.get(&k)?;
let data_get = store.get(StoreKey::hgid(k))?;
assert_eq!(data_get.unwrap(), data);
assert_eq!(data_get, StoreResult::Found(data.as_ref().to_vec()));
Ok(())
}
@ -575,13 +576,13 @@ mod tests {
.local_path(&localdir)
.remotestore(Arc::new(remotestore))
.build()?;
store.get(&k)?;
store.get(StoreKey::hgid(k.clone()))?;
drop(store);
let store = ContentStore::new(&localdir, &config)?;
let data_get = store.get(&k)?;
let data_get = store.get(StoreKey::hgid(k))?;
assert_eq!(data_get.unwrap(), data);
assert_eq!(data_get, StoreResult::Found(data.as_ref().to_vec()));
Ok(())
}
@ -601,8 +602,8 @@ mod tests {
.remotestore(Arc::new(remotestore))
.build()?;
let k = key("a", "1");
assert_eq!(store.get(&k)?, None);
let k = StoreKey::hgid(key("a", "1"));
assert_eq!(store.get(k.clone())?, StoreResult::NotFound(k));
Ok(())
}
@ -625,14 +626,17 @@ mod tests {
.local_path(&localdir)
.remotestore(Arc::new(remotestore))
.build()?;
store.get(&k)?;
store.shared_mutabledatastore.get(&k)?;
assert!(store
store.get(StoreKey::hgid(k.clone()))?;
store
.shared_mutabledatastore
.get(StoreKey::hgid(k.clone()))?;
let k = StoreKey::hgid(k);
let res = store
.local_mutabledatastore
.as_ref()
.unwrap()
.get(&k)?
.is_none());
.get(k.clone())?;
assert_eq!(res, StoreResult::NotFound(k));
Ok(())
}
@ -654,7 +658,8 @@ mod tests {
store.flush()?;
let store = ContentStoreBuilder::new(&config).no_local_store().build()?;
assert_eq!(store.get(&k1)?, None);
let k = StoreKey::hgid(k1);
assert_eq!(store.get(k.clone())?, StoreResult::NotFound(k));
Ok(())
}
@ -683,7 +688,10 @@ mod tests {
lfs_store.flush()?;
let store = ContentStore::new(&localdir, &config)?;
assert_eq!(store.get(&k1)?, Some(delta.data.as_ref().to_vec()));
assert_eq!(
store.get(StoreKey::hgid(k1))?,
StoreResult::Found(delta.data.as_ref().to_vec())
);
Ok(())
}
@ -707,7 +715,10 @@ mod tests {
lfs_store.flush()?;
let store = ContentStore::new(&localdir, &config)?;
assert_eq!(store.get(&k1)?, Some(delta.data.as_ref().to_vec()));
assert_eq!(
store.get(StoreKey::hgid(k1))?,
StoreResult::Found(delta.data.as_ref().to_vec())
);
Ok(())
}
@ -727,8 +738,8 @@ mod tests {
let store = ContentStore::new(&localdir, &config)?;
store.add(&delta, &Default::default())?;
let blob = store.blob(&StoreKey::from(k1))?;
assert_eq!(blob, Some(delta.data));
let blob = store.blob(StoreKey::from(k1))?;
assert_eq!(blob, StoreResult::Found(delta.data));
Ok(())
}
@ -751,10 +762,10 @@ mod tests {
let store = ContentStore::new(&localdir, &config)?;
store.add(&delta, &Default::default())?;
let metadata = store.metadata(&StoreKey::from(k1))?;
let metadata = store.metadata(StoreKey::from(k1))?;
assert_eq!(
metadata,
Some(ContentMetadata {
StoreResult::Found(ContentMetadata {
size: 5,
is_binary: false,
hash,
@ -782,8 +793,8 @@ mod tests {
store.flush()?;
let lfs_store = LfsStore::local(&localdir, &config)?;
let stored = lfs_store.get(&k1)?;
assert_eq!(stored.as_deref(), Some(delta.data.as_ref()));
let stored = lfs_store.get(StoreKey::hgid(k1))?;
assert_eq!(stored, StoreResult::Found(delta.data.as_ref().to_vec()));
Ok(())
}
@ -816,8 +827,8 @@ mod tests {
)?;
let store = Arc::new(ContentStore::new(&localdir, &config)?);
let stored = store.get(&k1)?;
assert_eq!(stored.as_deref(), Some(delta.data.as_ref()));
let stored = store.get(StoreKey::hgid(k1))?;
assert_eq!(stored, StoreResult::Found(delta.data.as_ref().to_vec()));
Ok(())
}
@ -859,9 +870,12 @@ mod tests {
.remotestore(Arc::new(remotestore))
.build()?;
let data = store.get(&k)?.map(|vec| Bytes::from(vec));
let data = store.get(StoreKey::hgid(k))?;
assert_eq!(data, Some(Bytes::from(&b"master"[..])));
assert_eq!(
data,
StoreResult::Found(Bytes::from(&b"master"[..]).as_ref().to_vec())
);
Ok(())
}
@ -959,8 +973,8 @@ mod tests {
.remotestore(Arc::new(remotestore))
.memcachestore(memcache.clone())
.build()?;
let data_get = store.get(&k)?;
assert_eq!(data_get.unwrap(), data);
let data_get = store.get(StoreKey::hgid(k.clone()))?;
assert_eq!(data_get, StoreResult::Found(data.as_ref().to_vec()));
loop {
let memcache_data = memcache
@ -1000,8 +1014,8 @@ mod tests {
.remotestore(Arc::new(remotestore))
.memcachestore(memcache.clone())
.build()?;
let data_get = store.get(&k)?;
assert_eq!(data_get.unwrap(), data);
let data_get = store.get(StoreKey::hgid(k.clone()))?;
assert_eq!(data_get, StoreResult::Found(data.as_ref().to_vec()));
let memcache_data = memcache.get_data_iter(&[k]).collect::<Result<Vec<_>>>()?;
assert_eq!(memcache_data, vec![]);
@ -1036,8 +1050,8 @@ mod tests {
.remotestore(Arc::new(remotestore))
.memcachestore(memcache)
.build()?;
let data_get = store.get(&k)?;
assert_eq!(data_get.unwrap(), data);
let data_get = store.get(StoreKey::hgid(k))?;
assert_eq!(data_get, StoreResult::Found(data.as_ref().to_vec()));
// Ideally, we should check that we didn't wait for memcache, but that's timing
// related and thus a bit hard to test.
@ -1079,8 +1093,11 @@ mod tests {
.memcachestore(memcache.clone())
.build()?;
let data = store.get(&k)?.map(Bytes::from);
assert_eq!(data, Some(Bytes::from(&b"master"[..])));
let data = store.get(StoreKey::hgid(k.clone()))?;
assert_eq!(
data,
StoreResult::Found(Bytes::from(&b"master"[..]).as_ref().to_vec())
);
loop {
let memcache_data = memcache
@ -1102,8 +1119,11 @@ mod tests {
.memcachestore(memcache)
.build()?;
let data = store.get(&k)?.map(Bytes::from);
assert_eq!(data, Some(Bytes::from(&b"master"[..])));
let data = store.get(StoreKey::hgid(k))?;
assert_eq!(
data,
StoreResult::Found(Bytes::from(&b"master"[..]).as_ref().to_vec())
);
Ok(())
}

43
eden/scm/lib/revisionstore/src/datapack.rs
View File

@ -102,7 +102,7 @@ use util::path::remove_file;
use crate::{
dataindex::{DataIndex, DeltaBaseOffset},
datastore::{Delta, HgIdDataStore, Metadata},
datastore::{Delta, HgIdDataStore, Metadata, StoreResult},
localstore::LocalStore,
repack::{Repackable, ToKeys},
sliceext::SliceExt,
@ -351,16 +351,21 @@ impl DataPack {
}
impl HgIdDataStore for DataPack {
fn get(&self, key: &Key) -> Result<Option<Vec<u8>>> {
let delta_chain = self.get_delta_chain(key)?;
fn get(&self, key: StoreKey) -> Result<StoreResult<Vec<u8>>> {
let key = match key {
StoreKey::HgId(key) => key,
content => return Ok(StoreResult::NotFound(content)),
};
let delta_chain = self.get_delta_chain(&key)?;
let delta_chain = match delta_chain {
Some(chain) => chain,
None => return Ok(None),
None => return Ok(StoreResult::NotFound(StoreKey::hgid(key))),
};
let (basetext, deltas) = match delta_chain.split_last() {
Some((base, delta)) => (base, delta),
None => return Ok(None),
None => return Ok(StoreResult::NotFound(StoreKey::hgid(key))),
};
let deltas: Vec<&[u8]> = deltas
@ -369,17 +374,22 @@ impl HgIdDataStore for DataPack {
.map(|delta| delta.data.as_ref())
.collect();
Ok(Some(
Ok(StoreResult::Found(
get_full_text(basetext.data.as_ref(), &deltas).map_err(Error::msg)?,
))
}
fn get_meta(&self, key: &Key) -> Result<Option<Metadata>> {
fn get_meta(&self, key: StoreKey) -> Result<StoreResult<Metadata>> {
let key = match key {
StoreKey::HgId(key) => key,
content => return Ok(StoreResult::NotFound(content)),
};
let index_entry = match self.index.get_entry(&key.hgid)? {
None => return Ok(None),
None => return Ok(StoreResult::NotFound(StoreKey::hgid(key))),
Some(entry) => entry,
};
Ok(Some(
Ok(StoreResult::Found(
self.read_entry(index_entry.pack_entry_offset())?.metadata,
))
}
@ -541,9 +551,9 @@ pub mod tests {
];
let pack = make_datapack(&tempdir, &revisions);
for &(ref delta, ref metadata) in revisions.iter() {
let meta = pack.get_meta(&delta.key).unwrap().unwrap();
assert_eq!(&meta, metadata);
for (delta, metadata) in revisions {
let meta = pack.get_meta(StoreKey::hgid(delta.key)).unwrap();
assert_eq!(meta, StoreResult::Found(metadata));
}
}
@ -721,8 +731,13 @@ pub mod tests {
)];
let pack = Rc::new(make_datapack(&tempdir, &revisions));
let data = pack.get(&revisions[0].0.key).unwrap();
assert_eq!(data.as_deref(), Some(revisions[0].0.data.as_ref()));
let data = pack
.get(StoreKey::hgid(revisions[0].0.key.clone()))
.unwrap();
assert_eq!(
data,
StoreResult::Found(revisions[0].0.data.as_ref().to_vec())
);
}
quickcheck! {

22
eden/scm/lib/revisionstore/src/datastore.rs
View File

@ -31,9 +31,15 @@ pub struct Delta {
pub key: Key,
}
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)]
pub enum StoreResult<T> {
Found(T),
NotFound(StoreKey),
}
pub trait HgIdDataStore: LocalStore + Send + Sync {
fn get(&self, key: &Key) -> Result<Option<Vec<u8>>>;
fn get_meta(&self, key: &Key) -> Result<Option<Metadata>>;
fn get(&self, key: StoreKey) -> Result<StoreResult<Vec<u8>>>;
fn get_meta(&self, key: StoreKey) -> Result<StoreResult<Metadata>>;
}
/// The `RemoteDataStore` trait indicates that data can fetched over the network. Care must be
@ -81,21 +87,21 @@ pub struct ContentMetadata {
pub trait ContentDataStore: Send + Sync {
/// Read the blob from the store, the blob returned is the pure blob and will not contain any
/// Mercurial copy_from header.
fn blob(&self, key: &StoreKey) -> Result<Option<Bytes>>;
fn blob(&self, key: StoreKey) -> Result<StoreResult<Bytes>>;
/// Read the blob metadata from the store.
fn metadata(&self, key: &StoreKey) -> Result<Option<ContentMetadata>>;
fn metadata(&self, key: StoreKey) -> Result<StoreResult<ContentMetadata>>;
// XXX: Add write operations.
}
/// Implement `HgIdDataStore` for all types that can be `Deref` into a `HgIdDataStore`. This includes all
/// the smart pointers like `Box`, `Rc`, `Arc`.
impl<T: HgIdDataStore + ?Sized, U: Deref<Target = T> + Send + Sync> HgIdDataStore for U {
fn get(&self, key: &Key) -> Result<Option<Vec<u8>>> {
fn get(&self, key: StoreKey) -> Result<StoreResult<Vec<u8>>> {
T::get(self, key)
}
fn get_meta(&self, key: &Key) -> Result<Option<Metadata>> {
fn get_meta(&self, key: StoreKey) -> Result<StoreResult<Metadata>> {
T::get_meta(self, key)
}
}
@ -126,11 +132,11 @@ impl<T: HgIdMutableDeltaStore + ?Sized, U: Deref<Target = T> + Send + Sync> HgId
/// Implement `ContentDataStore` for all types that can be `Deref` into a `ContentDataStore`.
impl<T: ContentDataStore + ?Sized, U: Deref<Target = T> + Send + Sync> ContentDataStore for U {
fn blob(&self, key: &StoreKey) -> Result<Option<Bytes>> {
fn blob(&self, key: StoreKey) -> Result<StoreResult<Bytes>> {
T::blob(self, key)
}
fn metadata(&self, key: &StoreKey) -> Result<Option<ContentMetadata>> {
fn metadata(&self, key: StoreKey) -> Result<StoreResult<ContentMetadata>> {
T::metadata(self, key)
}
}

82
eden/scm/lib/revisionstore/src/edenapi/data.rs
View File

@ -10,10 +10,8 @@ use std::sync::Arc;
use anyhow::Result;
use futures::prelude::*;
use types::Key;
use crate::{
datastore::{HgIdDataStore, HgIdMutableDeltaStore, Metadata, RemoteDataStore},
datastore::{HgIdDataStore, HgIdMutableDeltaStore, Metadata, RemoteDataStore, StoreResult},
localstore::LocalStore,
types::StoreKey,
};
@ -64,13 +62,13 @@ impl<T: EdenApiStoreKind> RemoteDataStore for EdenApiDataStore<T> {
}
impl<T: EdenApiStoreKind> HgIdDataStore for EdenApiDataStore<T> {
fn get(&self, key: &Key) -> Result<Option<Vec<u8>>> {
self.prefetch(&[StoreKey::hgid(key.clone())])?;
fn get(&self, key: StoreKey) -> Result<StoreResult<Vec<u8>>> {
self.prefetch(&[key.clone()])?;
self.store.get(key)
}
fn get_meta(&self, key: &Key) -> Result<Option<Metadata>> {
self.prefetch(&[StoreKey::hgid(key.clone())])?;
fn get_meta(&self, key: StoreKey) -> Result<StoreResult<Metadata>> {
self.prefetch(&[key.clone()])?;
self.store.get_meta(key)
}
}
@ -119,16 +117,29 @@ mod tests {
let edenapi_files = remote_files.datastore(local.clone());
// Attempt fetch.
let data = edenapi_files.get(&k)?.expect("data not found");
let meta = edenapi_files.get_meta(&k)?.expect("metadata not found");
assert_eq!(&data, &d.data);
assert_eq!(meta.size, Some(d.data.len() as u64));
let k = StoreKey::hgid(k);
let data = edenapi_files.get(k.clone())?;
let meta = edenapi_files.get_meta(k.clone())?;
assert_eq!(data, StoreResult::Found(d.data.as_ref().to_vec()));
assert_eq!(
meta,
StoreResult::Found(Metadata {
size: Some(d.data.len() as u64),
flags: None
})
);
// Check that data was written to the local store.
let data = local.get(&k)?.expect("data not found");
let meta = local.get_meta(&k)?.expect("metadata not found");
assert_eq!(&data, &d.data);
assert_eq!(meta.size, Some(d.data.len() as u64));
let data = local.get(k.clone())?;
let meta = local.get_meta(k.clone())?;
assert_eq!(data, StoreResult::Found(d.data.as_ref().to_vec()));
assert_eq!(
meta,
StoreResult::Found(Metadata {
size: Some(d.data.len() as u64),
flags: None
})
);
// Using the same mock client, set up a store for trees.
// Need to use a new local store since otherwise the key
@ -137,8 +148,8 @@ mod tests {
let local = Arc::new(IndexedLogHgIdDataStore::new(&tmp)?);
let edenapi_trees = remote_trees.datastore(local.clone());
// Check that the same key is not fetched by the tree store.
assert_eq!(edenapi_trees.get(&k)?, None);
// Check that the same key cannot be accessed via the tree store.
assert_eq!(edenapi_trees.get(k.clone())?, StoreResult::NotFound(k));
Ok(())
}
@ -163,16 +174,29 @@ mod tests {
let edenapi_trees = remote_trees.datastore(local.clone());
// Attempt fetch.
let data = edenapi_trees.get(&k)?.expect("data not found");
let meta = edenapi_trees.get_meta(&k)?.expect("metadata not found");
assert_eq!(&data, &d.data);
assert_eq!(meta.size, Some(d.data.len() as u64));
let k = StoreKey::hgid(k);
let data = edenapi_trees.get(k.clone())?;
let meta = edenapi_trees.get_meta(k.clone())?;
assert_eq!(data, StoreResult::Found(d.data.as_ref().to_vec()));
assert_eq!(
meta,
StoreResult::Found(Metadata {
size: Some(d.data.len() as u64),
flags: None
})
);
// Check that data was written to the local store.
let data = local.get(&k)?.expect("data not found");
let meta = local.get_meta(&k)?.expect("metadata not found");
assert_eq!(&data, &d.data);
assert_eq!(meta.size, Some(d.data.len() as u64));
let data = local.get(k.clone())?;
let meta = local.get_meta(k.clone())?;
assert_eq!(data, StoreResult::Found(d.data.as_ref().to_vec()));
assert_eq!(
meta,
StoreResult::Found(Metadata {
size: Some(d.data.len() as u64),
flags: None
})
);
// Using the same mock client, set up a store for files.
// Need to use a new local store since otherwise the key
@ -181,8 +205,8 @@ mod tests {
let local = Arc::new(IndexedLogHgIdDataStore::new(&tmp)?);
let edenapi_files = remote_files.datastore(local);
// Check that the same key is not fetched by the files store.
assert_eq!(edenapi_files.get(&k)?, None);
// Check that the same key cannot be accessed via the file store.
assert_eq!(edenapi_files.get(k.clone())?, StoreResult::NotFound(k));
Ok(())
}
@ -200,8 +224,8 @@ mod tests {
// Set up `EdenApiDataStore`.
let edenapi = remote.datastore(store.clone());
let k = key("a", "1");
assert_eq!(edenapi.get(&k)?, None);
let k = StoreKey::hgid(key("a", "1"));
assert_eq!(edenapi.get(k.clone())?, StoreResult::NotFound(k));
Ok(())
}

35
eden/scm/lib/revisionstore/src/indexedlogdatastore.rs
View File

@ -24,7 +24,7 @@ use lz4_pyframe::{compress, decompress};
use types::{hgid::ReadHgIdExt, HgId, Key, RepoPath};
use crate::{
datastore::{Delta, HgIdDataStore, HgIdMutableDeltaStore, Metadata},
datastore::{Delta, HgIdDataStore, HgIdMutableDeltaStore, Metadata, StoreResult},
localstore::LocalStore,
repack::ToKeys,
sliceext::SliceExt,
@ -213,19 +213,34 @@ impl LocalStore for IndexedLogHgIdDataStore {
}
impl HgIdDataStore for IndexedLogHgIdDataStore {
fn get(&self, key: &Key) -> Result<Option<Vec<u8>>> {
fn get(&self, key: StoreKey) -> Result<StoreResult<Vec<u8>>> {
let key = match key {
StoreKey::HgId(key) => key,
content => return Ok(StoreResult::NotFound(content)),
};
let inner = self.inner.read();
let mut entry = match Entry::from_log(&key, &inner.log)? {
None => return Ok(None),
None => return Ok(StoreResult::NotFound(StoreKey::HgId(key))),
Some(entry) => entry,
};
let content = entry.content()?;
Ok(Some(content.as_ref().to_vec()))
Ok(StoreResult::Found(content.as_ref().to_vec()))
}
fn get_meta(&self, key: &Key) -> Result<Option<Metadata>> {
fn get_meta(&self, key: StoreKey) -> Result<StoreResult<Metadata>> {
let key = match key {
StoreKey::HgId(key) => key,
content => return Ok(StoreResult::NotFound(content)),
};
let inner = self.inner.read();
Ok(Entry::from_log(&key, &inner.log)?.map(|entry| entry.metadata().clone()))
let entry = match Entry::from_log(&key, &inner.log)? {
None => return Ok(StoreResult::NotFound(StoreKey::HgId(key))),
Some(entry) => entry,
};
Ok(StoreResult::Found(entry.metadata().clone()))
}
}
@ -291,8 +306,8 @@ mod tests {
log.flush().unwrap();
let log = IndexedLogHgIdDataStore::new(&tempdir).unwrap();
let read_data = log.get(&delta.key).unwrap();
assert_eq!(Some(delta.data.as_ref()), read_data.as_deref());
let read_data = log.get(StoreKey::hgid(delta.key)).unwrap();
assert_eq!(StoreResult::Found(delta.data.as_ref().to_vec()), read_data);
}
#[test]
@ -300,8 +315,8 @@ mod tests {
let tempdir = TempDir::new().unwrap();
let log = IndexedLogHgIdDataStore::new(&tempdir).unwrap();
let key = key("a", "1");
assert!(log.get(&key).unwrap().is_none());
let key = StoreKey::hgid(key("a", "1"));
assert_eq!(log.get(key.clone()).unwrap(), StoreResult::NotFound(key));
}
#[test]

180
eden/scm/lib/revisionstore/src/lfs.rs
View File

@ -50,7 +50,7 @@ use util::path::{create_dir, create_shared_dir, remove_file};
use crate::{
datastore::{
strip_metadata, ContentDataStore, ContentMetadata, Delta, HgIdDataStore,
HgIdMutableDeltaStore, Metadata, RemoteDataStore,
HgIdMutableDeltaStore, Metadata, RemoteDataStore, StoreResult,
},
historystore::{HgIdMutableHistoryStore, RemoteHistoryStore},
indexedlogutil::{Store, StoreOpenOptions},
@ -214,8 +214,8 @@ impl LfsPointersStore {
}
/// Find the pointer corresponding to the passed in `Key`.
fn get(&self, key: &Key) -> Result<Option<LfsPointersEntry>> {
self.entry(&StoreKey::from(key))
fn get(&self, key: &StoreKey) -> Result<Option<LfsPointersEntry>> {
self.entry(key)
}
fn add(&mut self, entry: LfsPointersEntry) -> Result<()> {
@ -530,17 +530,19 @@ impl LfsStore {
LfsStore::new(pointers, blobs)
}
fn blob_impl(&self, key: &StoreKey) -> Result<Option<(LfsPointersEntry, Bytes)>> {
let pointer = self.pointers.read().entry(key)?;
fn blob_impl(&self, key: StoreKey) -> Result<StoreResult<(LfsPointersEntry, Bytes)>> {
let pointer = self.pointers.read().entry(&key)?;
match pointer {
None => Ok(None),
None => Ok(StoreResult::NotFound(key)),
Some(entry) => match entry.content_hashes.get(&ContentHashType::Sha256) {
None => Ok(None),
Some(content_hash) => Ok(self
.blobs
.get(&content_hash.clone().unwrap_sha256())?
.map(|blob| (entry, blob))),
None => Ok(StoreResult::NotFound(key)),
Some(content_hash) => {
match self.blobs.get(&content_hash.clone().unwrap_sha256())? {
None => Ok(StoreResult::NotFound(key)),
Some(blob) => Ok(StoreResult::Found((entry, blob))),
}
}
},
}
}
@ -552,7 +554,7 @@ impl LocalStore for LfsStore {
.iter()
.filter_map(|k| match k {
StoreKey::HgId(key) => {
let entry = self.pointers.read().get(key);
let entry = self.pointers.read().get(&k.clone());
match entry {
Ok(None) | Err(_) => Some(k.clone()),
Ok(Some(entry)) => match entry.content_hashes.get(&ContentHashType::Sha256)
@ -639,26 +641,26 @@ fn rebuild_metadata(data: Bytes, entry: &LfsPointersEntry) -> Bytes {
}
impl HgIdDataStore for LfsStore {
fn get(&self, key: &Key) -> Result<Option<Vec<u8>>> {
match self.blob_impl(&StoreKey::from(key))? {
Some((entry, content)) => {
fn get(&self, key: StoreKey) -> Result<StoreResult<Vec<u8>>> {
match self.blob_impl(key)? {
StoreResult::Found((entry, content)) => {
let content = rebuild_metadata(content, &entry);
// PERF: Consider changing HgIdDataStore::get() to return Bytes to avoid copying data.
Ok(Some(content.as_ref().to_vec()))
Ok(StoreResult::Found(content.as_ref().to_vec()))
}
None => Ok(None),
StoreResult::NotFound(key) => Ok(StoreResult::NotFound(key)),
}
}
fn get_meta(&self, key: &Key) -> Result<Option<Metadata>> {
let entry = self.pointers.read().get(key)?;
fn get_meta(&self, key: StoreKey) -> Result<StoreResult<Metadata>> {
let entry = self.pointers.read().get(&key)?;
if let Some(entry) = entry {
Ok(Some(Metadata {
Ok(StoreResult::Found(Metadata {
size: Some(entry.size.try_into()?),
flags: None,
}))
} else {
Ok(None)
Ok(StoreResult::NotFound(key))
}
}
}
@ -707,14 +709,20 @@ impl From<LfsPointersEntry> for ContentMetadata {
}
impl ContentDataStore for LfsStore {
fn blob(&self, key: &StoreKey) -> Result<Option<Bytes>> {
Ok(self.blob_impl(key)?.map(|(_, blob)| blob))
fn blob(&self, key: StoreKey) -> Result<StoreResult<Bytes>> {
match self.blob_impl(key)? {
StoreResult::Found((_, blob)) => Ok(StoreResult::Found(blob)),
StoreResult::NotFound(key) => Ok(StoreResult::NotFound(key)),
}
}
fn metadata(&self, key: &StoreKey) -> Result<Option<ContentMetadata>> {
let pointer = self.pointers.read().entry(key)?;
fn metadata(&self, key: StoreKey) -> Result<StoreResult<ContentMetadata>> {
let pointer = self.pointers.read().entry(&key)?;
Ok(pointer.map(Into::into))
match pointer {
None => Ok(StoreResult::NotFound(key)),
Some(pointer_entry) => Ok(StoreResult::Found(pointer_entry.into())),
}
}
}
@ -740,11 +748,11 @@ impl LfsMultiplexer {
}
impl HgIdDataStore for LfsMultiplexer {
fn get(&self, key: &Key) -> Result<Option<Vec<u8>>> {
fn get(&self, key: StoreKey) -> Result<StoreResult<Vec<u8>>> {
self.union.get(key)
}
fn get_meta(&self, key: &Key) -> Result<Option<Metadata>> {
fn get_meta(&self, key: StoreKey) -> Result<StoreResult<Metadata>> {
self.union.get_meta(key)
}
}
@ -1447,11 +1455,14 @@ impl RemoteDataStore for LfsRemoteStore {
let size = size.clone();
move |sha256| {
let key = StoreKey::from(ContentHash::Sha256(sha256));
let opt = local_store.blob(&key)?;
if let Some(blob) = opt.as_ref() {
size.fetch_add(blob.len(), Ordering::Relaxed);
match local_store.blob(key)? {
StoreResult::Found(blob) => {
size.fetch_add(blob.len(), Ordering::Relaxed);
Ok(Some(blob))
}
StoreResult::NotFound(_) => Ok(None),
}
Ok(opt)
}
})?;
@ -1474,19 +1485,17 @@ impl RemoteDataStore for LfsRemoteStore {
}
impl HgIdDataStore for LfsRemoteStore {
fn get(&self, key: &Key) -> Result<Option<Vec<u8>>> {
let missing = self.translate_lfs_missing(&[StoreKey::hgid(key.clone())])?;
match self.prefetch(&missing) {
fn get(&self, key: StoreKey) -> Result<StoreResult<Vec<u8>>> {
match self.prefetch(&[key.clone()]) {
Ok(()) => self.store.get(key),
Err(_) => Ok(None),
Err(_) => Ok(StoreResult::NotFound(key)),
}
}
fn get_meta(&self, key: &Key) -> Result<Option<Metadata>> {
let missing = self.translate_lfs_missing(&[StoreKey::hgid(key.clone())])?;
match self.prefetch(&missing) {
fn get_meta(&self, key: StoreKey) -> Result<StoreResult<Metadata>> {
match self.prefetch(&[key.clone()]) {
Ok(()) => self.store.get_meta(key),
Err(_) => Ok(None),
Err(_) => Ok(StoreResult::NotFound(key)),
}
}
}
@ -1618,8 +1627,8 @@ mod tests {
};
store.add(&delta, &Default::default())?;
let stored = store.get(&k1)?;
assert_eq!(Some(delta.data.as_ref()), stored.as_deref());
let stored = store.get(StoreKey::hgid(k1))?;
assert_eq!(StoreResult::Found(delta.data.as_ref().to_vec()), stored);
Ok(())
}
@ -1640,13 +1649,14 @@ mod tests {
};
store.add(&delta, &Default::default())?;
let stored = store.get(&k1)?;
assert_eq!(Some(delta.data.as_ref()), stored.as_deref());
let k = StoreKey::hgid(k1);
let stored = store.get(k.clone())?;
assert_eq!(StoreResult::Found(delta.data.as_ref().to_vec()), stored);
store.flush()?;
let stored = store.get(&k1)?;
assert_eq!(Some(delta.data.as_ref()), stored.as_deref());
let stored = store.get(k)?;
assert_eq!(StoreResult::Found(delta.data.as_ref().to_vec()), stored);
Ok(())
}
@ -1802,8 +1812,8 @@ mod tests {
};
store.add(&delta, &Default::default())?;
let stored = store.get(&k1)?;
assert_eq!(Some(delta.data.as_ref()), stored.as_deref());
let stored = store.get(StoreKey::hgid(k1))?;
assert_eq!(StoreResult::Found(delta.data.as_ref().to_vec()), stored);
Ok(())
}
@ -1827,9 +1837,10 @@ mod tests {
};
multiplexer.add(&delta, &Default::default())?;
let stored = multiplexer.get(&k1)?;
assert_eq!(stored.as_deref(), Some(delta.data.as_ref()));
assert_eq!(indexedlog.get_missing(&[k1.into()])?, vec![]);
let k = StoreKey::hgid(k1);
let stored = multiplexer.get(k.clone())?;
assert_eq!(stored, StoreResult::Found(delta.data.as_ref().to_vec()));
assert_eq!(indexedlog.get_missing(&[k])?, vec![]);
Ok(())
}
@ -1853,11 +1864,12 @@ mod tests {
};
multiplexer.add(&delta, &Default::default())?;
let stored = multiplexer.get(&k1)?;
assert_eq!(stored.as_deref(), Some(delta.data.as_ref()));
let k = StoreKey::hgid(k1);
let stored = multiplexer.get(k.clone())?;
assert_eq!(stored, StoreResult::Found(delta.data.as_ref().to_vec()));
assert_eq!(
indexedlog.get_missing(&[StoreKey::from(&k1)])?,
vec![StoreKey::from(&k1)]
indexedlog.get_missing(&[k.clone()])?,
vec![StoreKey::from(k)]
);
Ok(())
@ -1898,17 +1910,18 @@ mod tests {
flags: Some(Metadata::LFS_FLAG),
},
)?;
assert_eq!(
indexedlog.get_missing(&[StoreKey::from(&k1)])?,
vec![StoreKey::from(&k1)]
);
let k = StoreKey::hgid(k1.clone());
assert_eq!(indexedlog.get_missing(&[k.clone()])?, vec![k.clone()]);
// The blob isn't present, so we cannot get it.
assert_eq!(multiplexer.get(&k1)?, None);
assert_eq!(
multiplexer.get(k.clone())?,
StoreResult::NotFound(k.clone())
);
multiplexer.flush()?;
let lfs = LfsStore::shared(&lfsdir, &config)?;
let entry = lfs.pointers.read().get(&k1)?;
let entry = lfs.pointers.read().get(&k)?;
assert!(entry.is_some());
@ -1964,17 +1977,18 @@ mod tests {
flags: Some(Metadata::LFS_FLAG),
},
)?;
assert_eq!(
indexedlog.get_missing(&[StoreKey::from(&k1)])?,
vec![StoreKey::from(&k1)]
);
let k = StoreKey::hgid(k1.clone());
assert_eq!(indexedlog.get_missing(&[k.clone()])?, vec![k.clone()]);
// The blob isn't present, so we cannot get it.
assert_eq!(multiplexer.get(&k1)?, None);
assert_eq!(
multiplexer.get(k.clone())?,
StoreResult::NotFound(k.clone())
);
multiplexer.flush()?;
let lfs = LfsStore::shared(&lfsdir, &config)?;
let entry = lfs.pointers.read().get(&k1)?;
let entry = lfs.pointers.read().get(&k.clone())?;
assert!(entry.is_some());
@ -2031,10 +2045,9 @@ mod tests {
},
)?;
let read_blob = multiplexer.get(&k1)?.map(|vec| Bytes::from(vec));
let expected_blob = Some(Bytes::from(
&b"\x01\n\x01\n\x01\nTHIS IS A BLOB WITH A HEADER"[..],
));
let read_blob = multiplexer.get(StoreKey::hgid(k1))?;
let expected_blob =
StoreResult::Found(b"\x01\n\x01\n\x01\nTHIS IS A BLOB WITH A HEADER".to_vec());
assert_eq!(read_blob, expected_blob);
Ok(())
@ -2322,8 +2335,11 @@ mod tests {
key: key.clone(),
};
let stored = remotedatastore.get(&key)?;
assert_eq!(stored.as_deref(), Some(expected_delta.data.as_ref()));
let stored = remotedatastore.get(StoreKey::hgid(key))?;
assert_eq!(
stored,
StoreResult::Found(expected_delta.data.as_ref().to_vec())
);
Ok(())
}
@ -2448,11 +2464,15 @@ mod tests {
&config,
)?);
let remote = remote.datastore(shared_lfs.clone());
remote.upload(&[StoreKey::from(&k1)])?;
let k = StoreKey::hgid(k1.clone());
remote.upload(&[k.clone()])?;
// The blob was moved from the local store to the shared store.
assert_eq!(local_lfs.get(&k1)?, None);
assert_eq!(shared_lfs.get(&k1)?, Some(delta.data.to_vec()));
assert_eq!(local_lfs.get(k.clone())?, StoreResult::NotFound(k.clone()));
assert_eq!(
shared_lfs.get(k.clone())?,
StoreResult::Found(delta.data.to_vec())
);
Ok(())
}
@ -2473,8 +2493,8 @@ mod tests {
store.add(&delta, &Default::default())?;
let blob = store.blob(&StoreKey::from(k1))?;
assert_eq!(blob, Some(delta.data));
let blob = store.blob(StoreKey::from(k1))?;
assert_eq!(blob, StoreResult::Found(delta.data));
Ok(())
}
@ -2496,10 +2516,10 @@ mod tests {
store.add(&delta, &Default::default())?;
let metadata = store.metadata(&StoreKey::from(k1))?;
let metadata = store.metadata(StoreKey::from(k1))?;
assert_eq!(
metadata,
Some(ContentMetadata {
StoreResult::Found(ContentMetadata {
size: 4,
is_binary: false,
hash,

3
eden/scm/lib/revisionstore/src/lib.rs
View File

@ -161,7 +161,8 @@ pub mod unionhistorystore;
pub use crate::contentstore::{ContentStore, ContentStoreBuilder};
pub use crate::datapack::{DataEntry, DataPack, DataPackVersion};
pub use crate::datastore::{
ContentDataStore, ContentMetadata, Delta, HgIdDataStore, HgIdMutableDeltaStore, RemoteDataStore,
ContentDataStore, ContentMetadata, Delta, HgIdDataStore, HgIdMutableDeltaStore,
RemoteDataStore, StoreResult,
};
pub use crate::edenapi::{EdenApiFileStore, EdenApiRemoteStore, EdenApiTreeStore};
pub use crate::historypack::{HistoryEntry, HistoryPack, HistoryPackVersion};

26
eden/scm/lib/revisionstore/src/memcache.rs
View File

@ -17,7 +17,9 @@ use tracing::info_span;
use types::{Key, NodeInfo};
use crate::{
datastore::{Delta, HgIdDataStore, HgIdMutableDeltaStore, Metadata, RemoteDataStore},
datastore::{
Delta, HgIdDataStore, HgIdMutableDeltaStore, Metadata, RemoteDataStore, StoreResult,
},
historystore::{HgIdHistoryStore, HgIdMutableHistoryStore, RemoteHistoryStore},
localstore::LocalStore,
remotestore::HgIdRemoteStore,
@ -82,12 +84,12 @@ pub use crate::facebook::MemcacheStore;
pub use dummy::MemcacheStore;
impl HgIdDataStore for MemcacheStore {
fn get(&self, _key: &Key) -> Result<Option<Vec<u8>>> {
Ok(None)
fn get(&self, key: StoreKey) -> Result<StoreResult<Vec<u8>>> {
Ok(StoreResult::NotFound(key))
}
fn get_meta(&self, _key: &Key) -> Result<Option<Metadata>> {
Ok(None)
fn get_meta(&self, key: StoreKey) -> Result<StoreResult<Metadata>> {
Ok(StoreResult::NotFound(key))
}
}
@ -153,19 +155,17 @@ impl MemcacheHgIdDataStore {
}
impl HgIdDataStore for MemcacheHgIdDataStore {
fn get(&self, key: &Key) -> Result<Option<Vec<u8>>> {
let missing = self.translate_lfs_missing(&[StoreKey::hgid(key.clone())])?;
match self.prefetch(&missing) {
fn get(&self, key: StoreKey) -> Result<StoreResult<Vec<u8>>> {
match self.prefetch(&[key.clone()]) {
Ok(()) => self.store.get(key),
Err(_) => Ok(None),
Err(_) => Ok(StoreResult::NotFound(key)),
}
}
fn get_meta(&self, key: &Key) -> Result<Option<Metadata>> {
let missing = self.translate_lfs_missing(&[StoreKey::hgid(key.clone())])?;
match self.prefetch(&missing) {
fn get_meta(&self, key: StoreKey) -> Result<StoreResult<Metadata>> {
match self.prefetch(&[key.clone()]) {
Ok(()) => self.store.get_meta(key),
Err(_) => Ok(None),
Err(_) => Ok(StoreResult::NotFound(key)),
}
}
}

33
eden/scm/lib/revisionstore/src/multiplexstore.rs
View File

@ -12,7 +12,7 @@ use anyhow::Result;
use types::{Key, NodeInfo};
use crate::{
datastore::{Delta, HgIdDataStore, HgIdMutableDeltaStore, Metadata},
datastore::{Delta, HgIdDataStore, HgIdMutableDeltaStore, Metadata, StoreResult},
historystore::{HgIdHistoryStore, HgIdMutableHistoryStore},
localstore::LocalStore,
types::StoreKey,
@ -75,24 +75,26 @@ impl<T: HgIdMutableDeltaStore> HgIdMutableDeltaStore for MultiplexDeltaStore<T>
}
impl<T: HgIdMutableDeltaStore> HgIdDataStore for MultiplexDeltaStore<T> {
fn get(&self, key: &Key) -> Result<Option<Vec<u8>>> {
fn get(&self, mut key: StoreKey) -> Result<StoreResult<Vec<u8>>> {
for store in self.stores.iter() {
if let Some(result) = store.get(key)? {
return Ok(Some(result));
match store.get(key)? {
StoreResult::Found(data) => return Ok(StoreResult::Found(data)),
StoreResult::NotFound(next) => key = next,
}
}
Ok(None)
Ok(StoreResult::NotFound(key))
}
fn get_meta(&self, key: &Key) -> Result<Option<Metadata>> {
fn get_meta(&self, mut key: StoreKey) -> Result<StoreResult<Metadata>> {
for store in self.stores.iter() {
if let Some(result) = store.get_meta(key)? {
return Ok(Some(result));
match store.get_meta(key)? {
StoreResult::Found(data) => return Ok(StoreResult::Found(data)),
StoreResult::NotFound(next) => key = next,
}
}
Ok(None)
Ok(StoreResult::NotFound(key))
}
}
@ -198,8 +200,8 @@ mod tests {
multiplex.add(&delta, &metadata)?;
drop(multiplex);
let read_data = log.get(&delta.key)?;
assert_eq!(Some(delta.data.as_ref()), read_data.as_deref());
let read_data = log.get(StoreKey::hgid(delta.key))?;
assert_eq!(StoreResult::Found(delta.data.as_ref().to_vec()), read_data);
log.flush()?;
Ok(())
}
@ -224,11 +226,12 @@ mod tests {
multiplex.add(&delta, &metadata)?;
drop(multiplex);
let read_data = log.get(&delta.key)?;
assert_eq!(Some(delta.data.as_ref()), read_data.as_deref());
let k = StoreKey::hgid(delta.key);
let read_data = log.get(k.clone())?;
assert_eq!(StoreResult::Found(delta.data.as_ref().to_vec()), read_data);
let read_data = pack.get(&delta.key)?;
assert_eq!(Some(delta.data.as_ref()), read_data.as_deref());
let read_data = pack.get(k)?;
assert_eq!(StoreResult::Found(delta.data.as_ref().to_vec()), read_data);
log.flush()?;
pack.flush()?;

48
eden/scm/lib/revisionstore/src/mutabledatapack.rs
View File

@ -28,7 +28,7 @@ use mpatch::mpatch::get_full_text;
use crate::{
dataindex::{DataIndex, DeltaLocation},
datapack::{DataEntry, DataPackVersion},
datastore::{Delta, HgIdDataStore, HgIdMutableDeltaStore, Metadata},
datastore::{Delta, HgIdDataStore, HgIdMutableDeltaStore, Metadata, StoreResult},
error::EmptyMutablePack,
localstore::LocalStore,
mutablepack::MutablePack,
@ -247,16 +247,21 @@ impl MutablePack for MutableDataPack {
}
impl HgIdDataStore for MutableDataPack {
fn get(&self, key: &Key) -> Result<Option<Vec<u8>>> {
let delta_chain = self.get_delta_chain(key)?;
fn get(&self, key: StoreKey) -> Result<StoreResult<Vec<u8>>> {
let key = match key {
StoreKey::HgId(key) => key,
content => return Ok(StoreResult::NotFound(content)),
};
let delta_chain = self.get_delta_chain(&key)?;
let delta_chain = match delta_chain {
Some(chain) => chain,
None => return Ok(None),
None => return Ok(StoreResult::NotFound(StoreKey::HgId(key))),
};
let (basetext, deltas) = match delta_chain.split_last() {
Some((base, delta)) => (base, delta),
None => return Ok(None),
None => return Ok(StoreResult::NotFound(StoreKey::HgId(key))),
};
let deltas: Vec<&[u8]> = deltas
@ -265,17 +270,21 @@ impl HgIdDataStore for MutableDataPack {
.map(|delta| delta.data.as_ref())
.collect();
Ok(Some(
Ok(StoreResult::Found(
get_full_text(basetext.data.as_ref(), &deltas).map_err(Error::msg)?,
))
}
fn get_meta(&self, key: &Key) -> Result<Option<Metadata>> {
Ok(self
.inner
.lock()
.read_entry(&key)?
.map(|(_, metadata)| metadata))
fn get_meta(&self, key: StoreKey) -> Result<StoreResult<Metadata>> {
let key = match key {
StoreKey::HgId(key) => key,
content => return Ok(StoreResult::NotFound(content)),
};
match self.inner.lock().read_entry(&key)? {
None => Ok(StoreResult::NotFound(StoreKey::HgId(key))),
Some((_, metadata)) => Ok(StoreResult::Found(metadata)),
}
}
}
@ -421,16 +430,19 @@ mod tests {
mutdatapack.add(&delta2, &meta2).unwrap();
// Requesting a default metadata
let found_meta = mutdatapack.get_meta(&delta.key).unwrap();
assert_eq!(found_meta.unwrap(), Metadata::default());
let found_meta = mutdatapack.get_meta(StoreKey::hgid(delta.key)).unwrap();
assert_eq!(found_meta, StoreResult::Found(Metadata::default()));
// Requesting a specified metadata
let found_meta = mutdatapack.get_meta(&delta2.key).unwrap();
assert_eq!(found_meta.unwrap(), meta2);
let found_meta = mutdatapack.get_meta(StoreKey::hgid(delta2.key)).unwrap();
assert_eq!(found_meta, StoreResult::Found(meta2));
// Requesting a non-existent metadata
let not = key("not", "10000");
assert_eq!(mutdatapack.get_meta(&not).unwrap(), None);
let not = StoreKey::hgid(key("not", "10000"));
assert_eq!(
mutdatapack.get_meta(not.clone()).unwrap(),
StoreResult::NotFound(not)
);
}
#[test]

89
eden/scm/lib/revisionstore/src/packstore.rs
View File

@ -26,7 +26,7 @@ use types::{Key, NodeInfo};
use crate::{
datapack::{DataPack, DataPackVersion},
datastore::{Delta, HgIdDataStore, HgIdMutableDeltaStore, Metadata},
datastore::{Delta, HgIdDataStore, HgIdMutableDeltaStore, Metadata, StoreResult},
historypack::{HistoryPack, HistoryPackVersion},
historystore::{HgIdHistoryStore, HgIdMutableHistoryStore},
localstore::LocalStore,
@ -349,12 +349,34 @@ impl<T: LocalStore + Repackable> LocalStore for PackStore<T> {
}
impl HgIdDataStore for DataPackStore {
fn get(&self, key: &Key) -> Result<Option<Vec<u8>>> {
self.inner.lock().run(|store| store.get(key))
fn get(&self, key: StoreKey) -> Result<StoreResult<Vec<u8>>> {
let res = self
.inner
.lock()
.run(|store| match store.get(key.clone())? {
StoreResult::Found(content) => Ok(Some(content)),
StoreResult::NotFound(_) => Ok(None),
})?;
match res {
None => Ok(StoreResult::NotFound(key)),
Some(content) => Ok(StoreResult::Found(content)),
}
}
fn get_meta(&self, key: &Key) -> Result<Option<Metadata>> {
self.inner.lock().run(|store| store.get_meta(key))
fn get_meta(&self, key: StoreKey) -> Result<StoreResult<Metadata>> {
let res = self
.inner
.lock()
.run(|store| match store.get_meta(key.clone())? {
StoreResult::Found(meta) => Ok(Some(meta)),
StoreResult::NotFound(_) => Ok(None),
})?;
match res {
None => Ok(StoreResult::NotFound(key)),
Some(meta) => Ok(StoreResult::Found(meta)),
}
}
}
@ -394,11 +416,11 @@ impl MutableDataPackStore {
}
impl HgIdDataStore for MutableDataPackStore {
fn get(&self, key: &Key) -> Result<Option<Vec<u8>>> {
fn get(&self, key: StoreKey) -> Result<StoreResult<Vec<u8>>> {
self.inner.union_store.get(key)
}
fn get_meta(&self, key: &Key) -> Result<Option<Metadata>> {
fn get_meta(&self, key: StoreKey) -> Result<StoreResult<Metadata>> {
self.inner.union_store.get_meta(key)
}
}
@ -543,8 +565,11 @@ mod tests {
make_datapack(&tempdir, &vec![revision.clone()]);
let store = DataPackStore::new(&tempdir, CorruptionPolicy::REMOVE);
let stored = store.get(&k)?;
assert_eq!(stored.as_deref(), Some(revision.0.data.as_ref()));
let stored = store.get(StoreKey::hgid(k))?;
assert_eq!(
stored,
StoreResult::Found(revision.0.data.as_ref().to_vec())
);
Ok(())
}
@ -585,8 +610,11 @@ mod tests {
);
make_datapack(&tempdir, &vec![revision.clone()]);
let stored = store.get(&k)?;
assert_eq!(stored.as_deref(), Some(revision.0.data.as_ref()));
let stored = store.get(StoreKey::hgid(k))?;
assert_eq!(
stored,
StoreResult::Found(revision.0.data.as_ref().to_vec())
);
Ok(())
}
@ -610,7 +638,7 @@ mod tests {
);
make_datapack(&tempdir, &vec![revision.clone()]);
store.get(&k).unwrap();
store.get(StoreKey::hgid(k)).unwrap();
let k = key("a", "3");
let revision = (
@ -623,7 +651,8 @@ mod tests {
);
make_datapack(&tempdir, &vec![revision.clone()]);
assert_eq!(store.get(&k).unwrap(), None);
let k = StoreKey::hgid(k);
assert_eq!(store.get(k.clone()).unwrap(), StoreResult::NotFound(k));
}
#[test]
@ -646,7 +675,7 @@ mod tests {
);
make_datapack(&tempdir, &vec![revision.clone()]);
store.get(&k)?;
store.get(StoreKey::hgid(k))?;
let k = key("a", "3");
let revision = (
@ -660,7 +689,10 @@ mod tests {
make_datapack(&tempdir, &vec![revision.clone()]);
store.force_rescan();
assert!(store.get(&k)?.is_some());
assert_eq!(
store.get(StoreKey::hgid(k))?,
StoreResult::Found(vec![1, 2, 3, 4])
);
Ok(())
}
@ -708,14 +740,16 @@ mod tests {
let packstore = DataPackStore::new(&tempdir, CorruptionPolicy::REMOVE);
let _ = packstore.get(&k2)?;
let k2 = StoreKey::hgid(k2);
let _ = packstore.get(k2.clone())?;
assert!(packstore.inner.lock().packs.borrow().stores[0]
.get(&k2)
.get(k2)
.is_ok());
let _ = packstore.get(&k1)?;
let k1 = StoreKey::hgid(k1);
let _ = packstore.get(k1.clone())?;
assert!(packstore.inner.lock().packs.borrow().stores[0]
.get(&k1)
.get(k1)
.is_ok());
Ok(())
@ -760,7 +794,11 @@ mod tests {
.unwrap();
let packstore = DataPackStore::new(&tempdir, CorruptionPolicy::REMOVE);
assert_eq!(packstore.get(&k1).unwrap(), None);
let k1 = StoreKey::hgid(k1);
assert_eq!(
packstore.get(k1.clone()).unwrap(),
StoreResult::NotFound(k1)
);
}
#[test]
@ -791,7 +829,8 @@ mod tests {
assert_eq!(read_dir(&tempdir)?.count(), 2);
let packstore = DataPackStore::new(&tempdir, CorruptionPolicy::IGNORE);
assert!(packstore.get(&k1)?.is_none());
let k1 = StoreKey::hgid(k1);
assert_eq!(packstore.get(k1.clone())?, StoreResult::NotFound(k1));
assert_eq!(read_dir(&tempdir)?.count(), 2);
Ok(())
@ -827,8 +866,8 @@ mod tests {
};
packstore.add(&delta, &Default::default())?;
let stored = packstore.get(&k1)?;
assert_eq!(stored.as_deref(), Some(delta.data.as_ref()));
let stored = packstore.get(StoreKey::hgid(k1))?;
assert_eq!(stored, StoreResult::Found(delta.data.as_ref().to_vec()));
Ok(())
}
@ -846,8 +885,8 @@ mod tests {
packstore.add(&delta, &Default::default())?;
packstore.flush()?;
let stored = packstore.get(&k1)?;
assert_eq!(stored.as_deref(), Some(delta.data.as_ref()));
let stored = packstore.get(StoreKey::hgid(k1))?;
assert_eq!(stored, StoreResult::Found(delta.data.as_ref().to_vec()));
Ok(())
}

20
eden/scm/lib/revisionstore/src/repack.rs
View File

@ -22,7 +22,7 @@ use types::Key;
use crate::{
contentstore::ContentStore,
datapack::{DataPack, DataPackVersion},
datastore::{HgIdDataStore, HgIdMutableDeltaStore},
datastore::{HgIdDataStore, HgIdMutableDeltaStore, StoreResult},
historypack::{HistoryPack, HistoryPackVersion},
historystore::{HgIdHistoryStore, HgIdMutableHistoryStore},
localstore::LocalStore,
@ -64,8 +64,10 @@ fn repack_datapack(data_pack: &DataPack, mut_pack: &mut MutableDataPack) -> Resu
}
// If we managed to get a delta, the metadata must be present.
let meta = data_pack.get_meta(&delta.key)?.unwrap();
mut_pack.add(&delta, &meta)?;
match data_pack.get_meta(StoreKey::hgid(delta.key.clone()))? {
StoreResult::Found(meta) => mut_pack.add(&delta, &meta)?,
_ => (),
}
}
}
}
@ -311,11 +313,13 @@ fn repack_datapack_to_contentstore(
for key in pack.to_keys() {
let key = key?;
if let Some(content) = store.get(&key)? {
// If we managed to get a delta, the metadata must be present.
let meta = store.get_meta(&key)?.unwrap();
store.add_pending(&key, Bytes::from(content), meta, location)?;
if let StoreResult::Found(content) = store.get(StoreKey::hgid(key.clone()))? {
match store.get_meta(StoreKey::hgid(key.clone()))? {
StoreResult::Found(meta) => {
store.add_pending(&key, Bytes::from(content), meta, location)?
}
_ => (),
}
}
}
Ok(())

18
eden/scm/lib/revisionstore/src/testutil.rs
View File

@ -18,7 +18,9 @@ use edenapi_types::{DataEntry, HistoryEntry};
use types::{HgId, Key, NodeInfo, Parents, RepoPathBuf};
use crate::{
datastore::{Delta, HgIdDataStore, HgIdMutableDeltaStore, Metadata, RemoteDataStore},
datastore::{
Delta, HgIdDataStore, HgIdMutableDeltaStore, Metadata, RemoteDataStore, StoreResult,
},
historystore::{HgIdHistoryStore, HgIdMutableHistoryStore, RemoteHistoryStore},
localstore::LocalStore,
remotestore::HgIdRemoteStore,
@ -118,18 +120,16 @@ impl RemoteDataStore for FakeRemoteDataStore {
}
impl HgIdDataStore for FakeRemoteDataStore {
fn get(&self, key: &Key) -> Result<Option<Vec<u8>>> {
let missing = self.translate_lfs_missing(&[StoreKey::hgid(key.clone())])?;
match self.prefetch(&missing) {
Err(_) => Ok(None),
fn get(&self, key: StoreKey) -> Result<StoreResult<Vec<u8>>> {
match self.prefetch(&[key.clone()]) {
Err(_) => Ok(StoreResult::NotFound(key)),
Ok(()) => self.store.get(key),
}
}
fn get_meta(&self, key: &Key) -> Result<Option<Metadata>> {
let missing = self.translate_lfs_missing(&[StoreKey::hgid(key.clone())])?;
match self.prefetch(&missing) {
Err(_) => Ok(None),
fn get_meta(&self, key: StoreKey) -> Result<StoreResult<Metadata>> {
match self.prefetch(&[key.clone()]) {
Err(_) => Ok(StoreResult::NotFound(key)),
Ok(()) => self.store.get_meta(key),
}
}

76
eden/scm/lib/revisionstore/src/uniondatastore.rs
View File

@ -10,10 +10,10 @@ use anyhow::Result;
use bytes::Bytes;
use types::Key;
use crate::{
datastore::{ContentDataStore, ContentMetadata, HgIdDataStore, Metadata, RemoteDataStore},
datastore::{
ContentDataStore, ContentMetadata, HgIdDataStore, Metadata, RemoteDataStore, StoreResult,
},
types::StoreKey,
unionstore::UnionStore,
};
@ -21,24 +21,26 @@ use crate::{
pub type UnionHgIdDataStore<T> = UnionStore<T>;
impl<T: HgIdDataStore> HgIdDataStore for UnionHgIdDataStore<T> {
fn get(&self, key: &Key) -> Result<Option<Vec<u8>>> {
fn get(&self, mut key: StoreKey) -> Result<StoreResult<Vec<u8>>> {
for store in self {
if let Some(result) = store.get(key)? {
return Ok(Some(result));
match store.get(key)? {
StoreResult::Found(data) => return Ok(StoreResult::Found(data)),
StoreResult::NotFound(next) => key = next,
}
}
Ok(None)
Ok(StoreResult::NotFound(key))
}
fn get_meta(&self, key: &Key) -> Result<Option<Metadata>> {
fn get_meta(&self, mut key: StoreKey) -> Result<StoreResult<Metadata>> {
for store in self {
if let Some(meta) = store.get_meta(key)? {
return Ok(Some(meta));
match store.get_meta(key)? {
StoreResult::Found(meta) => return Ok(StoreResult::Found(meta)),
StoreResult::NotFound(next) => key = next,
}
}
Ok(None)
Ok(StoreResult::NotFound(key))
}
}
@ -79,24 +81,26 @@ impl<T: RemoteDataStore> RemoteDataStore for UnionHgIdDataStore<T> {
pub type UnionContentDataStore<T> = UnionStore<T>;
impl<T: ContentDataStore> ContentDataStore for UnionContentDataStore<T> {
fn blob(&self, key: &StoreKey) -> Result<Option<Bytes>> {
fn blob(&self, mut key: StoreKey) -> Result<StoreResult<Bytes>> {
for store in self {
if let Some(data) = store.blob(key)? {
return Ok(Some(data));
match store.blob(key)? {
StoreResult::Found(blob) => return Ok(StoreResult::Found(blob)),
StoreResult::NotFound(next) => key = next,
}
}
Ok(None)
Ok(StoreResult::NotFound(key))
}
fn metadata(&self, key: &StoreKey) -> Result<Option<ContentMetadata>> {
fn metadata(&self, mut key: StoreKey) -> Result<StoreResult<ContentMetadata>> {
for store in self {
if let Some(meta) = store.metadata(key)? {
return Ok(Some(meta));
match store.metadata(key)? {
StoreResult::Found(meta) => return Ok(StoreResult::Found(meta)),
StoreResult::NotFound(next) => key = next,
}
}
Ok(None)
Ok(StoreResult::NotFound(key))
}
}
@ -107,6 +111,8 @@ mod tests {
use quickcheck::quickcheck;
use thiserror::Error;
use types::Key;
use crate::{localstore::LocalStore, types::StoreKey};
struct BadHgIdDataStore;
@ -118,12 +124,12 @@ mod tests {
struct EmptyHgIdDataStore;
impl HgIdDataStore for EmptyHgIdDataStore {
fn get(&self, _key: &Key) -> Result<Option<Vec<u8>>> {
Ok(None)
fn get(&self, key: StoreKey) -> Result<StoreResult<Vec<u8>>> {
Ok(StoreResult::NotFound(key))
}
fn get_meta(&self, _key: &Key) -> Result<Option<Metadata>> {
Ok(None)
fn get_meta(&self, key: StoreKey) -> Result<StoreResult<Metadata>> {
Ok(StoreResult::NotFound(key))
}
}
@ -134,11 +140,11 @@ mod tests {
}
impl HgIdDataStore for BadHgIdDataStore {
fn get(&self, _key: &Key) -> Result<Option<Vec<u8>>> {
fn get(&self, _key: StoreKey) -> Result<StoreResult<Vec<u8>>> {
Err(BadHgIdDataStoreError.into())
}
fn get_meta(&self, _key: &Key) -> Result<Option<Metadata>> {
fn get_meta(&self, _key: StoreKey) -> Result<StoreResult<Metadata>> {
Err(BadHgIdDataStoreError.into())
}
}
@ -151,8 +157,8 @@ mod tests {
quickcheck! {
fn test_empty_unionstore_get(key: Key) -> bool {
match UnionHgIdDataStore::<EmptyHgIdDataStore>::new().get(&key) {
Ok(None) => true,
match UnionHgIdDataStore::<EmptyHgIdDataStore>::new().get(StoreKey::hgid(key)) {
Ok(StoreResult::NotFound(_)) => true,
_ => false,
}
}
@ -160,8 +166,8 @@ mod tests {
fn test_empty_datastore_get(key: Key) -> bool {
let mut unionstore = UnionHgIdDataStore::new();
unionstore.add(EmptyHgIdDataStore);
match unionstore.get(&key) {
Ok(None) => true,
match unionstore.get(StoreKey::hgid(key)) {
Ok(StoreResult::NotFound(_)) => true,
_ => false,
}
}
@ -169,15 +175,15 @@ mod tests {
fn test_bad_datastore_get(key: Key) -> bool {
let mut unionstore = UnionHgIdDataStore::new();
unionstore.add(BadHgIdDataStore);
match unionstore.get(&key) {
match unionstore.get(StoreKey::hgid(key)) {
Err(_) => true,
_ => false,
}
}
fn test_empty_unionstore_get_meta(key: Key) -> bool {
match UnionHgIdDataStore::<EmptyHgIdDataStore>::new().get_meta(&key) {
Ok(None) => true,
match UnionHgIdDataStore::<EmptyHgIdDataStore>::new().get_meta(StoreKey::hgid(key)) {
Ok(StoreResult::NotFound(_)) => true,
_ => false,
}
}
@ -185,8 +191,8 @@ mod tests {
fn test_empty_datastore_get_meta(key: Key) -> bool {
let mut unionstore = UnionHgIdDataStore::new();
unionstore.add(EmptyHgIdDataStore);
match unionstore.get_meta(&key) {
Ok(None) => true,
match unionstore.get_meta(StoreKey::hgid(key)) {
Ok(StoreResult::NotFound(_)) => true,
_ => false,
}
}
@ -194,7 +200,7 @@ mod tests {
fn test_bad_datastore_get_meta(key: Key) -> bool {
let mut unionstore = UnionHgIdDataStore::new();
unionstore.add(BadHgIdDataStore);
match unionstore.get_meta(&key) {
match unionstore.get_meta(StoreKey::hgid(key)) {
Err(_) => true,
_ => false,
}

6
eden/scm/tests/test-fb-hgext-remotefilelog-repack-corrupt.t
View File

@ -142,10 +142,10 @@ The local data for n is still available
The history for n is lost
$ hg log -qf n 2>&1 | grep 'KeyError'
KeyError: 'Key not found Key { path: RepoPathBuf("n"), hgid: HgId("c972a0820002b32c6fec4b7ca47d3aecdad8e1c5") }'
KeyError: 'Key not found HgId(Key { path: RepoPathBuf("n"), hgid: HgId("c972a0820002b32c6fec4b7ca47d3aecdad8e1c5") })'
The local data and history for o is lost
$ hg cat -q o 2>&1 | grep 'KeyError'
KeyError: 'Key not found Key { path: RepoPathBuf("o"), hgid: HgId("fd94f81d01bf8c9d960bb57abdd4e8375309ae43") }'
KeyError: 'Key not found HgId(Key { path: RepoPathBuf("o"), hgid: HgId("fd94f81d01bf8c9d960bb57abdd4e8375309ae43") })'
$ hg log -qf o 2>&1 | grep 'KeyError'
KeyError: 'Key not found Key { path: RepoPathBuf("o"), hgid: HgId("fd94f81d01bf8c9d960bb57abdd4e8375309ae43") }'
KeyError: 'Key not found HgId(Key { path: RepoPathBuf("o"), hgid: HgId("fd94f81d01bf8c9d960bb57abdd4e8375309ae43") })'