facebook/sapling
1
1
Fork 0
mirror of https://github.com/facebook/sapling.git synced 2024-10-10 16:57:49 +03:00
Code Issues Projects Releases Wiki Activity

Convert scm/mononoke/common/rust/caching_ext to Rust 2018

Browse Source 
Summary:
Rust 2018 updates to:
  //scm/mononoke/common/rust/caching_ext:caching_ext
  //scm/mononoke/common/rust/caching_ext:caching_ext-unittest

Reviewed By: StanislavGlebik

Differential Revision: D15465983

fbshipit-source-id: e68fdc2324a68c9884fdcf1c45273443917e758e
This commit is contained in:
Jeremy Fitzhardinge 2019-05-24 09:49:11 -07:00 committed by Facebook Github Bot
parent aafc38da2b
commit bc85fa4edb
3 changed files with 42 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
common/rust/caching_ext/src/cachelib_utils.rs
View File

@ -8,11 +8,11 @@ use std::collections::HashMap;
use std::hash::Hash; use std::hash::Hash;
use std::sync::atomic::Ordering; use std::sync::atomic::Ordering;
use crate::failure::prelude::*;
use crate::mock_store::MockStore;
use cachelib::{get_cached, set_cached, Abomonation, LruCachePool}; use cachelib::{get_cached, set_cached, Abomonation, LruCachePool};
use failure::prelude::*;
use mock_store::MockStore;
use CachelibKey; use crate::CachelibKey;
#[derive(Clone)] #[derive(Clone)]
pub enum CachelibHandler<T> { pub enum CachelibHandler<T> {

71
common/rust/caching_ext/src/lib.rs
View File

@ -13,7 +13,6 @@ extern crate cloned;
extern crate futures; extern crate futures;
extern crate tokio; extern crate tokio;
extern crate bytes_ext;
extern crate cachelib; extern crate cachelib;
extern crate failure_ext as failure; extern crate failure_ext as failure;
#[macro_use] #[macro_use]
@ -37,18 +36,18 @@ use std::collections::{HashMap, HashSet};
use std::hash::Hash; use std::hash::Hash;
use std::sync::Arc; use std::sync::Arc;
use crate::failure::prelude::*;
use bytes::Bytes; use bytes::Bytes;
use cachelib::Abomonation; use cachelib::Abomonation;
use failure::prelude::*;
use futures::{future::join_all, prelude::*}; use futures::{future::join_all, prelude::*};
use futures_ext::{BoxFuture, FutureExt}; use futures_ext::{BoxFuture, FutureExt};
use iobuf::IOBuf; use iobuf::IOBuf;
use memcache::{KeyGen, MEMCACHE_VALUE_MAX_SIZE}; use memcache::{KeyGen, MEMCACHE_VALUE_MAX_SIZE};
use mononoke_types::RepositoryId; use mononoke_types::RepositoryId;
pub use cachelib_utils::CachelibHandler; pub use crate::cachelib_utils::CachelibHandler;
pub use memcache_utils::MemcacheHandler; pub use crate::memcache_utils::MemcacheHandler;
pub use mock_store::MockStoreStats; pub use crate::mock_store::MockStoreStats;
/// Error type to help with proper reporting of memcache errors /// Error type to help with proper reporting of memcache errors
pub enum McErrorKind { pub enum McErrorKind {
@ -67,15 +66,15 @@ struct MemcacheKey(String);
pub struct GetOrFillMultipleFromCacheLayers<Key, T> { pub struct GetOrFillMultipleFromCacheLayers<Key, T> {
pub repo_id: RepositoryId, pub repo_id: RepositoryId,
pub get_cache_key: Arc<Fn(RepositoryId, &Key) -> String>, pub get_cache_key: Arc<dyn Fn(RepositoryId, &Key) -> String>,
pub cachelib: CachelibHandler<T>, pub cachelib: CachelibHandler<T>,
pub keygen: KeyGen, pub keygen: KeyGen,
pub memcache: MemcacheHandler, pub memcache: MemcacheHandler,
pub deserialize: Arc<Fn(IOBuf) -> ::std::result::Result<T, ()> + Send + Sync + 'static>, pub deserialize: Arc<dyn Fn(IOBuf) -> ::std::result::Result<T, ()> + Send + Sync + 'static>,
pub serialize: Arc<Fn(&T) -> Bytes + Send + Sync + 'static>, pub serialize: Arc<dyn Fn(&T) -> Bytes + Send + Sync + 'static>,
pub report_mc_result: Arc<Fn(McResult<()>) + Send + Sync + 'static>, pub report_mc_result: Arc<dyn Fn(McResult<()>) + Send + Sync + 'static>,
pub get_from_db: pub get_from_db:
Arc<Fn(HashSet<Key>) -> BoxFuture<HashMap<Key, T>, Error> + Send + Sync + 'static>, Arc<dyn Fn(HashSet<Key>) -> BoxFuture<HashMap<Key, T>, Error> + Send + Sync + 'static>,
} }
impl<Key, T> GetOrFillMultipleFromCacheLayers<Key, T> impl<Key, T> GetOrFillMultipleFromCacheLayers<Key, T>
@ -84,7 +83,8 @@ where
T: Abomonation + Clone + Send + 'static, T: Abomonation + Clone + Send + 'static,
{ {
pub fn run(&self, keys: HashSet<Key>) -> BoxFuture<HashMap<Key, T>, Error> { pub fn run(&self, keys: HashSet<Key>) -> BoxFuture<HashMap<Key, T>, Error> {
let keys: Vec<(Key, CachelibKey)> = keys.into_iter() let keys: Vec<(Key, CachelibKey)> = keys
.into_iter()
.map(|key| { .map(|key| {
let cache_key = CachelibKey((self.get_cache_key)(self.repo_id, &key)); let cache_key = CachelibKey((self.get_cache_key)(self.repo_id, &key));
(key, cache_key) (key, cache_key)
@ -106,7 +106,8 @@ where
&self.memcache, &self.memcache,
self.deserialize.clone(), self.deserialize.clone(),
self.report_mc_result.clone(), self.report_mc_result.clone(),
).then(move |result: ::std::result::Result<_, !>| { )
.then(move |result: ::std::result::Result<_, !>| {
let (fetched_from_memcache, left_to_fetch) = match result { let (fetched_from_memcache, left_to_fetch) = match result {
Ok(result) => result, Ok(result) => result,
Err(never) => never, Err(never) => never,
@ -146,7 +147,7 @@ where
fetched fetched
}) })
}) })
.boxify() .boxify()
} }
} }
@ -154,8 +155,8 @@ fn get_multiple_from_memcache<Key, T>(
keys: Vec<(Key, CachelibKey)>, keys: Vec<(Key, CachelibKey)>,
keygen: &KeyGen, keygen: &KeyGen,
memcache: &MemcacheHandler, memcache: &MemcacheHandler,
deserialize: Arc<Fn(IOBuf) -> ::std::result::Result<T, ()> + Send + Sync + 'static>, deserialize: Arc<dyn Fn(IOBuf) -> ::std::result::Result<T, ()> + Send + Sync + 'static>,
report_mc_result: Arc<Fn(McResult<()>) + Send + Sync + 'static>, report_mc_result: Arc<dyn Fn(McResult<()>) + Send + Sync + 'static>,
) -> impl Future< ) -> impl Future<
Item = ( Item = (
HashMap<Key, (T, CachelibKey)>, HashMap<Key, (T, CachelibKey)>,
@ -166,7 +167,8 @@ fn get_multiple_from_memcache<Key, T>(
where where
Key: Eq + Hash, Key: Eq + Hash,
{ {
let mc_fetch_futs: Vec<_> = keys.into_iter() let mc_fetch_futs: Vec<_> = keys
.into_iter()
.map(move |(key, cache_key)| { .map(move |(key, cache_key)| {
let memcache_key = MemcacheKey(keygen.key(&cache_key.0)); let memcache_key = MemcacheKey(keygen.key(&cache_key.0));
memcache memcache
@ -207,7 +209,7 @@ where
fn fill_multiple_memcache<Key, T>( fn fill_multiple_memcache<Key, T>(
keys: HashMap<Key, (T, CachelibKey, MemcacheKey)>, keys: HashMap<Key, (T, CachelibKey, MemcacheKey)>,
memcache: MemcacheHandler, memcache: MemcacheHandler,
serialize: Arc<Fn(&T) -> Bytes + Send + Sync + 'static>, serialize: Arc<dyn Fn(&T) -> Bytes + Send + Sync + 'static>,
) -> HashMap<Key, (T, CachelibKey)> ) -> HashMap<Key, (T, CachelibKey)>
where where
Key: Eq + Hash, Key: Eq + Hash,
@ -230,7 +232,10 @@ mod test {
use super::*; use super::*;
use futures::future::ok; use futures::future::ok;
use std::sync::{Arc, atomic::{AtomicUsize, Ordering}}; use std::sync::{
atomic::{AtomicUsize, Ordering},
Arc,
};
fn test_cachelib_cachekey(_repoid: RepositoryId, key: &String) -> String { fn test_cachelib_cachekey(_repoid: RepositoryId, key: &String) -> String {
key.clone() key.clone()
@ -272,7 +277,7 @@ mod test {
#[test] #[test]
fn simple() { fn simple() {
let db_data = hashmap!{}; let db_data = hashmap! {};
let db_data_fetches = Arc::new(AtomicUsize::new(0)); let db_data_fetches = Arc::new(AtomicUsize::new(0));
let cachelib = CachelibHandler::create_mock(); let cachelib = CachelibHandler::create_mock();
let memcache = MemcacheHandler::create_mock(); let memcache = MemcacheHandler::create_mock();
@ -285,13 +290,13 @@ mod test {
); );
let mut runtime = tokio::runtime::Runtime::new().unwrap(); let mut runtime = tokio::runtime::Runtime::new().unwrap();
let f = params.run(hashset!{}); let f = params.run(hashset! {});
let res = runtime.block_on(f).unwrap(); let res = runtime.block_on(f).unwrap();
assert_eq!(res.len(), 0); assert_eq!(res.len(), 0);
assert_eq!(cachelib.gets_count(), 0); assert_eq!(cachelib.gets_count(), 0);
assert_eq!(memcache.gets_count(), 0); assert_eq!(memcache.gets_count(), 0);
let f = params.run(hashset!{"key".to_string()}); let f = params.run(hashset! {"key".to_string()});
let res = runtime.block_on(f).unwrap(); let res = runtime.block_on(f).unwrap();
assert_eq!(res.len(), 0); assert_eq!(res.len(), 0);
assert_eq!(cachelib.gets_count(), 1); assert_eq!(cachelib.gets_count(), 1);
@ -301,7 +306,7 @@ mod test {
#[test] #[test]
fn fetch_from_db_cachelib_memcache() { fn fetch_from_db_cachelib_memcache() {
let db_data = hashmap!{"key".to_string() => 0}; let db_data = hashmap! {"key".to_string() => 0};
let db_data_fetches = Arc::new(AtomicUsize::new(0)); let db_data_fetches = Arc::new(AtomicUsize::new(0));
let cachelib = CachelibHandler::create_mock(); let cachelib = CachelibHandler::create_mock();
let memcache = MemcacheHandler::create_mock(); let memcache = MemcacheHandler::create_mock();
@ -314,7 +319,7 @@ mod test {
); );
// Fetch from db // Fetch from db
let f = params.run(hashset!{"key".to_string()}); let f = params.run(hashset! {"key".to_string()});
let mut runtime = tokio::runtime::Runtime::new().unwrap(); let mut runtime = tokio::runtime::Runtime::new().unwrap();
let res = runtime.block_on(f).unwrap(); let res = runtime.block_on(f).unwrap();
assert_eq!(res.len(), 1); assert_eq!(res.len(), 1);
@ -324,7 +329,7 @@ mod test {
assert_eq!(db_data_fetches.load(Ordering::Relaxed), 1); assert_eq!(db_data_fetches.load(Ordering::Relaxed), 1);
// Now fetch from cachelib // Now fetch from cachelib
let f = params.run(hashset!{"key".to_string()}); let f = params.run(hashset! {"key".to_string()});
let res = runtime.block_on(f).unwrap(); let res = runtime.block_on(f).unwrap();
assert_eq!(res.len(), 1); assert_eq!(res.len(), 1);
assert_eq!(cachelib.gets_count(), 2); assert_eq!(cachelib.gets_count(), 2);
@ -334,7 +339,7 @@ mod test {
// Reset cachelib, fetch from memcache // Reset cachelib, fetch from memcache
let cachelib = CachelibHandler::create_mock(); let cachelib = CachelibHandler::create_mock();
params.cachelib = cachelib.clone(); params.cachelib = cachelib.clone();
let f = params.run(hashset!{"key".to_string()}); let f = params.run(hashset! {"key".to_string()});
let res = runtime.block_on(f).unwrap(); let res = runtime.block_on(f).unwrap();
assert_eq!(res.len(), 1); assert_eq!(res.len(), 1);
assert_eq!(cachelib.gets_count(), 1); assert_eq!(cachelib.gets_count(), 1);
@ -344,7 +349,7 @@ mod test {
#[test] #[test]
fn fetch_from_db() { fn fetch_from_db() {
let db_data = hashmap!{ let db_data = hashmap! {
"key0".to_string() => 0, "key0".to_string() => 0,
"key1".to_string() => 1, "key1".to_string() => 1,
"key2".to_string() => 2, "key2".to_string() => 2,
@ -360,7 +365,7 @@ mod test {
db_data, db_data,
); );
let mut runtime = tokio::runtime::Runtime::new().unwrap(); let mut runtime = tokio::runtime::Runtime::new().unwrap();
let f = params.run(hashset!{ let f = params.run(hashset! {
"key0".to_string(), "key1".to_string(), "key2".to_string() "key0".to_string(), "key1".to_string(), "key2".to_string()
}); });
runtime.block_on(f).unwrap(); runtime.block_on(f).unwrap();
@ -372,7 +377,7 @@ mod test {
#[test] #[test]
fn fetch_from_all() { fn fetch_from_all() {
let db_data = hashmap!{ let db_data = hashmap! {
"key0".to_string() => 0, "key0".to_string() => 0,
"key1".to_string() => 1, "key1".to_string() => 1,
"key2".to_string() => 2, "key2".to_string() => 2,
@ -390,7 +395,7 @@ mod test {
); );
let mut runtime = tokio::runtime::Runtime::new().unwrap(); let mut runtime = tokio::runtime::Runtime::new().unwrap();
let f = params.run(hashset!{"key1".to_string()}); let f = params.run(hashset! {"key1".to_string()});
runtime.block_on(f).unwrap(); runtime.block_on(f).unwrap();
assert_eq!(cachelib.gets_count(), 1); assert_eq!(cachelib.gets_count(), 1);
assert_eq!(memcache.gets_count(), 1); assert_eq!(memcache.gets_count(), 1);
@ -399,13 +404,13 @@ mod test {
// Reset cachelib // Reset cachelib
let cachelib = CachelibHandler::create_mock(); let cachelib = CachelibHandler::create_mock();
params.cachelib = cachelib.clone(); params.cachelib = cachelib.clone();
let f = params.run(hashset!{"key0".to_string()}); let f = params.run(hashset! {"key0".to_string()});
runtime.block_on(f).unwrap(); runtime.block_on(f).unwrap();
assert_eq!(cachelib.gets_count(), 1); assert_eq!(cachelib.gets_count(), 1);
assert_eq!(memcache.gets_count(), 2); assert_eq!(memcache.gets_count(), 2);
assert_eq!(db_data_fetches.load(Ordering::Relaxed), 2); assert_eq!(db_data_fetches.load(Ordering::Relaxed), 2);
let f = params.run(hashset!{ let f = params.run(hashset! {
"key0".to_string(), "key1".to_string(), "key2".to_string() "key0".to_string(), "key1".to_string(), "key2".to_string()
}); });
let res = runtime.block_on(f).unwrap(); let res = runtime.block_on(f).unwrap();
@ -415,7 +420,7 @@ mod test {
assert_eq!(db_data_fetches.load(Ordering::Relaxed), 2 + 1); // 1 fetch from db assert_eq!(db_data_fetches.load(Ordering::Relaxed), 2 + 1); // 1 fetch from db
// // Only from cachelib // // Only from cachelib
let f = params.run(hashset!{ let f = params.run(hashset! {
"key0".to_string(), "key1".to_string(), "key2".to_string() "key0".to_string(), "key1".to_string(), "key2".to_string()
}); });
let res = runtime.block_on(f).unwrap(); let res = runtime.block_on(f).unwrap();
@ -427,7 +432,7 @@ mod test {
// // Reset cachelib, only from memcache // // Reset cachelib, only from memcache
let cachelib = CachelibHandler::create_mock(); let cachelib = CachelibHandler::create_mock();
params.cachelib = cachelib.clone(); params.cachelib = cachelib.clone();
let f = params.run(hashset!{ let f = params.run(hashset! {
"key0".to_string(), "key1".to_string(), "key2".to_string() "key0".to_string(), "key1".to_string(), "key2".to_string()
}); });
let res = runtime.block_on(f).unwrap(); let res = runtime.block_on(f).unwrap();

2
common/rust/caching_ext/src/memcache_utils.rs
View File

@ -4,12 +4,12 @@
// This software may be used and distributed according to the terms of the // This software may be used and distributed according to the terms of the
// GNU General Public License version 2 or any later version. // GNU General Public License version 2 or any later version.
use crate::mock_store::MockStore;
use bytes::Bytes; use bytes::Bytes;
use futures::{future::ok, Future}; use futures::{future::ok, Future};
use futures_ext::FutureExt; use futures_ext::FutureExt;
use iobuf::IOBuf; use iobuf::IOBuf;
use memcache::MemcacheClient; use memcache::MemcacheClient;
use mock_store::MockStore;
use std::sync::atomic::Ordering; use std::sync::atomic::Ordering;
#[derive(Clone)] #[derive(Clone)]