facebook/sapling
1
1
Fork 0
mirror of https://github.com/facebook/sapling.git synced 2024-10-10 08:47:12 +03:00
Code Issues Projects Releases Wiki Activity

censoredblobstore logging access

Browse Source 
Summary: Log when a `censored file` is accessed via calling `get` or `put` method.

Reviewed By: StanislavGlebik

Differential Revision: D16089138

fbshipit-source-id: 0d2f0e21e7afcad8783be7587e6b676af20ba029
This commit is contained in:
George-Catalin Tintareanu 2019-07-05 07:13:18 -07:00 committed by Facebook Github Bot
parent bd50a8cde1
commit 7fe5117c31
3 changed files with 94 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
blobstore/cacheblob/src/locking_cache.rs
View File

@ -4,20 +4,18 @@
// This software may be used and distributed according to the terms of the
// GNU General Public License version 2 or any later version.
use std::fmt;
use std::sync::Arc;
use failure_ext::Error;
use blobstore::{Blobstore, CountedBlobstore};
use censoredblob::CensoredBlob;
use context::CoreContext;
use failure_ext::Error;
use futures::{future, future::Either, Future, IntoFuture};
use futures_ext::{BoxFuture, FutureExt};
use prefixblob::PrefixBlobstore;
use tokio;
use context::CoreContext;
use mononoke_types::BlobstoreBytes;
use prefixblob::PrefixBlobstore;
use slog::debug;
use std::fmt;
use std::sync::Arc;
use tokio;
/// Extra operations that can be performed on a cache. Other wrappers can implement this trait for
/// e.g. all `WrapperBlobstore<CacheBlobstore<T>>`.
@ -29,7 +27,11 @@ pub trait CacheBlobstoreExt: Blobstore {
ctx: CoreContext,
key: String,
) -> BoxFuture<Option<BlobstoreBytes>, Error>;
fn get_cache_only(&self, key: String) -> BoxFuture<Option<BlobstoreBytes>, Error>;
fn get_cache_only(
&self,
ctx: CoreContext,
key: String,
) -> BoxFuture<Option<BlobstoreBytes>, Error>;
}
/// The operations a cache must provide in order to be usable as the caching layer for a
@ -335,7 +337,11 @@ where
.boxify()
}
fn get_cache_only(&self, key: String) -> BoxFuture<Option<BlobstoreBytes>, Error> {
fn get_cache_only(
&self,
_ctx: CoreContext,
key: String,
) -> BoxFuture<Option<BlobstoreBytes>, Error> {
CacheOpsUtil::get(&self.cache, &key).boxify()
}
}
@ -366,8 +372,12 @@ impl<T: CacheBlobstoreExt> CacheBlobstoreExt for CountedBlobstore<T> {
}
#[inline]
fn get_cache_only(&self, key: String) -> BoxFuture<Option<BlobstoreBytes>, Error> {
self.as_inner().get_cache_only(key)
fn get_cache_only(
&self,
ctx: CoreContext,
key: String,
) -> BoxFuture<Option<BlobstoreBytes>, Error> {
self.as_inner().get_cache_only(ctx, key)
}
}
@ -382,8 +392,12 @@ impl<T: CacheBlobstoreExt + Clone> CacheBlobstoreExt for PrefixBlobstore<T> {
}
#[inline]
fn get_cache_only(&self, key: String) -> BoxFuture<Option<BlobstoreBytes>, Error> {
self.as_inner().get_cache_only(self.prepend(key))
fn get_cache_only(
&self,
ctx: CoreContext,
key: String,
) -> BoxFuture<Option<BlobstoreBytes>, Error> {
self.as_inner().get_cache_only(ctx, self.prepend(key))
}
}
@ -394,17 +408,33 @@ impl<T: CacheBlobstoreExt + Clone> CacheBlobstoreExt for CensoredBlob<T> {
ctx: CoreContext,
key: String,
) -> BoxFuture<Option<BlobstoreBytes>, Error> {
match self.is_censored(key.clone()) {
match self.err_if_censored(&key) {
Ok(()) => self.as_inner().get_no_cache_fill(ctx, key),
Err(err) => Err(err).into_future().boxify(),
Err(err) => {
debug!(
ctx.logger(),
"Accessing censored blobstore with key {:?}", key
);
Err(err).into_future().boxify()
}
}
}
#[inline]
fn get_cache_only(&self, key: String) -> BoxFuture<Option<BlobstoreBytes>, Error> {
match self.is_censored(key.clone()) {
Err(err) => Err(err).into_future().boxify(),
Ok(()) => self.as_inner().get_cache_only(key),
fn get_cache_only(
&self,
ctx: CoreContext,
key: String,
) -> BoxFuture<Option<BlobstoreBytes>, Error> {
match self.err_if_censored(&key) {
Ok(()) => self.as_inner().get_cache_only(ctx, key),
Err(err) => {
debug!(
ctx.logger(),
"Accessing censored blobstore with key {:?}", key
);
Err(err).into_future().boxify()
}
}
}
}

63
blobstore/censoredblob/src/lib.rs
View File

@ -9,11 +9,13 @@
use blobstore::{Blobstore, BlobstoreBytes};
use context::CoreContext;
use failure_ext::Error;
use futures::future;
use futures::future::{Future, IntoFuture};
use futures_ext::{BoxFuture, FutureExt};
use slog::debug;
use std::collections::HashMap;
mod errors;
use cloned::cloned;
use crate::errors::ErrorKind;
mod store;
@ -35,12 +37,11 @@ impl<T: Blobstore + Clone> CensoredBlob<T> {
}
}
pub fn is_censored(&self, key: String) -> Result<(), Error> {
pub fn err_if_censored(&self, key: &String) -> Result<(), Error> {
match &self.censored {
Some(censored) => match censored.get(&key) {
Some(task) => Err(ErrorKind::Censored(key, task.clone()).into()),
None => Ok(()),
},
Some(censored) => censored.get(key).map_or(Ok(()), |task| {
Err(ErrorKind::Censored(key.to_string(), task.to_string()).into())
}),
None => Ok(()),
}
}
@ -58,23 +59,43 @@ impl<T: Blobstore + Clone> CensoredBlob<T> {
impl<T: Blobstore + Clone> Blobstore for CensoredBlob<T> {
fn get(&self, ctx: CoreContext, key: String) -> BoxFuture<Option<BlobstoreBytes>, Error> {
match &self.censored {
Some(censored) => match censored.get(&key) {
Some(task) => future::err(ErrorKind::Censored(key, task.clone()).into()).boxify(),
None => self.blobstore.get(ctx, key),
},
None => self.blobstore.get(ctx, key),
}
self.err_if_censored(&key)
.into_future()
.map_err({
cloned!(ctx, key);
move |err| {
debug!(
ctx.logger(),
"Accessing censored blobstore with key {:?}", key
);
err
}
})
.and_then({
cloned!(self.blobstore);
move |()| blobstore.get(ctx, key)
})
.boxify()
}
fn put(&self, ctx: CoreContext, key: String, value: BlobstoreBytes) -> BoxFuture<(), Error> {
match &self.censored {
Some(censored) => match censored.get(&key) {
Some(task) => future::err(ErrorKind::Censored(key, task.clone()).into()).boxify(),
None => self.blobstore.put(ctx, key, value),
},
None => self.blobstore.put(ctx, key, value),
}
self.err_if_censored(&key)
.into_future()
.map_err({
cloned!(ctx, key);
move |err| {
debug!(
ctx.logger(),
"Updating censored blobstore with key {:?}", key
);
err
}
})
.and_then({
cloned!(self.blobstore);
move |()| blobstore.put(ctx, key, value)
})
.boxify()
}
fn is_present(&self, ctx: CoreContext, key: String) -> BoxFuture<bool, Error> {

2
cmds/admin/blobstore_fetch.rs
View File

@ -188,7 +188,7 @@ fn get_cache<B: CacheBlobstoreExt>(
mode: String,
) -> BoxFuture<Option<BlobstoreBytes>, Error> {
if mode == "cache-only" {
blobstore.get_cache_only(key)
blobstore.get_cache_only(ctx, key)
} else if mode == "no-fill" {
blobstore.get_no_cache_fill(ctx, key)
} else {