mirror of
https://github.com/facebook/sapling.git
synced 2024-10-10 08:47:12 +03:00
rotatelog: build indexes for older logs on access
Summary: This ensures indexes are complete even if index format or definition has been changed. Reviewed By: DurhamG Differential Revision: D20286509 fbshipit-source-id: fcc4ebc616a4501e4b6fd2f1a9826f54f40b99b8
This commit is contained in:
parent
669c58bd56
commit
66e60bacb9
@ -299,6 +299,16 @@ impl OpenOptions {
|
||||
self
|
||||
}
|
||||
|
||||
/// Remove index lagging.
|
||||
///
|
||||
/// Used by `RotateLog` to make sure old logs have complete indexes.
|
||||
pub(crate) fn with_zero_index_lag(mut self) -> Self {
|
||||
for def in self.index_defs.iter_mut() {
|
||||
def.lag_threshold = 0;
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Construct [`Log`] at given directory. Incrementally build up specified
|
||||
/// indexes.
|
||||
///
|
||||
|
@ -544,7 +544,12 @@ impl RotateLog {
|
||||
let id = self.latest.wrapping_sub(index as u8);
|
||||
if let Some(dir) = &self.dir {
|
||||
Ok(Some(cell.get_or_try_init(|| {
|
||||
load_log(&dir, id, &self.open_options)
|
||||
let mut open_options = self.open_options.log_open_options.clone();
|
||||
if index > 0 {
|
||||
open_options = open_options.with_zero_index_lag();
|
||||
}
|
||||
let log = load_log(&dir, id, open_options);
|
||||
log
|
||||
})?))
|
||||
} else {
|
||||
Ok(cell.get())
|
||||
@ -577,14 +582,10 @@ fn create_log_cell(log: Log) -> OnceCell<Log> {
|
||||
}
|
||||
|
||||
/// Load a single log at the given location.
|
||||
fn load_log(dir: &Path, id: u8, open_options: &OpenOptions) -> crate::Result<Log> {
|
||||
fn load_log(dir: &Path, id: u8, open_options: log::OpenOptions) -> crate::Result<Log> {
|
||||
let name = format!("{}", id);
|
||||
let log_path = dir.join(&name);
|
||||
open_options
|
||||
.log_open_options
|
||||
.clone()
|
||||
.create(false)
|
||||
.open(&log_path)
|
||||
open_options.create(false).open(&log_path)
|
||||
}
|
||||
|
||||
/// Get access to internals of [`RotateLog`].
|
||||
@ -743,7 +744,7 @@ fn read_logs(
|
||||
let mut logs = Vec::with_capacity(open_options.max_log_count as usize);
|
||||
|
||||
// Make sure the first log (latest) can be loaded.
|
||||
let log = load_log(dir, latest, open_options)?;
|
||||
let log = load_log(dir, latest, open_options.log_open_options.clone())?;
|
||||
logs.push(create_log_cell(log));
|
||||
|
||||
// Lazily load the rest of logs.
|
||||
@ -1331,6 +1332,44 @@ mod tests {
|
||||
assert_eq!(rotate.logs()[0].iter_dirty().count(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_reindex_old_logs() {
|
||||
let dir = tempdir().unwrap();
|
||||
let opts = OpenOptions::new()
|
||||
.max_bytes_per_log(10)
|
||||
.max_log_count(10)
|
||||
.create(true);
|
||||
|
||||
let mut rotate = opts.clone().create(true).open(&dir).unwrap();
|
||||
for i in 0..2u8 {
|
||||
rotate.append(vec![i; 50]).unwrap();
|
||||
rotate.sync().unwrap(); // rotate
|
||||
}
|
||||
|
||||
// New OpenOptions: With indexes.
|
||||
let opts = opts.index("a", |_data| vec![IndexOutput::Reference(0..1)]);
|
||||
|
||||
// Triggers rebuilding indexes.
|
||||
let rotate = opts.clone().create(true).open(&dir).unwrap();
|
||||
|
||||
// Because older log is lazy. It hasn't been loaded yet. So it does not have the index.
|
||||
assert!(!dir.path().join("1/index2-a").exists());
|
||||
assert!(!dir.path().join("0/index2-a").exists());
|
||||
|
||||
// Do an index lookup. This will trigger loading old logs.
|
||||
let mut iter = rotate.lookup(0, b"\x00".to_vec()).unwrap();
|
||||
|
||||
// The iterator is lazy. So it does not build the index immediately.
|
||||
assert!(!dir.path().join("1/index2-a").exists());
|
||||
|
||||
// Iterate through all logs.
|
||||
assert_eq!(iter.nth(0).unwrap().unwrap(), &[0; 50][..]);
|
||||
|
||||
// Now the index is built for older logs.
|
||||
assert!(dir.path().join("1/index2-a").exists());
|
||||
assert!(dir.path().join("0/index2-a").exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_repair_latest() {
|
||||
assert_eq!(guess_latest(vec![]), 0);
|
||||
|
Loading…
Reference in New Issue
Block a user