indexedlog: write Checksum inline with Log

Summary:
Enhance the index format: The Root entry can be followed by an optional
Checksum entry which replaces the need of ChecksumTable.

The format is backwards compatible since the old format will be just
treated as "there is no ChecksumTable", and the ChecksumTable will be built on
the next "flush".

This change is non-trivial. But the tests are pretty strong - the bitflip test
alone covered a lot of issues, and the dump of Index content helps a lot too.

For the index itself without ".sum", checksum, this change is bi-directional
compatible:
1. New code reading old file will just think the old file does not have the
   checksum entry, similar to new code having checksum disabled.
2. Old code will think the root+checksum slice is the "root" entry. Parsing
   the root entry is fine since it does not complain about unknown data at the
   end.

However, this change dropped the logic updating ".sum" files. That part is an
issue blocking old clients from reading new data.

Reviewed By: DurhamG

Differential Revision: D19850741

fbshipit-source-id: 551a45cd5422f1fb4c5b08e3b207a2ffe3d93dea

eden/scm/lib/indexedlog/benches/index.rs

@@ -26,7 +26,7 @@ fn gen_buf(size: usize) -> Vec<u8> {
 /// Default open options: 4K checksum chunk
 fn open_opts() -> OpenOptions {
     let mut open_opts = OpenOptions::new();
-    open_opts.checksum_chunk_size(4096);
+    open_opts.checksum_chunk_size_log(12);
     open_opts
 }
 
@@ -108,7 +108,7 @@ fn main() {
     {
         let dir = tempdir().unwrap();
         let mut idx = open_opts()
-            .checksum_chunk_size(0)
+            .checksum_enabled(false)
             .open(dir.path().join("i"))
             .expect("open");
         let buf = gen_buf(N * 20);

eden/scm/lib/indexedlog/src/log/mod.rs

@@ -77,7 +77,6 @@ const ENTRY_FLAG_HAS_XXHASH32: u32 = 2;
 
 // 1MB index checksum. This makes checksum file within one block (4KB) for 512MB index.
 const INDEX_CHECKSUM_CHUNK_SIZE_LOG: u32 = 20;
-const INDEX_CHECKSUM_CHUNK_SIZE: u64 = 1u64 << INDEX_CHECKSUM_CHUNK_SIZE_LOG;
 
 /// An append-only storage with indexes and integrity checks.
 ///
@@ -1245,7 +1244,7 @@ impl Log {
             Some(dir) => {
                 let path = dir.join(format!("{}{}", INDEX_FILE_PREFIX, name));
                 index::OpenOptions::new()
-                    .checksum_chunk_size(INDEX_CHECKSUM_CHUNK_SIZE)
+                    .checksum_chunk_size_log(INDEX_CHECKSUM_CHUNK_SIZE_LOG)
                     .logical_len(Some(len))
                     .key_buf(Some(buf))
                     .fsync(fsync)

eden/scm/lib/indexedlog/src/log/tests.rs

@@ -793,7 +793,6 @@ fn test_repair_and_delete_content() {
     let truncate = |name: &str| fs::write(path.join(name), "garbage").unwrap();
     let delete = |name: &str| fs::remove_file(path.join(name)).unwrap();
     let index_file = format!("{}c", INDEX_FILE_PREFIX);
-    let checksum_file = format!("{}c.sum", INDEX_FILE_PREFIX);
     let append = || {
         let mut log = open().unwrap();
         log.append(&[b'x'; 50_000][..]).unwrap();
@@ -927,16 +926,6 @@ Rebuilt index "c""#
     );
     verify_len(5);
 
-    // Corrupt index checksum
-    corrupt(&checksum_file, -2);
-    verify_corrupted();
-    assert_eq!(
-        repair(),
-        r#"Verified 5 entries, 250072 bytes in log
-Rebuilt index "c""#
-    );
-    verify_len(5);
-
     // Replace index with garbage
     truncate(&index_file);
     verify_corrupted();
@@ -947,16 +936,6 @@ Rebuilt index "c""#
     );
     verify_len(5);
 
-    // Replace index checksum with garbage
-    truncate(&checksum_file);
-    verify_corrupted();
-    assert_eq!(
-        repair(),
-        r#"Verified 5 entries, 250072 bytes in log
-Rebuilt index "c""#
-    );
-    verify_len(5);
-
     // Replace log with garbage
     truncate(PRIMARY_FILE);
     verify_corrupted();
@@ -984,16 +963,6 @@ Rebuilt index "c""#
     );
     verify_len(3);
 
-    // Delete checksum
-    delete(&checksum_file);
-    verify_corrupted();
-    assert_eq!(
-        repair(),
-        r#"Verified 3 entries, 150048 bytes in log
-Rebuilt index "c""#
-    );
-    verify_len(3);
-
     // Delete log
     delete(PRIMARY_FILE);
     verify_corrupted();
@@ -1077,12 +1046,12 @@ Rebuilt index "c""#
         log.append(&[b'z'; 50_000][..]).unwrap();
         log.sync().unwrap();
         assert_eq!(len(PRIMARY_FILE), PRIMARY_START_OFFSET + 150036);
-        assert_eq!(len(&index_file), 70);
+        assert_eq!(len(&index_file), 100);
     };
     let delete_content = || {
         open_opts.delete_content(path).unwrap();
         assert_eq!(len(PRIMARY_FILE), PRIMARY_START_OFFSET);
-        assert_eq!(len(&index_file), 10);
+        assert_eq!(len(&index_file), 25);
         // Check SIGBUS
         try_trigger_sigbus();
         // Check log is empty
@@ -1093,9 +1062,6 @@ Rebuilt index "c""#
     fs::remove_dir_all(&path).unwrap();
     delete_content();
 
-    // Empty log
-    delete_content();
-
     // Normal log
     append();
     if cfg!(unix) {
@@ -1113,11 +1079,6 @@ Rebuilt index "c""#
     corrupt(&index_file, -10);
     delete_content();
 
-    // Corrupt checksum
-    append();
-    corrupt(&checksum_file, -10);
-    delete_content();
-
     // Corrupt log and index
     append();
     corrupt(PRIMARY_FILE, -25_000);
@@ -1125,9 +1086,6 @@ Rebuilt index "c""#
     delete_content();
 
     // Deleted various files
-    delete(&checksum_file);
-    delete_content();
-
     delete(&index_file);
     delete_content();
 

eden/scm/lib/indexedlog/src/rotate.rs

@@ -1273,7 +1273,7 @@ mod tests {
         // The "current" log is still mutable. Its index respects lag_threshold,
         // and is logically empty (because side effect of delete_content, the
         // index has some bytes in it).
-        assert_eq!(size("2/index-idx"), 10);
+        assert_eq!(size("2/index-idx"), 25);
         assert!(size("2/log") < 100);
     }