mononoke: skiplist thrift serialization

Summary:
We'd like to save skiplist data in blobstore so that we won't have to recompute
it. Let's use thrift serialization for it

Reviewed By: lukaspiatkowski

Differential Revision: D13122386

fbshipit-source-id: f44cdcf38fa6a219df9217906a872e60c319e391

mononoke-types/src/generation.rs

@@ -32,6 +32,10 @@ impl Generation {
         Generation(u64::MAX)
     }
 
+    pub fn value(&self) -> u64 {
+        self.0
+    }
+
     /// The difference from this generation to the other as the difference in their
     /// generation numbers.
     /// If this Generation is smaller than the other, return None.

reachabilityindex/if/skiplist.thrift

@@ -0,0 +1,37 @@
+// Copyright (c) 2018-present, Facebook, Inc.
+// All Rights Reserved.
+//
+// This software may be used and distributed according to the terms of the
+// GNU General Public License version 2 or any later version.
+
+include "scm/mononoke/mononoke-types/if/mononoke_types_thrift.thrift"
+
+struct SkiplistEntry {
+  1: RepoId repo_id,
+  2: mononoke_types_thrift.ChangesetId cs_id,
+  3: GenerationNum gen,
+  4: SkiplistNodeType node_type,
+}
+
+typedef i32 RepoId (hs.newtype)
+
+ // Thrift does not support unsigned, so using i64 here
+typedef i64 GenerationNum (hs.newtype)
+
+struct CommitAndGenerationNumber {
+  1: mononoke_types_thrift.ChangesetId cs_id,
+  2: GenerationNum gen,
+}
+
+struct SkipEdges {
+  1: list<CommitAndGenerationNumber> edges,
+}
+
+struct ParentEdges {
+  1: list<CommitAndGenerationNumber> edges,
+}
+
+union SkiplistNodeType {
+  1: SkipEdges SkipEdges,
+  2: ParentEdges ParentEdges,
+}

reachabilityindex/src/lib.rs

@@ -5,6 +5,7 @@
 // GNU General Public License version 2 or any later version.
 
 #![deny(warnings)]
+extern crate bytes;
 extern crate chashmap;
 #[macro_use]
 extern crate cloned;
@@ -17,6 +18,9 @@ extern crate blobrepo;
 extern crate mercurial_types;
 extern crate mononoke_types;
 
+extern crate rust_thrift;
+extern crate skiplist_thrift;
+
 mod helpers;
 
 pub mod errors;
@@ -29,7 +33,7 @@ mod genbfs;
 pub use genbfs::GenerationNumberBFS;
 
 mod skiplist;
-pub use skiplist::SkiplistIndex;
+pub use skiplist::{SkiplistIndex, SkiplistNodeType};
 #[cfg(test)]
 pub extern crate async_unit;
 #[cfg(test)]

reachabilityindex/src/skiplist.rs

@@ -8,6 +8,7 @@ use std::collections::{HashMap, HashSet};
 use std::ops::Deref;
 use std::sync::Arc;
 
+use bytes::Bytes;
 use chashmap::CHashMap;
 use failure::Error;
 use futures::Stream;
@@ -22,13 +23,17 @@ use mononoke_types::{ChangesetId, Generation};
 use helpers::{advance_bfs_layer, changesets_with_generation_numbers, check_if_node_exists,
               fetch_generation_and_join, get_parents};
 use index::{LeastCommonAncestorsHint, NodeFrontier, ReachabilityIndex};
+use skiplist_thrift;
+
+use rust_thrift::compact_protocol;
 
 const DEFAULT_EDGE_COUNT: u32 = 10;
 
 // Each indexed node fits into one of two categories:
 // - It has skiplist edges
 // - It only has edges to its parents.
-enum SkiplistNodeType {
+#[derive(Clone)]
+pub enum SkiplistNodeType {
     // A list of skip edges which keep doubling
     // in distance from their root node.
     // The ith skip edge is at most 2^i commits away.
@@ -36,6 +41,41 @@ enum SkiplistNodeType {
     ParentEdges(Vec<(ChangesetId, Generation)>),
 }
 
+impl SkiplistNodeType {
+    pub fn to_thrift(&self) -> skiplist_thrift::SkiplistNodeType {
+        fn encode_vec_to_thrift(
+            cs_gen: Vec<(ChangesetId, Generation)>,
+        ) -> Vec<skiplist_thrift::CommitAndGenerationNumber> {
+            cs_gen
+                .into_iter()
+                .map(|(cs_id, gen_num)| {
+                    let cs_id = cs_id.into_thrift();
+                    let gen = skiplist_thrift::GenerationNum(gen_num.value() as i64);
+                    skiplist_thrift::CommitAndGenerationNumber { cs_id, gen }
+                })
+                .collect()
+        }
+
+        match self {
+            SkiplistNodeType::SkipEdges(edges) => {
+                let edges = encode_vec_to_thrift(edges.clone());
+                let skip_edges = skiplist_thrift::SkipEdges { edges };
+                skiplist_thrift::SkiplistNodeType::SkipEdges(skip_edges)
+            }
+            SkiplistNodeType::ParentEdges(parent_edges) => {
+                let edges = encode_vec_to_thrift(parent_edges.clone());
+                let parent_edges = skiplist_thrift::ParentEdges { edges };
+                skiplist_thrift::SkiplistNodeType::ParentEdges(parent_edges)
+            }
+        }
+    }
+
+    pub fn serialize(&self) -> Bytes {
+        let thrift_skiplist_node_type = self.to_thrift();
+        compact_protocol::serialize(&thrift_skiplist_node_type)
+    }
+}
+
 struct SkiplistEdgeMapping {
     pub mapping: CHashMap<ChangesetId, SkiplistNodeType>,
     pub skip_edges_per_node: u32,