From 00c526f4af48db09ded963cff4266071b1066417 Mon Sep 17 00:00:00 2001 From: Johan Schuijt-Li Date: Thu, 28 Nov 2019 06:32:44 -0800 Subject: [PATCH] config: convert structs to thrift definitions Reviewed By: krallin Differential Revision: D18725768 fbshipit-source-id: 8463d693ec9b907cb6419c1c45882b3c7272e707 --- .../common/commitsyncmap.toml | 2 +- metaconfig/parser/if/config.thrift | 33 ++++++ metaconfig/parser/src/repoconfig.rs | 106 +++++------------- metaconfig/types/src/lib.rs | 20 +++- tests/integration/commitsyncmap.toml | 4 +- tests/integration/library-push-redirector.sh | 2 +- ...test-push-redirector-pushrebase-onesided.t | 4 +- 7 files changed, 83 insertions(+), 88 deletions(-) create mode 100644 metaconfig/parser/if/config.thrift diff --git a/cmds/tests/fixtures/BAD-incorrect-commitsyncmap/common/commitsyncmap.toml b/cmds/tests/fixtures/BAD-incorrect-commitsyncmap/common/commitsyncmap.toml index 6c4ab4d57d..ab948bbf81 100644 --- a/cmds/tests/fixtures/BAD-incorrect-commitsyncmap/common/commitsyncmap.toml +++ b/cmds/tests/fixtures/BAD-incorrect-commitsyncmap/common/commitsyncmap.toml @@ -14,5 +14,5 @@ common_pushrebase_bookmarks = ["master"] bookmark_prefix = "repo2" default_action = "preserve" - [coolsync.small_repos.map] + [coolsync.small_repos.mapping] p1 = "subdir/p2" diff --git a/metaconfig/parser/if/config.thrift b/metaconfig/parser/if/config.thrift new file mode 100644 index 0000000000..6190ca3273 --- /dev/null +++ b/metaconfig/parser/if/config.thrift @@ -0,0 +1,33 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This software may be used and distributed according to the terms of the + * GNU General Public License found in the LICENSE file in the root + * directory of this source tree. + */ + +struct RawCommitSyncSmallRepoConfig { + 1: i32 repoid, + 2: string default_action, + 3: optional string default_prefix, + 4: string bookmark_prefix, + 5: map mapping, + 6: string direction, +} + +struct RawCommitSyncConfig { + 1: i32 large_repo_id, + 2: list common_pushrebase_bookmarks, + 3: list small_repos, +} + + struct RawWireprotoLoggingConfig { + 1: string scribe_category; + 2: string storage_config; + } + +// Raw configuration for health monitoring of the +// source-control-as-a-service solutions +struct RawSourceControlServiceMonitoring { + 1: list bookmarks_to_report_age, +} diff --git a/metaconfig/parser/src/repoconfig.rs b/metaconfig/parser/src/repoconfig.rs index 17de05f813..d9e4cdee20 100644 --- a/metaconfig/parser/src/repoconfig.rs +++ b/metaconfig/parser/src/repoconfig.rs @@ -25,14 +25,18 @@ use ascii::AsciiString; use bookmarks::BookmarkName; use failure_ext::{format_err, prelude::*}; use itertools::Itertools; +use metaconfig_thrift::{ + RawCommitSyncConfig, RawCommitSyncSmallRepoConfig, RawSourceControlServiceMonitoring, + RawWireprotoLoggingConfig, +}; use metaconfig_types::{ BlobConfig, BlobstoreId, BookmarkOrRegex, BookmarkParams, Bundle2ReplayParams, CacheWarmupParams, CommitSyncConfig, CommitSyncDirection, CommonConfig, DefaultSmallToLargeCommitSyncPathAction, FilestoreParams, HookBypass, HookConfig, HookManagerParams, HookParams, HookType, InfinitepushNamespace, InfinitepushParams, LfsParams, MetadataDBConfig, PushParams, PushrebaseParams, Redaction, RepoConfig, RepoReadOnly, - ShardedFilenodesParams, SmallRepoCommitSyncConfig, SourceControlServiceMonitoring, - StorageConfig, WhitelistEntry, WireprotoLoggingConfig, + ShardedFilenodesParams, SmallRepoCommitSyncConfig, StorageConfig, WhitelistEntry, + WireprotoLoggingConfig, }; use mononoke_types::{MPath, RepositoryId}; use regex::Regex; @@ -317,7 +321,7 @@ impl RepoConfigs { default_action, default_prefix, bookmark_prefix, - map, + mapping, direction, } = raw_small_repo_config; @@ -333,7 +337,7 @@ impl RepoConfigs { other => return Err(format_err!("unknown default_action: \"{}\"", other)) }; - let map: Result> = map + let mapping: Result> = mapping .into_iter() .map(|(k, v)| { let k = MPath::new(k)?; @@ -351,7 +355,7 @@ impl RepoConfigs { Ok((RepositoryId::new(repoid), SmallRepoCommitSyncConfig { default_action, - map: map?, + map: mapping?, bookmark_prefix: bookmark_prefix?, direction, })) @@ -1164,71 +1168,11 @@ impl From for FilestoreParams { } } -/// Raw Commit Sync Config for a small repo -#[derive(Clone, Debug, Deserialize, Eq, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct RawCommitSyncSmallRepoConfig { - repoid: i32, - default_action: String, - default_prefix: Option, - bookmark_prefix: String, - map: HashMap, - direction: String, -} - -/// Raw Commit Sync Config -#[derive(Clone, Debug, Deserialize, Eq, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct RawCommitSyncConfig { - large_repo_id: i32, - common_pushrebase_bookmarks: Vec, - small_repos: Vec, -} - -/// Raw Wireproto logging configuration -#[derive(Clone, Debug, Deserialize, Eq, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct RawWireprotoLoggingConfig { - /// Scribe category to log requests to - pub scribe_category: String, - /// Storage where to store arguments for replay - pub storage_config: String, -} - -/// Raw configuration for health monitoring of the source-control-as-a-service -/// solutions -#[derive(Clone, Debug, Deserialize, Eq, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct RawSourceControlServiceMonitoring { - /// Bookmarks, for which we want our services to log - /// freshness values to monitoring counters. For example, - /// a freshness value may be the `now - author_date` of - /// the commit, to which the bookmark points - pub bookmarks_to_report_age: Vec, -} - -impl TryFrom for SourceControlServiceMonitoring { - type Error = Error; - - fn try_from(raw: RawSourceControlServiceMonitoring) -> Result { - let RawSourceControlServiceMonitoring { - bookmarks_to_report_age, - } = raw; - let converted: Result> = bookmarks_to_report_age - .into_iter() - .map(|bookmark| BookmarkName::new(bookmark)) - .collect(); - - Ok(Self { - bookmarks_to_report_age: converted?, - }) - } -} - #[cfg(test)] mod test { use super::*; use maplit::{btreemap, hashmap}; + use metaconfig_types::SourceControlServiceMonitoring; use pretty_assertions::assert_eq; use std::fs::{create_dir_all, write}; use tempdir::TempDir; @@ -1263,7 +1207,7 @@ mod test { bookmark_prefix = "repo2" direction = "small_to_large" - [mega.small_repos.map] + [mega.small_repos.mapping] "p1" = ".r2-legacy/p1" "p5" = ".r2-legacy/p5" @@ -1274,7 +1218,7 @@ mod test { default_prefix = "subdir" direction = "small_to_large" - [mega.small_repos.map] + [mega.small_repos.mapping] "p1" = "p1" "p4" = "p5/p4" "#; @@ -1329,7 +1273,7 @@ mod test { default_action = "preserve" direction = "small_to_large" - [mega.small_repos.map] + [mega.small_repos.mapping] "p1" = ".r2-legacy/p1" "p5" = "subdir" @@ -1340,7 +1284,7 @@ mod test { default_prefix = "subdir" direction = "small_to_large" - [mega.small_repos.map] + [mega.small_repos.mapping] "p1" = "p1" "p4" = "p5/p4" "#; @@ -1367,7 +1311,7 @@ mod test { default_action = "preserve" direction = "large_to_small" - [mega.small_repos.map] + [mega.small_repos.mapping] "p5" = "subdir" [[mega.small_repos]] @@ -1377,7 +1321,7 @@ mod test { default_prefix = "subdir" direction = "large_to_small" - [mega.small_repos.map] + [mega.small_repos.mapping] "p1" = "p1" "#; @@ -1400,7 +1344,7 @@ mod test { default_action = "preserve" direction = "large_to_small" - [mega.small_repos.map] + [mega.small_repos.mapping] "p5" = "subdir/bla" [[mega.small_repos]] @@ -1410,7 +1354,7 @@ mod test { default_prefix = "subdir" direction = "large_to_small" - [mega.small_repos.map] + [mega.small_repos.mapping] "p1" = "p1" "#; @@ -1436,7 +1380,7 @@ mod test { default_action = "preserve" direction = "large_to_small" - [mega.small_repos.map] + [mega.small_repos.mapping] "p5" = "subdir" [[mega.small_repos]] @@ -1446,7 +1390,7 @@ mod test { default_prefix = "subdir" direction = "small_to_large" - [mega.small_repos.map] + [mega.small_repos.mapping] "p1" = "p1" "#; @@ -1470,7 +1414,7 @@ mod test { default_action = "preserve" direction = "large_to_small" - [mega.small_repos.map] + [mega.small_repos.mapping] "p5" = "subdir" [[mega.small_repos]] @@ -1480,7 +1424,7 @@ mod test { default_prefix = "r3" direction = "small_to_large" - [mega.small_repos.map] + [mega.small_repos.mapping] "p1" = "subdir/bla" [[mega.small_repos]] @@ -1490,7 +1434,7 @@ mod test { default_prefix = "r4" direction = "large_to_small" - [mega.small_repos.map] + [mega.small_repos.mapping] "p4" = "subdir" "#; @@ -1515,7 +1459,7 @@ mod test { default_action = "preserve" direction = "small_to_large" - [mega.small_repos.map] + [mega.small_repos.mapping] "p1" = ".r2-legacy/p1" [[mega.small_repos]] @@ -1525,7 +1469,7 @@ mod test { default_prefix = "subdir" direction = "small_to_large" - [mega.small_repos.map] + [mega.small_repos.mapping] "p1" = "p1" "p4" = "p5/p4" "#; diff --git a/metaconfig/types/src/lib.rs b/metaconfig/types/src/lib.rs index 0142f7f589..5bcc05ced8 100644 --- a/metaconfig/types/src/lib.rs +++ b/metaconfig/types/src/lib.rs @@ -12,12 +12,15 @@ #![deny(missing_docs)] #![deny(warnings)] +use anyhow::Error; use std::{ - collections::HashMap, mem, num::NonZeroUsize, path::PathBuf, str, sync::Arc, time::Duration, + collections::HashMap, convert::TryFrom, mem, num::NonZeroUsize, path::PathBuf, str, sync::Arc, + time::Duration, }; use ascii::AsciiString; use bookmarks::BookmarkName; +use metaconfig_thrift::RawSourceControlServiceMonitoring; use mononoke_types::{MPath, RepositoryId}; use regex::Regex; use scuba::ScubaValue; @@ -777,3 +780,18 @@ pub struct SourceControlServiceMonitoring { /// the commit, to which the bookmark points pub bookmarks_to_report_age: Vec, } + +impl TryFrom for SourceControlServiceMonitoring { + type Error = Error; + + fn try_from(t: RawSourceControlServiceMonitoring) -> Result { + let bookmarks_to_report_age = t + .bookmarks_to_report_age + .into_iter() + .map(|bookmark| BookmarkName::new(bookmark)) + .collect::, _>>()?; + Ok(SourceControlServiceMonitoring { + bookmarks_to_report_age, + }) + } +} diff --git a/tests/integration/commitsyncmap.toml b/tests/integration/commitsyncmap.toml index 454dcb1cd2..e8ab85ae31 100644 --- a/tests/integration/commitsyncmap.toml +++ b/tests/integration/commitsyncmap.toml @@ -29,7 +29,7 @@ common_pushrebase_bookmarks = ["master"] default_action = "preserve" direction = "small_to_large" - [megarepo_test.small_repos.map] + [megarepo_test.small_repos.mapping] "arvr" = ".fbsource-rest/arvr" [[megarepo_test.small_repos]] @@ -39,7 +39,7 @@ common_pushrebase_bookmarks = ["master"] default_prefix = "arvr-legacy" direction = "small_to_large" - [megarepo_test.small_repos.map] + [megarepo_test.small_repos.mapping] "arvr" = "arvr" "fbcode" = ".ovrsource-rest/fbcode" "fbandroid" = ".ovrsource-rest/fbandroid" diff --git a/tests/integration/library-push-redirector.sh b/tests/integration/library-push-redirector.sh index 64af860b4c..6bb68829b4 100644 --- a/tests/integration/library-push-redirector.sh +++ b/tests/integration/library-push-redirector.sh @@ -31,7 +31,7 @@ common_pushrebase_bookmarks = ["master_bookmark"] default_action = "prepend_prefix" default_prefix = "smallrepofolder" direction = "large_to_small" - [megarepo_test.small_repos.map] + [megarepo_test.small_repos.mapping] "non_path_shifting" = "non_path_shifting" EOF diff --git a/tests/integration/test-push-redirector-pushrebase-onesided.t b/tests/integration/test-push-redirector-pushrebase-onesided.t index d65f757a6a..c4da1649c9 100644 --- a/tests/integration/test-push-redirector-pushrebase-onesided.t +++ b/tests/integration/test-push-redirector-pushrebase-onesided.t @@ -15,7 +15,7 @@ setup configuration > default_action = "prepend_prefix" > default_prefix = "smallrepofolder1" > direction = "large_to_small" - > [megarepo_test.small_repos.map] + > [megarepo_test.small_repos.mapping] > "special"="specialsmallrepofolder1" > [[megarepo_test.small_repos]] > repoid = 2 @@ -23,7 +23,7 @@ setup configuration > default_action = "prepend_prefix" > default_prefix = "smallrepofolder2" > direction = "small_to_large" - > [megarepo_test.small_repos.map] + > [megarepo_test.small_repos.mapping] > "special"="specialsmallrepofolder2" > EOF