From 188d65defbb5bcf1d4b08fd592a25e99348b6a19 Mon Sep 17 00:00:00 2001 From: Ilia Medianikov Date: Wed, 16 Mar 2022 03:41:42 -0700 Subject: [PATCH] path_acls: parse repo regions configuration in Mononoke Summary: Adding a new field to the `RepoConfig`. Not used anywhere yet. Reviewed By: markbt Differential Revision: D34555579 fbshipit-source-id: cba8e3eeeb1ebbc2341f1bf55cc741bdfa7a1279 --- .../structs/scm/mononoke/repos/repos.thrift | 55 +++++++++++++- eden/mononoke/metaconfig/parser/Cargo.toml | 1 + eden/mononoke/metaconfig/parser/src/config.rs | 73 +++++++++++++++---- .../parser/src/convert/acl_regions.rs | 61 ++++++++++++++++ .../metaconfig/parser/src/convert/mod.rs | 1 + eden/mononoke/metaconfig/parser/src/raw.rs | 12 ++- eden/mononoke/metaconfig/types/src/lib.rs | 49 +++++++++++++ 7 files changed, 236 insertions(+), 16 deletions(-) create mode 100644 eden/mononoke/metaconfig/parser/src/convert/acl_regions.rs diff --git a/configerator/structs/scm/mononoke/repos/repos.thrift b/configerator/structs/scm/mononoke/repos/repos.thrift index 7dafd8ed50..2b418796b9 100644 --- a/configerator/structs/scm/mononoke/repos/repos.thrift +++ b/configerator/structs/scm/mononoke/repos/repos.thrift @@ -1,4 +1,4 @@ -// @generated SignedSource<> +// @generated SignedSource<<9d8514cd3c8cd9adab4e3a6a50f2b688>> // DO NOT EDIT THIS FILE MANUALLY! // This file is a mechanical copy of the version in the configerator repo. To // modify it, edit the copy in the configerator repo instead and copy it over by @@ -43,6 +43,7 @@ struct RawRepoConfigs { 2: RawCommonConfig common, 3: map (rust.type = "HashMap") repos, # to be renamed to repo_configs 4: map (rust.type = "HashMap") storage, # to be renamed to storage_configs + 6: map (rust.type = "HashMap") acl_region_configs, 5: RawRepoDefinitions repo_definitions, } (rust.exhaustive) @@ -86,6 +87,10 @@ struct RawRepoDefinition { // In case this is a backup repo, what's the origin repo name? 10: optional string backup_source_repo_name, + + // Key into RawRepoConfigs.acl_region_configs for the definition of + // ACL regions for this repo. + 11: optional string acl_region_config, } (rust.exhaustive) struct RawRepoConfig { @@ -650,3 +655,51 @@ struct RawSegmentedChangelogConfig { // ancestors to the reseeded segmented changelog. 8: optional list bonsai_changesets_to_include, } (rust.exhaustive) + +// Describe ACL Regions for a repository. +// +// This is a set of rules which define regions of the repository (commits and paths) +struct RawAclRegionConfig { + // List of rules that grant access to regions of the repo. + 1: list allow_rules, + + // List of regions to which default access to the repository is denied. + // 2: list deny_default_regions, (not yet implemented) +} (rust.exhaustive) + +struct RawAclRegionRule { + // The name of this region rule. This is used in error messages and diagnostics. + 1: string name, + + // A list of regions that this rule applies to. + 2: list regions, + + // The hipster ACL that defines who is permitted to access the regions of + // the repo defined by this rule. + 3: string hipster_acl, +} (rust.exhaustive) + +// Define a region of the repository, in terms of commits and path prefixes. +// +// The commit range is equivalent to the Mercurial revset +// descendants(roots) - descendants(heads) +// +// If the roots and heads lists are both empty then this region covers the +// entire repo. +struct RawAclRegion { + // List of roots that begin this region. Any commit that is a descendant of any + // root, including the root itself, will be included in the region. If this + // list is empty then all commits are included (provided they are not the + // descendant of a head). + 1: list roots, + + // List of heads that end this region. Any commit that is a descendant of + // any head, includin the head itself, will NOT be included in the region. + // If this list is empty then all commits that are descendants of the roots + // are included. + 2: list heads, + + // List of path prefixes that apply to this region. Prefixes are in terms of + // path elements, so the prefix a/b applies to a/b/c but not a/bb. + 3: list path_prefixes, +} (rust.exhaustive) diff --git a/eden/mononoke/metaconfig/parser/Cargo.toml b/eden/mononoke/metaconfig/parser/Cargo.toml index 5772a34041..bce1258bef 100644 --- a/eden/mononoke/metaconfig/parser/Cargo.toml +++ b/eden/mononoke/metaconfig/parser/Cargo.toml @@ -27,5 +27,6 @@ toml = "=0.5.8" [dev-dependencies] maplit = "1.0" +mononoke_types-mocks = { version = "0.1.0", path = "../../mononoke_types/mocks" } pretty_assertions = "0.6" tempdir = "0.3" diff --git a/eden/mononoke/metaconfig/parser/src/config.rs b/eden/mononoke/metaconfig/parser/src/config.rs index 88a4735dd9..fd56e78c4e 100644 --- a/eden/mononoke/metaconfig/parser/src/config.rs +++ b/eden/mononoke/metaconfig/parser/src/config.rs @@ -23,7 +23,10 @@ use metaconfig_types::{ StorageConfig, }; use mononoke_types::RepositoryId; -use repos::{RawCommonConfig, RawRepoConfig, RawRepoConfigs, RawRepoDefinition, RawStorageConfig}; +use repos::{ + RawAclRegionConfig, RawCommonConfig, RawRepoConfig, RawRepoConfigs, RawRepoDefinition, + RawStorageConfig, +}; const LIST_KEYS_PATTERNS_MAX_DEFAULT: u64 = 500_000; const HOOK_MAX_FILE_SIZE_DEFAULT: u64 = 8 * 1024 * 1024; // 8MiB @@ -59,6 +62,7 @@ pub fn load_repo_configs( common, repos, storage, + acl_region_configs, repo_definitions, } = crate::raw::read_raw_configs(config_path.as_ref(), config_store)?; let repo_definitions = repo_definitions.repo_definitions; @@ -70,8 +74,12 @@ pub fn load_repo_configs( let mut repoids = HashSet::new(); for (reponame, raw_repo_definition) in repo_definitions.into_iter() { - let repo_config = - parse_with_repo_definition(raw_repo_definition, &repo_configs, &storage_configs)?; + let repo_config = parse_with_repo_definition( + raw_repo_definition, + &repo_configs, + &storage_configs, + &acl_region_configs, + )?; if !repoids.insert(repo_config.repoid) { return Err(ConfigurationError::DuplicatedRepoId(repo_config.repoid).into()); @@ -92,6 +100,7 @@ fn parse_with_repo_definition( repo_definition: RawRepoDefinition, named_repo_configs: &HashMap, named_storage_configs: &HashMap, + named_acl_region_configs: &HashMap, ) -> Result { let RawRepoDefinition { repo_id: repoid, @@ -104,6 +113,7 @@ fn parse_with_repo_definition( readonly, needs_backup: _, external_repo_id: _, + acl_region_config, } = repo_definition; let named_repo_config_name = repo_config @@ -266,6 +276,17 @@ fn parse_with_repo_definition( let repo_client_knobs = repo_client_knobs.convert()?.unwrap_or_default(); + let acl_region_config = acl_region_config + .map(|key| { + named_acl_region_configs.get(&key).cloned().ok_or_else(|| { + ConfigurationError::InvalidConfig(format!( + "ACL region config \"{}\" not defined", + key + )) + }) + }) + .transpose()? + .convert()?; Ok(RepoConfig { enabled, @@ -303,6 +324,7 @@ fn parse_with_repo_definition( repo_client_knobs, phabricator_callsign, backup_repo_config, + acl_region_config, }) } @@ -444,18 +466,19 @@ mod test { use cached_config::TestSource; use maplit::{btreemap, hashmap, hashset}; use metaconfig_types::{ - BlameVersion, BlobConfig, BlobstoreId, BookmarkParams, Bundle2ReplayParams, - CacheWarmupParams, CommitSyncConfig, CommitSyncConfigVersion, DatabaseConfig, - DefaultSmallToLargeCommitSyncPathAction, DerivedDataConfig, DerivedDataTypesConfig, - EphemeralBlobstoreConfig, FilestoreParams, HookBypass, HookConfig, HookManagerParams, - HookParams, InfinitepushNamespace, InfinitepushParams, LfsParams, LocalDatabaseConfig, - MetadataDatabaseConfig, MultiplexId, MultiplexedStoreType, PushParams, PushrebaseFlags, - PushrebaseParams, RemoteDatabaseConfig, RemoteMetadataDatabaseConfig, RepoClientKnobs, - SegmentedChangelogConfig, ShardableRemoteDatabaseConfig, ShardedRemoteDatabaseConfig, - SmallRepoCommitSyncConfig, SourceControlServiceMonitoring, SourceControlServiceParams, - UnodeVersion, + AclRegion, AclRegionConfig, AclRegionRule, BlameVersion, BlobConfig, BlobstoreId, + BookmarkParams, Bundle2ReplayParams, CacheWarmupParams, CommitSyncConfig, + CommitSyncConfigVersion, DatabaseConfig, DefaultSmallToLargeCommitSyncPathAction, + DerivedDataConfig, DerivedDataTypesConfig, EphemeralBlobstoreConfig, FilestoreParams, + HookBypass, HookConfig, HookManagerParams, HookParams, InfinitepushNamespace, + InfinitepushParams, LfsParams, LocalDatabaseConfig, MetadataDatabaseConfig, MultiplexId, + MultiplexedStoreType, PushParams, PushrebaseFlags, PushrebaseParams, RemoteDatabaseConfig, + RemoteMetadataDatabaseConfig, RepoClientKnobs, SegmentedChangelogConfig, + ShardableRemoteDatabaseConfig, ShardedRemoteDatabaseConfig, SmallRepoCommitSyncConfig, + SourceControlServiceMonitoring, SourceControlServiceParams, UnodeVersion, }; use mononoke_types::MPath; + use mononoke_types_mocks::changesetid::ONES_CSID; use nonzero_ext::nonzero; use pretty_assertions::assert_eq; use regex::Regex; @@ -797,6 +820,7 @@ mod test { repo_config="fbsource" needs_backup=false backup_source_repo_name="source" + acl_region_config="fbsource" "#; let www_content = r#" scuba_table_hooks="scm_hooks" @@ -861,11 +885,22 @@ mod test { path = "/tmp/www-ephemeral" "#; + let acl_region_configs = r#" + [[fbsource.allow_rules]] + name = "name_test" + hipster_acl = "acl_test" + [[fbsource.allow_rules.regions]] + roots = ["1111111111111111111111111111111111111111111111111111111111111111"] + heads = [] + path_prefixes = ["test/prefix"] + "#; + let paths = btreemap! { "common/storage.toml" => storage, "common/common.toml" => common_content, "common/commitsyncmap.toml" => "", + "common/acl_regions.toml" => acl_region_configs, "repos/fbsource/server.toml" => fbsource_content, "repos/www/server.toml" => www_content, "repo_definitions/fbsource/server.toml" => fbsource_repo_def, @@ -1087,6 +1122,17 @@ mod test { backup_repo_config: Some(BackupRepoConfig { source_repo_name: "source".to_string(), }), + acl_region_config: Some(AclRegionConfig { + allow_rules: vec![AclRegionRule { + name: "name_test".to_string(), + regions: vec![AclRegion { + roots: vec![ONES_CSID], + heads: vec![], + path_prefixes: vec![MPath::new("test/prefix").unwrap()], + }], + hipster_acl: "acl_test".to_string(), + }], + }), }, ); @@ -1153,6 +1199,7 @@ mod test { repo_client_knobs: RepoClientKnobs::default(), phabricator_callsign: Some("WWW".to_string()), backup_repo_config: None, + acl_region_config: None, }, ); assert_eq!( diff --git a/eden/mononoke/metaconfig/parser/src/convert/acl_regions.rs b/eden/mononoke/metaconfig/parser/src/convert/acl_regions.rs new file mode 100644 index 0000000000..9029e4e992 --- /dev/null +++ b/eden/mononoke/metaconfig/parser/src/convert/acl_regions.rs @@ -0,0 +1,61 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This software may be used and distributed according to the terms of the + * GNU General Public License version 2. + */ + +use std::str::FromStr; + +use anyhow::Result; +use metaconfig_types::{AclRegion, AclRegionConfig, AclRegionRule}; +use mononoke_types::{ChangesetId, MPath}; +use repos::{RawAclRegion, RawAclRegionConfig, RawAclRegionRule}; + +use crate::convert::Convert; + +impl Convert for RawAclRegion { + type Output = AclRegion; + + fn convert(self) -> Result { + Ok(AclRegion { + roots: self + .roots + .into_iter() + .map(|s| ChangesetId::from_str(&s)) + .collect::>>()?, + heads: self + .heads + .into_iter() + .map(|s| ChangesetId::from_str(&s)) + .collect::>>()?, + path_prefixes: self + .path_prefixes + .into_iter() + .map(|b| MPath::try_from(&*b)) + .collect::>>()?, + }) + } +} + +impl Convert for RawAclRegionRule { + type Output = AclRegionRule; + + fn convert(self) -> Result { + Ok(AclRegionRule { + name: self.name, + regions: self.regions.convert()?, + hipster_acl: self.hipster_acl, + }) + } +} + +impl Convert for RawAclRegionConfig { + type Output = AclRegionConfig; + + fn convert(self) -> Result { + Ok(AclRegionConfig { + allow_rules: self.allow_rules.convert()?, + }) + } +} diff --git a/eden/mononoke/metaconfig/parser/src/convert/mod.rs b/eden/mononoke/metaconfig/parser/src/convert/mod.rs index 9c343f4a86..1f8ec3213f 100644 --- a/eden/mononoke/metaconfig/parser/src/convert/mod.rs +++ b/eden/mononoke/metaconfig/parser/src/convert/mod.rs @@ -7,6 +7,7 @@ use anyhow::Result; +mod acl_regions; mod commit_sync; pub(crate) mod repo; mod storage; diff --git a/eden/mononoke/metaconfig/parser/src/raw.rs b/eden/mononoke/metaconfig/parser/src/raw.rs index b41d75e6a9..13ba397e8d 100644 --- a/eden/mononoke/metaconfig/parser/src/raw.rs +++ b/eden/mononoke/metaconfig/parser/src/raw.rs @@ -11,8 +11,8 @@ use std::path::Path; use anyhow::{anyhow, Result}; use cached_config::ConfigStore; use repos::{ - RawCommitSyncConfig, RawCommonConfig, RawRepoConfig, RawRepoConfigs, RawRepoDefinition, - RawRepoDefinitions, RawStorageConfig, + RawAclRegionConfig, RawCommitSyncConfig, RawCommonConfig, RawRepoConfig, RawRepoConfigs, + RawRepoDefinition, RawRepoDefinitions, RawStorageConfig, }; use crate::errors::ConfigurationError; @@ -62,6 +62,13 @@ fn read_raw_configs_toml(config_path: &Path) -> Result { config_path.join("common").join("storage.toml").as_path(), true, )?; + let acl_region_configs = read_toml_path::>( + config_path + .join("common") + .join("acl_regions.toml") + .as_path(), + true, + )?; let mut repo_definitions_map = HashMap::new(); let repo_definitions_dir = config_path.join("repo_definitions"); @@ -129,6 +136,7 @@ fn read_raw_configs_toml(config_path: &Path) -> Result { common, repos, storage, + acl_region_configs, repo_definitions, }) } diff --git a/eden/mononoke/metaconfig/types/src/lib.rs b/eden/mononoke/metaconfig/types/src/lib.rs index a1261ec6d2..2979419cdb 100644 --- a/eden/mononoke/metaconfig/types/src/lib.rs +++ b/eden/mononoke/metaconfig/types/src/lib.rs @@ -207,6 +207,8 @@ pub struct RepoConfig { /// If it's a backup repo, then this field stores information /// about the backup configuration pub backup_repo_config: Option, + /// ACL region configuration + pub acl_region_config: Option, } /// Backup repo configuration @@ -1534,3 +1536,50 @@ impl Default for SegmentedChangelogConfig { } } } + +/// Define a region of the repository, in terms of commits and path prefixes. +/// +/// The commit range is equivalent to the Mercurial revset +/// descendants(roots) - descendants(heads) +/// +/// If the roots and heads lists are both empty then this region covers the +/// entire repo. +#[derive(Debug, Default, Clone, Eq, PartialEq)] +pub struct AclRegion { + /// List of roots that begin this region. Any commit that is a descendant of any + /// root, including the root itself, will be included in the region. If this + /// list is empty then all commits are included (provided they are not the + /// descendant of a head). + pub roots: Vec, + + /// List of heads that end this region. Any commit that is a descendant of + /// any head, includin the head itself, will NOT be included in the region. + /// If this list is empty then all commits that are descendants of the roots + /// are included. + pub heads: Vec, + + /// List of path prefixes that apply to this region. Prefixes are in terms of + /// path elements, so the prefix a/b applies to a/b/c but not a/bb. + pub path_prefixes: Vec, +} + +/// ACL region rule consisting of multiple regions and path prefixes +#[derive(Debug, Default, Clone, Eq, PartialEq)] +pub struct AclRegionRule { + /// The name of this region rule. This is used in error messages and diagnostics. + pub name: String, + /// A list of regions that this rule applies to. + pub regions: Vec, + /// The hipster ACL that defines who is permitted to access the regions of + /// the repo defined by this rule. + pub hipster_acl: String, +} + +/// Describe ACL Regions for a repository. +/// +/// This is a set of rules which define regions of the repository (commits and paths) +#[derive(Debug, Default, Clone, Eq, PartialEq)] +pub struct AclRegionConfig { + /// List of rules that grant access to regions of the repo. + pub allow_rules: Vec, +}