From d552144478889587ae558b9e829873073550b481 Mon Sep 17 00:00:00 2001 From: "Zeyi (Rice) Fan" Date: Fri, 12 Feb 2021 12:30:49 -0800 Subject: [PATCH] configparser: move conversion related to a separated module Summary: Move these conversion related function and trait out of `hg` module so EdenFS can use it too. Changes: * Moved `get_opt`, `get_or` and `get_or_default` directly into `ConfigSet`. * Moved `FromConfigValue` and `ByteCount` into `configparser::convert`. Reviewed By: quark-zju Differential Revision: D26355403 fbshipit-source-id: 9096b7b737bc4a0cccee1a3883e89a323f864fac --- .../modules/pyconfigparser/Cargo.toml | 1 - .../modules/pyconfigparser/src/lib.rs | 3 +- eden/scm/lib/backingstore/src/backingstore.rs | 1 - eden/scm/lib/clidispatch/Cargo.toml | 3 - eden/scm/lib/clidispatch/src/dispatch.rs | 3 +- eden/scm/lib/clidispatch/src/io.rs | 2 +- eden/scm/lib/configparser/src/config.rs | 102 +++ eden/scm/lib/configparser/src/convert.rs | 507 +++++++++++++++ eden/scm/lib/configparser/src/hg.rs | 585 ------------------ eden/scm/lib/configparser/src/lib.rs | 1 + eden/scm/lib/edenapi/src/builder.rs | 2 +- .../scm/lib/revisionstore/src/contentstore.rs | 5 +- .../revisionstore/src/indexedlogdatastore.rs | 5 +- .../src/indexedloghistorystore.rs | 5 +- eden/scm/lib/revisionstore/src/lfs.rs | 5 +- .../lib/revisionstore/src/metadatastore.rs | 5 +- eden/scm/lib/revisionstore/src/repack.rs | 2 +- eden/scm/lib/revisionstore/src/util.rs | 2 +- 18 files changed, 622 insertions(+), 617 deletions(-) create mode 100644 eden/scm/lib/configparser/src/convert.rs diff --git a/eden/scm/edenscmnative/bindings/modules/pyconfigparser/Cargo.toml b/eden/scm/edenscmnative/bindings/modules/pyconfigparser/Cargo.toml index 20a16d7374..6587de0750 100644 --- a/eden/scm/edenscmnative/bindings/modules/pyconfigparser/Cargo.toml +++ b/eden/scm/edenscmnative/bindings/modules/pyconfigparser/Cargo.toml @@ -5,7 +5,6 @@ edition = "2018" [dependencies] anyhow = "1.0" -bytes = "0.5" configparser = { path = "../../../../lib/configparser" } cpython = { version = "0.5", default-features = false } cpython-ext = { path = "../../../../lib/cpython-ext", default-features = false } diff --git a/eden/scm/edenscmnative/bindings/modules/pyconfigparser/src/lib.rs b/eden/scm/edenscmnative/bindings/modules/pyconfigparser/src/lib.rs index 97fe98041b..00deead030 100644 --- a/eden/scm/edenscmnative/bindings/modules/pyconfigparser/src/lib.rs +++ b/eden/scm/edenscmnative/bindings/modules/pyconfigparser/src/lib.rs @@ -13,8 +13,9 @@ use cpython::*; use configparser::{ config::{ConfigSet, Options}, + convert::parse_list, dynamicconfig::Generator, - hg::{generate_dynamicconfig, parse_list, ConfigSetHgExt, OptionsHgExt}, + hg::{generate_dynamicconfig, ConfigSetHgExt, OptionsHgExt}, }; use cpython_ext::{error::ResultPyErrExt, PyNone, PyPath, PyPathBuf, Str}; diff --git a/eden/scm/lib/backingstore/src/backingstore.rs b/eden/scm/lib/backingstore/src/backingstore.rs index 144b1ea55c..248cb2e692 100644 --- a/eden/scm/lib/backingstore/src/backingstore.rs +++ b/eden/scm/lib/backingstore/src/backingstore.rs @@ -9,7 +9,6 @@ use crate::remotestore::FakeRemoteStore; use crate::treecontentstore::TreeContentStore; use crate::utils::key_from_path_node_slice; use anyhow::Result; -use configparser::hg::ConfigSetHgExt; use edenapi::{Builder as EdenApiBuilder, EdenApi}; use log::warn; use manifest::{List, Manifest}; diff --git a/eden/scm/lib/clidispatch/Cargo.toml b/eden/scm/lib/clidispatch/Cargo.toml index 6d4d26a4ec..bde24a09d9 100644 --- a/eden/scm/lib/clidispatch/Cargo.toml +++ b/eden/scm/lib/clidispatch/Cargo.toml @@ -18,6 +18,3 @@ thiserror = "1.0.5" thrift-types = { path = "../thrift-types" } tracing = "0.1" util = { path = "../util" } - -[dev-dependencies] -tempfile = "3.0.7" diff --git a/eden/scm/lib/clidispatch/src/dispatch.rs b/eden/scm/lib/clidispatch/src/dispatch.rs index a7ee60a95c..eafb3f1a03 100644 --- a/eden/scm/lib/clidispatch/src/dispatch.rs +++ b/eden/scm/lib/clidispatch/src/dispatch.rs @@ -14,7 +14,6 @@ use anyhow::Error; use cliparser::alias::{expand_aliases, find_command_name}; use cliparser::parser::{ParseError, ParseOptions, ParseOutput, StructFlags}; use configparser::config::ConfigSet; -use configparser::hg::ConfigSetHgExt; use std::convert::TryInto; use std::sync::atomic::Ordering::SeqCst; use std::{env, path::Path}; @@ -104,7 +103,7 @@ fn initialize_blackbox(optional_repo: &OptionalRepo) -> Result<()> { let config = repo.config(); let max_size = config .get_or("blackbox", "maxsize", || { - configparser::hg::ByteCount::from(1u64 << 12) + configparser::convert::ByteCount::from(1u64 << 12) })? .value(); let max_files = config.get_or("blackbox", "maxfiles", || 3)?; diff --git a/eden/scm/lib/clidispatch/src/io.rs b/eden/scm/lib/clidispatch/src/io.rs index 507ded1614..3cfa9bf019 100644 --- a/eden/scm/lib/clidispatch/src/io.rs +++ b/eden/scm/lib/clidispatch/src/io.rs @@ -5,7 +5,7 @@ * GNU General Public License version 2. */ -use configparser::{config::ConfigSet, hg::ConfigSetHgExt}; +use configparser::config::ConfigSet; use pipe::pipe; use std::any::Any; use std::io; diff --git a/eden/scm/lib/configparser/src/config.rs b/eden/scm/lib/configparser/src/config.rs index 30af359e84..4cf70c2057 100644 --- a/eden/scm/lib/configparser/src/config.rs +++ b/eden/scm/lib/configparser/src/config.rs @@ -15,11 +15,13 @@ use std::path::{Path, PathBuf}; use std::str; use std::sync::Arc; +use anyhow::Result; use indexmap::IndexMap; use minibytes::Text; use pest::{self, Parser, Span}; use util::path::expand_path; +use crate::convert::FromConfigValue; use crate::error::Error; use crate::parser::{ConfigParser, Rule}; @@ -152,6 +154,36 @@ impl ConfigSet { .unwrap_or_default() } + /// Get a config item. Convert to type `T`. + pub fn get_opt(&self, section: &str, name: &str) -> Result> { + self.get(section, name) + .map(|bytes| T::try_from_str(&bytes)) + .transpose() + } + + /// Get a config item. Convert to type `T`. + /// + /// If the config item is not set, calculate it using `default_func`. + pub fn get_or( + &self, + section: &str, + name: &str, + default_func: impl Fn() -> T, + ) -> Result { + Ok(self.get_opt(section, name)?.unwrap_or_else(default_func)) + } + + /// Get a config item. Convert to type `T`. + /// + /// If the config item is not set, return `T::default()`. + pub fn get_or_default( + &self, + section: &str, + name: &str, + ) -> Result { + self.get_or(section, name, Default::default) + } + /// Set a config item directly. `section`, `name` locates the config. `value` is the new value. /// `source` is some annotation about who set it, ex. "reporc", "userrc", "--config", etc. pub fn set( @@ -632,6 +664,7 @@ impl SupersetVerification { #[cfg(test)] pub(crate) mod tests { use super::*; + use crate::convert::ByteCount; use std::io::Write; use tempdir::TempDir; @@ -1333,4 +1366,73 @@ space_list=value1.a value1.b None, ); } + + #[test] + fn test_get_or() { + let mut cfg = ConfigSet::new(); + cfg.parse( + "[foo]\n\ + bool1 = yes\n\ + bool2 = unknown\n\ + bools = 1, TRUE, On, aLwAys, 0, false, oFF, never\n\ + int1 = -33\n\ + list1 = x y z\n\ + list3 = 2, 3, 1\n\ + byte1 = 1.5 KB\n\ + byte2 = 500\n\ + byte3 = 0.125M\n\ + float = 1.42\n\ + ", + &"test".into(), + ); + + assert_eq!(cfg.get_or("foo", "bar", || 3).unwrap(), 3); + assert_eq!(cfg.get_or("foo", "bool1", || false).unwrap(), true); + assert_eq!( + format!("{}", cfg.get_or("foo", "bool2", || true).unwrap_err()), + "invalid bool: unknown" + ); + assert_eq!(cfg.get_or("foo", "int1", || 42).unwrap(), -33); + assert_eq!( + cfg.get_or("foo", "list1", || vec!["x".to_string()]) + .unwrap(), + vec!["x", "y", "z"] + ); + assert_eq!( + cfg.get_or("foo", "list3", || vec![0]).unwrap(), + vec![2, 3, 1] + ); + + assert_eq!(cfg.get_or_default::("foo", "bool1").unwrap(), true); + assert_eq!( + cfg.get_or_default::>("foo", "bools").unwrap(), + vec![true, true, true, true, false, false, false, false] + ); + + assert_eq!( + cfg.get_or_default::("foo", "byte1") + .unwrap() + .value(), + 1536 + ); + assert_eq!( + cfg.get_or_default::("foo", "byte2") + .unwrap() + .value(), + 500 + ); + assert_eq!( + cfg.get_or_default::("foo", "byte3") + .unwrap() + .value(), + 131072 + ); + assert_eq!( + cfg.get_or("foo", "missing", || ByteCount::from(3)) + .unwrap() + .value(), + 3 + ); + assert_eq!(cfg.get_or("foo", "float", || 42f32).unwrap(), 1.42f32); + } } diff --git a/eden/scm/lib/configparser/src/convert.rs b/eden/scm/lib/configparser/src/convert.rs new file mode 100644 index 0000000000..42dfd18ad9 --- /dev/null +++ b/eden/scm/lib/configparser/src/convert.rs @@ -0,0 +1,507 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This software may be used and distributed according to the terms of the + * GNU General Public License version 2. + */ + +use std::path::PathBuf; + +use anyhow::Result; +use minibytes::Text; +use util::path::expand_path; + +use crate::error::Error; + +pub trait FromConfigValue: Sized { + fn try_from_str(s: &str) -> Result; +} + +impl FromConfigValue for bool { + fn try_from_str(s: &str) -> Result { + let value = s.to_lowercase(); + match value.as_ref() { + "1" | "yes" | "true" | "on" | "always" => Ok(true), + "0" | "no" | "false" | "off" | "never" => Ok(false), + _ => Err(Error::Convert(format!("invalid bool: {}", value)).into()), + } + } +} + +impl FromConfigValue for i8 { + fn try_from_str(s: &str) -> Result { + let value = s.parse()?; + Ok(value) + } +} + +impl FromConfigValue for i16 { + fn try_from_str(s: &str) -> Result { + let value = s.parse()?; + Ok(value) + } +} + +impl FromConfigValue for i32 { + fn try_from_str(s: &str) -> Result { + let value = s.parse()?; + Ok(value) + } +} + +impl FromConfigValue for i64 { + fn try_from_str(s: &str) -> Result { + let value = s.parse()?; + Ok(value) + } +} + +impl FromConfigValue for isize { + fn try_from_str(s: &str) -> Result { + let value = s.parse()?; + Ok(value) + } +} + +impl FromConfigValue for u8 { + fn try_from_str(s: &str) -> Result { + let value = s.parse()?; + Ok(value) + } +} + +impl FromConfigValue for u16 { + fn try_from_str(s: &str) -> Result { + let value = s.parse()?; + Ok(value) + } +} + +impl FromConfigValue for u32 { + fn try_from_str(s: &str) -> Result { + let value = s.parse()?; + Ok(value) + } +} + +impl FromConfigValue for u64 { + fn try_from_str(s: &str) -> Result { + let value = s.parse()?; + Ok(value) + } +} + +impl FromConfigValue for usize { + fn try_from_str(s: &str) -> Result { + let value = s.parse()?; + Ok(value) + } +} + +impl FromConfigValue for f32 { + fn try_from_str(s: &str) -> Result { + let value = s.parse()?; + Ok(value) + } +} + +impl FromConfigValue for f64 { + fn try_from_str(s: &str) -> Result { + let value = s.parse()?; + Ok(value) + } +} + +impl FromConfigValue for String { + fn try_from_str(s: &str) -> Result { + Ok(s.to_string()) + } +} + +/// Byte count specified with a unit. For example: `1.5 MB`. +#[derive(Copy, Clone, Default)] +pub struct ByteCount(u64); + +impl ByteCount { + /// Get the value of bytes. For example, `1K` has a value of `1024`. + pub fn value(self) -> u64 { + self.0 + } +} + +impl From for ByteCount { + fn from(value: u64) -> ByteCount { + ByteCount(value) + } +} + +impl FromConfigValue for ByteCount { + fn try_from_str(s: &str) -> Result { + // This implementation matches mercurial/util.py:sizetoint + let sizeunits = [ + ("kb", 1u64 << 10), + ("mb", 1 << 20), + ("gb", 1 << 30), + ("tb", 1 << 40), + ("k", 1 << 10), + ("m", 1 << 20), + ("g", 1 << 30), + ("t", 1 << 40), + ("b", 1), + ("", 1), + ]; + + let value = s.to_lowercase(); + for (suffix, unit) in sizeunits.iter() { + if value.ends_with(suffix) { + let number_str: &str = value[..value.len() - suffix.len()].trim(); + let number: f64 = number_str.parse()?; + if number < 0.0 { + return Err(Error::Convert(format!( + "byte size '{:?}' cannot be negative", + value + )) + .into()); + } + let unit = *unit as f64; + return Ok(ByteCount((number * unit) as u64)); + } + } + + Err(Error::Convert(format!("'{:?}' cannot be parsed as a byte size", value)).into()) + } +} + +impl FromConfigValue for PathBuf { + fn try_from_str(s: &str) -> Result { + Ok(expand_path(s)) + } +} + +impl FromConfigValue for Vec { + fn try_from_str(s: &str) -> Result { + let items = parse_list(s); + items.into_iter().map(|s| T::try_from_str(&s)).collect() + } +} + +impl FromConfigValue for Option { + fn try_from_str(s: &str) -> Result { + T::try_from_str(s).map(Option::Some) + } +} + +/// Parse a configuration value as a list of comma/space separated strings. +/// It is ported from `mercurial.config.parselist`. +/// +/// The function never complains about syntax and always returns some result. +/// +/// Example: +/// +/// ``` +/// use configparser::convert::parse_list; +/// +/// assert_eq!( +/// parse_list("this,is \"a small\" ,test"), +/// vec!["this".to_string(), "is".to_string(), "a small".to_string(), "test".to_string()] +/// ); +/// ``` +pub fn parse_list>(value: B) -> Vec { + let mut value = value.as_ref(); + + // ```python + // if value is not None and isinstance(value, bytes): + // result = _configlist(value.lstrip(' ,\n')) + // ``` + + while [" ", ",", "\n"].iter().any(|b| value.starts_with(b)) { + value = &value[1..] + } + + parse_list_internal(value) + .into_iter() + .map(Text::from) + .collect() +} + +fn parse_list_internal(value: &str) -> Vec { + let mut value = value; + + // ```python + // def _configlist(s): + // s = s.rstrip(' ,') + // if not s: + // return [] + // parser, parts, offset = _parse_plain, [''], 0 + // while parser: + // parser, parts, offset = parser(parts, s, offset) + // return parts + // ``` + + value = value.trim_end_matches(|c| " ,\n".contains(c)); + + if value.is_empty() { + return Vec::new(); + } + + #[derive(Copy, Clone)] + enum State { + Plain, + Quote, + } + + let mut offset = 0; + let mut parts: Vec = vec![String::new()]; + let mut state = State::Plain; + let value: Vec = value.chars().collect(); + + loop { + match state { + // ```python + // def _parse_plain(parts, s, offset): + // whitespace = False + // while offset < len(s) and (s[offset:offset + 1].isspace() + // or s[offset:offset + 1] == ','): + // whitespace = True + // offset += 1 + // if offset >= len(s): + // return None, parts, offset + // if whitespace: + // parts.append('') + // if s[offset:offset + 1] == '"' and not parts[-1]: + // return _parse_quote, parts, offset + 1 + // elif s[offset:offset + 1] == '"' and parts[-1][-1:] == '\\': + // parts[-1] = parts[-1][:-1] + s[offset:offset + 1] + // return _parse_plain, parts, offset + 1 + // parts[-1] += s[offset:offset + 1] + // return _parse_plain, parts, offset + 1 + // ``` + State::Plain => { + let mut whitespace = false; + while offset < value.len() && " \n\r\t,".contains(value[offset]) { + whitespace = true; + offset += 1; + } + if offset >= value.len() { + break; + } + if whitespace { + parts.push(Default::default()); + } + if value[offset] == '"' { + let branch = { + match parts.last() { + None => 1, + Some(last) => { + if last.is_empty() { + 1 + } else if last.ends_with('\\') { + 2 + } else { + 3 + } + } + } + }; // manual NLL, to drop reference on "parts". + if branch == 1 { + // last.is_empty() + state = State::Quote; + offset += 1; + continue; + } else if branch == 2 { + // last.ends_with(b"\\") + let last = parts.last_mut().unwrap(); + last.pop(); + last.push(value[offset]); + offset += 1; + continue; + } + } + let last = parts.last_mut().unwrap(); + last.push(value[offset]); + offset += 1; + } + + // ```python + // def _parse_quote(parts, s, offset): + // if offset < len(s) and s[offset:offset + 1] == '"': # "" + // parts.append('') + // offset += 1 + // while offset < len(s) and (s[offset:offset + 1].isspace() or + // s[offset:offset + 1] == ','): + // offset += 1 + // return _parse_plain, parts, offset + // while offset < len(s) and s[offset:offset + 1] != '"': + // if (s[offset:offset + 1] == '\\' and offset + 1 < len(s) + // and s[offset + 1:offset + 2] == '"'): + // offset += 1 + // parts[-1] += '"' + // else: + // parts[-1] += s[offset:offset + 1] + // offset += 1 + // if offset >= len(s): + // real_parts = _configlist(parts[-1]) + // if not real_parts: + // parts[-1] = '"' + // else: + // real_parts[0] = '"' + real_parts[0] + // parts = parts[:-1] + // parts.extend(real_parts) + // return None, parts, offset + // offset += 1 + // while offset < len(s) and s[offset:offset + 1] in [' ', ',']: + // offset += 1 + // if offset < len(s): + // if offset + 1 == len(s) and s[offset:offset + 1] == '"': + // parts[-1] += '"' + // offset += 1 + // else: + // parts.append('') + // else: + // return None, parts, offset + // return _parse_plain, parts, offset + // ``` + State::Quote => { + if offset < value.len() && value[offset] == '"' { + parts.push(Default::default()); + offset += 1; + while offset < value.len() && " \n\r\t,".contains(value[offset]) { + offset += 1; + } + state = State::Plain; + continue; + } + while offset < value.len() && value[offset] != '"' { + if value[offset] == '\\' && offset + 1 < value.len() && value[offset + 1] == '"' + { + offset += 1; + parts.last_mut().unwrap().push('"'); + } else { + parts.last_mut().unwrap().push(value[offset]); + } + offset += 1; + } + if offset >= value.len() { + let mut real_parts: Vec = parse_list_internal(parts.last().unwrap()); + if real_parts.is_empty() { + parts.pop(); + parts.push("\"".to_string()); + } else { + real_parts[0].insert(0, '"'); + parts.pop(); + parts.append(&mut real_parts); + } + break; + } + offset += 1; + while offset < value.len() && " ,".contains(value[offset]) { + offset += 1; + } + if offset < value.len() { + if offset + 1 == value.len() && value[offset] == '"' { + parts.last_mut().unwrap().push('"'); + offset += 1; + } else { + parts.push(Default::default()); + } + } else { + break; + } + state = State::Plain; + } + } + } + + parts +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_list() { + fn b>(bytes: B) -> Text { + Text::copy_from_slice(bytes.as_ref()) + } + + // From test-ui-config.py + assert_eq!(parse_list("foo"), vec![b("foo")]); + assert_eq!( + parse_list("foo bar baz"), + vec![b("foo"), b("bar"), b("baz")] + ); + assert_eq!(parse_list("alice, bob"), vec![b("alice"), b("bob")]); + assert_eq!( + parse_list("foo bar baz alice, bob"), + vec![b("foo"), b("bar"), b("baz"), b("alice"), b("bob")] + ); + assert_eq!( + parse_list("abc d\"ef\"g \"hij def\""), + vec![b("abc"), b("d\"ef\"g"), b("hij def")] + ); + assert_eq!( + parse_list("\"hello world\", \"how are you?\""), + vec![b("hello world"), b("how are you?")] + ); + assert_eq!( + parse_list("Do\"Not\"Separate"), + vec![b("Do\"Not\"Separate")] + ); + assert_eq!(parse_list("\"Do\"Separate"), vec![b("Do"), b("Separate")]); + assert_eq!( + parse_list("\"Do\\\"NotSeparate\""), + vec![b("Do\"NotSeparate")] + ); + assert_eq!( + parse_list("string \"with extraneous\" quotation mark\""), + vec![ + b("string"), + b("with extraneous"), + b("quotation"), + b("mark\""), + ] + ); + assert_eq!(parse_list("x, y"), vec![b("x"), b("y")]); + assert_eq!(parse_list("\"x\", \"y\""), vec![b("x"), b("y")]); + assert_eq!( + parse_list("\"\"\" key = \"x\", \"y\" \"\"\""), + vec![b(""), b(" key = "), b("x\""), b("y"), b(""), b("\"")] + ); + assert_eq!(parse_list(",,,, "), Vec::::new()); + assert_eq!( + parse_list("\" just with starting quotation"), + vec![b("\""), b("just"), b("with"), b("starting"), b("quotation")] + ); + assert_eq!( + parse_list("\"longer quotation\" with \"no ending quotation"), + vec![ + b("longer quotation"), + b("with"), + b("\"no"), + b("ending"), + b("quotation"), + ] + ); + assert_eq!( + parse_list("this is \\\" \"not a quotation mark\""), + vec![b("this"), b("is"), b("\""), b("not a quotation mark")] + ); + assert_eq!(parse_list("\n \n\nding\ndong"), vec![b("ding"), b("dong")]); + + // Other manually written cases + assert_eq!(parse_list("a,b,,c"), vec![b("a"), b("b"), b("c")]); + assert_eq!(parse_list("a b c"), vec![b("a"), b("b"), b("c")]); + assert_eq!( + parse_list(" , a , , b, , c , "), + vec![b("a"), b("b"), b("c")] + ); + assert_eq!(parse_list("a,\"b,c\" d"), vec![b("a"), b("b,c"), b("d")]); + assert_eq!(parse_list("a,\",c"), vec![b("a"), b("\""), b("c")]); + assert_eq!(parse_list("a,\" c\" \""), vec![b("a"), b(" c\"")]); + assert_eq!( + parse_list("a,\" c\" \" d"), + vec![b("a"), b(" c"), b("\""), b("d")] + ); + } +} diff --git a/eden/scm/lib/configparser/src/hg.rs b/eden/scm/lib/configparser/src/hg.rs index 618f260d81..6d9f12f8c8 100644 --- a/eden/scm/lib/configparser/src/hg.rs +++ b/eden/scm/lib/configparser/src/hg.rs @@ -77,32 +77,6 @@ pub trait ConfigSetHgExt { /// Load a specified config file. Respect HGPLAIN environment variables. /// Return errors parsing files. fn load_hgrc(&mut self, path: impl AsRef, source: &'static str) -> Vec; - - /// Get a config item. Convert to type `T`. - fn get_opt(&self, section: &str, name: &str) -> Result>; - - /// Get a config item. Convert to type `T`. - /// - /// If the config item is not set, calculate it using `default_func`. - fn get_or( - &self, - section: &str, - name: &str, - default_func: impl Fn() -> T, - ) -> Result { - Ok(self.get_opt(section, name)?.unwrap_or_else(default_func)) - } - - /// Get a config item. Convert to type `T`. - /// - /// If the config item is not set, return `T::default()`. - fn get_or_default(&self, section: &str, name: &str) -> Result { - self.get_or(section, name, Default::default) - } -} - -pub trait FromConfigValue: Sized { - fn try_from_str(s: &str) -> Result; } pub fn load, N: Into>( @@ -445,12 +419,6 @@ impl ConfigSetHgExt for ConfigSet { let opts = Options::new().source(source).process_hgplain(); self.load_path(path, &opts) } - - fn get_opt(&self, section: &str, name: &str) -> Result> { - ConfigSet::get(self, section, name) - .map(|bytes| T::try_from_str(&bytes)) - .transpose() - } } impl ConfigSet { @@ -514,404 +482,6 @@ impl ConfigSet { } } -impl FromConfigValue for bool { - fn try_from_str(s: &str) -> Result { - let value = s.to_lowercase(); - match value.as_ref() { - "1" | "yes" | "true" | "on" | "always" => Ok(true), - "0" | "no" | "false" | "off" | "never" => Ok(false), - _ => Err(Error::Convert(format!("invalid bool: {}", value)).into()), - } - } -} - -impl FromConfigValue for i8 { - fn try_from_str(s: &str) -> Result { - let value = s.parse()?; - Ok(value) - } -} - -impl FromConfigValue for i16 { - fn try_from_str(s: &str) -> Result { - let value = s.parse()?; - Ok(value) - } -} - -impl FromConfigValue for i32 { - fn try_from_str(s: &str) -> Result { - let value = s.parse()?; - Ok(value) - } -} - -impl FromConfigValue for i64 { - fn try_from_str(s: &str) -> Result { - let value = s.parse()?; - Ok(value) - } -} - -impl FromConfigValue for isize { - fn try_from_str(s: &str) -> Result { - let value = s.parse()?; - Ok(value) - } -} - -impl FromConfigValue for u8 { - fn try_from_str(s: &str) -> Result { - let value = s.parse()?; - Ok(value) - } -} - -impl FromConfigValue for u16 { - fn try_from_str(s: &str) -> Result { - let value = s.parse()?; - Ok(value) - } -} - -impl FromConfigValue for u32 { - fn try_from_str(s: &str) -> Result { - let value = s.parse()?; - Ok(value) - } -} - -impl FromConfigValue for u64 { - fn try_from_str(s: &str) -> Result { - let value = s.parse()?; - Ok(value) - } -} - -impl FromConfigValue for usize { - fn try_from_str(s: &str) -> Result { - let value = s.parse()?; - Ok(value) - } -} - -impl FromConfigValue for f32 { - fn try_from_str(s: &str) -> Result { - let value = s.parse()?; - Ok(value) - } -} - -impl FromConfigValue for f64 { - fn try_from_str(s: &str) -> Result { - let value = s.parse()?; - Ok(value) - } -} - -impl FromConfigValue for String { - fn try_from_str(s: &str) -> Result { - Ok(s.to_string()) - } -} - -/// Byte count specified with a unit. For example: `1.5 MB`. -#[derive(Copy, Clone, Default)] -pub struct ByteCount(u64); - -impl ByteCount { - /// Get the value of bytes. For example, `1K` has a value of `1024`. - pub fn value(self) -> u64 { - self.0 - } -} - -impl From for ByteCount { - fn from(value: u64) -> ByteCount { - ByteCount(value) - } -} - -impl FromConfigValue for ByteCount { - fn try_from_str(s: &str) -> Result { - // This implementation matches mercurial/util.py:sizetoint - let sizeunits = [ - ("kb", 1u64 << 10), - ("mb", 1 << 20), - ("gb", 1 << 30), - ("tb", 1 << 40), - ("k", 1 << 10), - ("m", 1 << 20), - ("g", 1 << 30), - ("t", 1 << 40), - ("b", 1), - ("", 1), - ]; - - let value = s.to_lowercase(); - for (suffix, unit) in sizeunits.iter() { - if value.ends_with(suffix) { - let number_str: &str = value[..value.len() - suffix.len()].trim(); - let number: f64 = number_str.parse()?; - if number < 0.0 { - return Err(Error::Convert(format!( - "byte size '{:?}' cannot be negative", - value - )) - .into()); - } - let unit = *unit as f64; - return Ok(ByteCount((number * unit) as u64)); - } - } - - Err(Error::Convert(format!("'{:?}' cannot be parsed as a byte size", value)).into()) - } -} - -impl FromConfigValue for PathBuf { - fn try_from_str(s: &str) -> Result { - Ok(expand_path(s)) - } -} - -impl FromConfigValue for Vec { - fn try_from_str(s: &str) -> Result { - let items = parse_list(s); - items.into_iter().map(|s| T::try_from_str(&s)).collect() - } -} - -impl FromConfigValue for Option { - fn try_from_str(s: &str) -> Result { - T::try_from_str(s).map(Option::Some) - } -} - -/// Parse a configuration value as a list of comma/space separated strings. -/// It is ported from `mercurial.config.parselist`. -/// -/// The function never complains about syntax and always returns some result. -/// -/// Example: -/// -/// ``` -/// use configparser::hg::parse_list; -/// -/// assert_eq!( -/// parse_list("this,is \"a small\" ,test"), -/// vec!["this".to_string(), "is".to_string(), "a small".to_string(), "test".to_string()] -/// ); -/// ``` -pub fn parse_list>(value: B) -> Vec { - let mut value = value.as_ref(); - - // ```python - // if value is not None and isinstance(value, bytes): - // result = _configlist(value.lstrip(' ,\n')) - // ``` - - while [" ", ",", "\n"].iter().any(|b| value.starts_with(b)) { - value = &value[1..] - } - - parse_list_internal(value) - .into_iter() - .map(Text::from) - .collect() -} - -fn parse_list_internal(value: &str) -> Vec { - let mut value = value; - - // ```python - // def _configlist(s): - // s = s.rstrip(' ,') - // if not s: - // return [] - // parser, parts, offset = _parse_plain, [''], 0 - // while parser: - // parser, parts, offset = parser(parts, s, offset) - // return parts - // ``` - - value = value.trim_end_matches(|c| " ,\n".contains(c)); - - if value.is_empty() { - return Vec::new(); - } - - #[derive(Copy, Clone)] - enum State { - Plain, - Quote, - } - - let mut offset = 0; - let mut parts: Vec = vec![String::new()]; - let mut state = State::Plain; - let value: Vec = value.chars().collect(); - - loop { - match state { - // ```python - // def _parse_plain(parts, s, offset): - // whitespace = False - // while offset < len(s) and (s[offset:offset + 1].isspace() - // or s[offset:offset + 1] == ','): - // whitespace = True - // offset += 1 - // if offset >= len(s): - // return None, parts, offset - // if whitespace: - // parts.append('') - // if s[offset:offset + 1] == '"' and not parts[-1]: - // return _parse_quote, parts, offset + 1 - // elif s[offset:offset + 1] == '"' and parts[-1][-1:] == '\\': - // parts[-1] = parts[-1][:-1] + s[offset:offset + 1] - // return _parse_plain, parts, offset + 1 - // parts[-1] += s[offset:offset + 1] - // return _parse_plain, parts, offset + 1 - // ``` - State::Plain => { - let mut whitespace = false; - while offset < value.len() && " \n\r\t,".contains(value[offset]) { - whitespace = true; - offset += 1; - } - if offset >= value.len() { - break; - } - if whitespace { - parts.push(Default::default()); - } - if value[offset] == '"' { - let branch = { - match parts.last() { - None => 1, - Some(last) => { - if last.is_empty() { - 1 - } else if last.ends_with('\\') { - 2 - } else { - 3 - } - } - } - }; // manual NLL, to drop reference on "parts". - if branch == 1 { - // last.is_empty() - state = State::Quote; - offset += 1; - continue; - } else if branch == 2 { - // last.ends_with(b"\\") - let last = parts.last_mut().unwrap(); - last.pop(); - last.push(value[offset]); - offset += 1; - continue; - } - } - let last = parts.last_mut().unwrap(); - last.push(value[offset]); - offset += 1; - } - - // ```python - // def _parse_quote(parts, s, offset): - // if offset < len(s) and s[offset:offset + 1] == '"': # "" - // parts.append('') - // offset += 1 - // while offset < len(s) and (s[offset:offset + 1].isspace() or - // s[offset:offset + 1] == ','): - // offset += 1 - // return _parse_plain, parts, offset - // while offset < len(s) and s[offset:offset + 1] != '"': - // if (s[offset:offset + 1] == '\\' and offset + 1 < len(s) - // and s[offset + 1:offset + 2] == '"'): - // offset += 1 - // parts[-1] += '"' - // else: - // parts[-1] += s[offset:offset + 1] - // offset += 1 - // if offset >= len(s): - // real_parts = _configlist(parts[-1]) - // if not real_parts: - // parts[-1] = '"' - // else: - // real_parts[0] = '"' + real_parts[0] - // parts = parts[:-1] - // parts.extend(real_parts) - // return None, parts, offset - // offset += 1 - // while offset < len(s) and s[offset:offset + 1] in [' ', ',']: - // offset += 1 - // if offset < len(s): - // if offset + 1 == len(s) and s[offset:offset + 1] == '"': - // parts[-1] += '"' - // offset += 1 - // else: - // parts.append('') - // else: - // return None, parts, offset - // return _parse_plain, parts, offset - // ``` - State::Quote => { - if offset < value.len() && value[offset] == '"' { - parts.push(Default::default()); - offset += 1; - while offset < value.len() && " \n\r\t,".contains(value[offset]) { - offset += 1; - } - state = State::Plain; - continue; - } - while offset < value.len() && value[offset] != '"' { - if value[offset] == '\\' && offset + 1 < value.len() && value[offset + 1] == '"' - { - offset += 1; - parts.last_mut().unwrap().push('"'); - } else { - parts.last_mut().unwrap().push(value[offset]); - } - offset += 1; - } - if offset >= value.len() { - let mut real_parts: Vec = parse_list_internal(parts.last().unwrap()); - if real_parts.is_empty() { - parts.pop(); - parts.push("\"".to_string()); - } else { - real_parts[0].insert(0, '"'); - parts.pop(); - parts.append(&mut real_parts); - } - break; - } - offset += 1; - while offset < value.len() && " ,".contains(value[offset]) { - offset += 1; - } - if offset < value.len() { - if offset + 1 == value.len() && value[offset] == '"' { - parts.last_mut().unwrap().push('"'); - offset += 1; - } else { - parts.push(Default::default()); - } - } else { - break; - } - state = State::Plain; - } - } - } - - parts -} - fn get_shared_path(repo_path: &Path) -> Result { let shared_path = repo_path.join("sharedpath"); Ok(if shared_path.exists() { @@ -1187,161 +757,6 @@ mod tests { assert_eq!(cfg.get("y", "b"), None); assert_eq!(cfg.get("z", "c"), Some("3".into())); } - - #[test] - fn test_parse_list() { - fn b>(bytes: B) -> Text { - Text::copy_from_slice(bytes.as_ref()) - } - - // From test-ui-config.py - assert_eq!(parse_list("foo"), vec![b("foo")]); - assert_eq!( - parse_list("foo bar baz"), - vec![b("foo"), b("bar"), b("baz")] - ); - assert_eq!(parse_list("alice, bob"), vec![b("alice"), b("bob")]); - assert_eq!( - parse_list("foo bar baz alice, bob"), - vec![b("foo"), b("bar"), b("baz"), b("alice"), b("bob")] - ); - assert_eq!( - parse_list("abc d\"ef\"g \"hij def\""), - vec![b("abc"), b("d\"ef\"g"), b("hij def")] - ); - assert_eq!( - parse_list("\"hello world\", \"how are you?\""), - vec![b("hello world"), b("how are you?")] - ); - assert_eq!( - parse_list("Do\"Not\"Separate"), - vec![b("Do\"Not\"Separate")] - ); - assert_eq!(parse_list("\"Do\"Separate"), vec![b("Do"), b("Separate")]); - assert_eq!( - parse_list("\"Do\\\"NotSeparate\""), - vec![b("Do\"NotSeparate")] - ); - assert_eq!( - parse_list("string \"with extraneous\" quotation mark\""), - vec![ - b("string"), - b("with extraneous"), - b("quotation"), - b("mark\""), - ] - ); - assert_eq!(parse_list("x, y"), vec![b("x"), b("y")]); - assert_eq!(parse_list("\"x\", \"y\""), vec![b("x"), b("y")]); - assert_eq!( - parse_list("\"\"\" key = \"x\", \"y\" \"\"\""), - vec![b(""), b(" key = "), b("x\""), b("y"), b(""), b("\"")] - ); - assert_eq!(parse_list(",,,, "), Vec::::new()); - assert_eq!( - parse_list("\" just with starting quotation"), - vec![b("\""), b("just"), b("with"), b("starting"), b("quotation")] - ); - assert_eq!( - parse_list("\"longer quotation\" with \"no ending quotation"), - vec![ - b("longer quotation"), - b("with"), - b("\"no"), - b("ending"), - b("quotation"), - ] - ); - assert_eq!( - parse_list("this is \\\" \"not a quotation mark\""), - vec![b("this"), b("is"), b("\""), b("not a quotation mark")] - ); - assert_eq!(parse_list("\n \n\nding\ndong"), vec![b("ding"), b("dong")]); - - // Other manually written cases - assert_eq!(parse_list("a,b,,c"), vec![b("a"), b("b"), b("c")]); - assert_eq!(parse_list("a b c"), vec![b("a"), b("b"), b("c")]); - assert_eq!( - parse_list(" , a , , b, , c , "), - vec![b("a"), b("b"), b("c")] - ); - assert_eq!(parse_list("a,\"b,c\" d"), vec![b("a"), b("b,c"), b("d")]); - assert_eq!(parse_list("a,\",c"), vec![b("a"), b("\""), b("c")]); - assert_eq!(parse_list("a,\" c\" \""), vec![b("a"), b(" c\"")]); - assert_eq!( - parse_list("a,\" c\" \" d"), - vec![b("a"), b(" c"), b("\""), b("d")] - ); - } - - #[test] - fn test_get_or() { - let mut cfg = ConfigSet::new(); - cfg.parse( - "[foo]\n\ - bool1 = yes\n\ - bool2 = unknown\n\ - bools = 1, TRUE, On, aLwAys, 0, false, oFF, never\n\ - int1 = -33\n\ - list1 = x y z\n\ - list3 = 2, 3, 1\n\ - byte1 = 1.5 KB\n\ - byte2 = 500\n\ - byte3 = 0.125M\n\ - float = 1.42\n\ - ", - &"test".into(), - ); - - assert_eq!(cfg.get_or("foo", "bar", || 3).unwrap(), 3); - assert_eq!(cfg.get_or("foo", "bool1", || false).unwrap(), true); - assert_eq!( - format!("{}", cfg.get_or("foo", "bool2", || true).unwrap_err()), - "invalid bool: unknown" - ); - assert_eq!(cfg.get_or("foo", "int1", || 42).unwrap(), -33); - assert_eq!( - cfg.get_or("foo", "list1", || vec!["x".to_string()]) - .unwrap(), - vec!["x", "y", "z"] - ); - assert_eq!( - cfg.get_or("foo", "list3", || vec![0]).unwrap(), - vec![2, 3, 1] - ); - - assert_eq!(cfg.get_or_default::("foo", "bool1").unwrap(), true); - assert_eq!( - cfg.get_or_default::>("foo", "bools").unwrap(), - vec![true, true, true, true, false, false, false, false] - ); - - assert_eq!( - cfg.get_or_default::("foo", "byte1") - .unwrap() - .value(), - 1536 - ); - assert_eq!( - cfg.get_or_default::("foo", "byte2") - .unwrap() - .value(), - 500 - ); - assert_eq!( - cfg.get_or_default::("foo", "byte3") - .unwrap() - .value(), - 131072 - ); - assert_eq!( - cfg.get_or("foo", "missing", || ByteCount::from(3)) - .unwrap() - .value(), - 3 - ); - assert_eq!(cfg.get_or("foo", "float", || 42f32).unwrap(), 1.42f32); - } } const MERGE_TOOLS_CONFIG: &str = r#"# Some default global settings for common merge tools diff --git a/eden/scm/lib/configparser/src/lib.rs b/eden/scm/lib/configparser/src/lib.rs index 5630690308..cc8f73ac73 100644 --- a/eden/scm/lib/configparser/src/lib.rs +++ b/eden/scm/lib/configparser/src/lib.rs @@ -68,6 +68,7 @@ pub mod c_api; pub mod config; +pub mod convert; pub mod dynamicconfig; pub mod error; pub mod hg; diff --git a/eden/scm/lib/edenapi/src/builder.rs b/eden/scm/lib/edenapi/src/builder.rs index c2fce32114..1b17d08054 100644 --- a/eden/scm/lib/edenapi/src/builder.rs +++ b/eden/scm/lib/edenapi/src/builder.rs @@ -15,7 +15,7 @@ use url::Url; use anyhow::anyhow; use auth::AuthConfig; -use configparser::{config::ConfigSet, hg::ConfigSetHgExt}; +use configparser::config::ConfigSet; use http_client::HttpVersion; use crate::client::Client; diff --git a/eden/scm/lib/revisionstore/src/contentstore.rs b/eden/scm/lib/revisionstore/src/contentstore.rs index 75f99bc0cf..1ae6c71139 100644 --- a/eden/scm/lib/revisionstore/src/contentstore.rs +++ b/eden/scm/lib/revisionstore/src/contentstore.rs @@ -17,10 +17,7 @@ use minibytes::Bytes; use regex::Regex; use tracing::info_span; -use configparser::{ - config::ConfigSet, - hg::{ByteCount, ConfigSetHgExt}, -}; +use configparser::{config::ConfigSet, convert::ByteCount}; use hgtime::HgTime; use types::{Key, RepoPathBuf}; diff --git a/eden/scm/lib/revisionstore/src/indexedlogdatastore.rs b/eden/scm/lib/revisionstore/src/indexedlogdatastore.rs index 13a1fe1f02..8bdec01c3c 100644 --- a/eden/scm/lib/revisionstore/src/indexedlogdatastore.rs +++ b/eden/scm/lib/revisionstore/src/indexedlogdatastore.rs @@ -20,10 +20,7 @@ use minibytes::Bytes; use parking_lot::RwLock; use tokio::task::spawn_blocking; -use configparser::{ - config::ConfigSet, - hg::{ByteCount, ConfigSetHgExt}, -}; +use configparser::{config::ConfigSet, convert::ByteCount}; use edenapi_types::TreeEntry; use indexedlog::log::IndexOutput; use lz4_pyframe::{compress, decompress}; diff --git a/eden/scm/lib/revisionstore/src/indexedloghistorystore.rs b/eden/scm/lib/revisionstore/src/indexedloghistorystore.rs index 18f039e35d..b6fa5adff9 100644 --- a/eden/scm/lib/revisionstore/src/indexedloghistorystore.rs +++ b/eden/scm/lib/revisionstore/src/indexedloghistorystore.rs @@ -15,10 +15,7 @@ use anyhow::Result; use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt}; use sha1::{Digest, Sha1}; -use configparser::{ - config::ConfigSet, - hg::{ByteCount, ConfigSetHgExt}, -}; +use configparser::{config::ConfigSet, convert::ByteCount}; use indexedlog::log::IndexOutput; use types::{ hgid::{ReadHgIdExt, WriteHgIdExt}, diff --git a/eden/scm/lib/revisionstore/src/lfs.rs b/eden/scm/lib/revisionstore/src/lfs.rs index 27a66c31a1..e2e29a1a62 100644 --- a/eden/scm/lib/revisionstore/src/lfs.rs +++ b/eden/scm/lib/revisionstore/src/lfs.rs @@ -39,10 +39,7 @@ use url::Url; use async_runtime::block_on_exclusive as block_on_future; use auth::{Auth, AuthConfig}; -use configparser::{ - config::ConfigSet, - hg::{ByteCount, ConfigSetHgExt}, -}; +use configparser::{config::ConfigSet, convert::ByteCount}; use hg_http::http_client; use http_client::{HttpClient, HttpClientError, HttpVersion, Method, MinTransferSpeed, Request}; use indexedlog::{log::IndexOutput, rotate, DefaultOpenOptions, Repair}; diff --git a/eden/scm/lib/revisionstore/src/metadatastore.rs b/eden/scm/lib/revisionstore/src/metadatastore.rs index 347ea34d9b..ea5a87b7e1 100644 --- a/eden/scm/lib/revisionstore/src/metadatastore.rs +++ b/eden/scm/lib/revisionstore/src/metadatastore.rs @@ -12,10 +12,7 @@ use std::{ use anyhow::{format_err, Result}; -use configparser::{ - config::ConfigSet, - hg::{ByteCount, ConfigSetHgExt}, -}; +use configparser::{config::ConfigSet, convert::ByteCount}; use types::{Key, NodeInfo}; use crate::{ diff --git a/eden/scm/lib/revisionstore/src/repack.rs b/eden/scm/lib/revisionstore/src/repack.rs index 198c1286df..29ca2ec4c1 100644 --- a/eden/scm/lib/revisionstore/src/repack.rs +++ b/eden/scm/lib/revisionstore/src/repack.rs @@ -14,7 +14,7 @@ use std::{ }; use anyhow::{format_err, Error, Result}; -use configparser::{config::ConfigSet, hg::ByteCount, hg::ConfigSetHgExt}; +use configparser::{config::ConfigSet, convert::ByteCount}; use minibytes::Bytes; use thiserror::Error; diff --git a/eden/scm/lib/revisionstore/src/util.rs b/eden/scm/lib/revisionstore/src/util.rs index 1fe38940fa..48637741a2 100644 --- a/eden/scm/lib/revisionstore/src/util.rs +++ b/eden/scm/lib/revisionstore/src/util.rs @@ -13,7 +13,7 @@ use anyhow::Result; use hgtime::HgTime; use thiserror::Error; -use configparser::{config::ConfigSet, hg::ConfigSetHgExt}; +use configparser::config::ConfigSet; use util::path::{create_dir, create_shared_dir}; #[derive(Error, Debug)]