Merge pull request #62 from numtide/include-exclude

yet another refactor
This commit is contained in:
Jonas Chevalier 2021-02-18 14:19:48 +00:00 committed by GitHub
commit f53c70d5ac
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 275 additions and 326 deletions

View File

@ -33,48 +33,61 @@ check for code changes.
`treefmt` is responsible for traversing the file-system and mapping files to
specific code formatters.
Only *one* formatter per file. `treefmt` enforces that only one tool is
Only _one_ formatter per file. `treefmt` enforces that only one tool is
executed per file. Guaranteeing two tools to product idempotent outputs is
quite difficult.
## Usage
`$ cargo run -- --help`
```
treefmt [options] [<file>...]
treefmt 0.1.0
The various kinds of commands that `treefmt` can execute
USAGE:
treefmt [FLAGS] [OPTIONS] [SUBCOMMAND]
FLAGS:
-h, --help Prints help information
-q, --quiet No output printed to stdout
-V, --version Prints version information
-v, --verbose Log verbosity is based off the number of v used
OPTIONS:
-C, --config <config> Specify where to look for the treefmt.toml file
--log-level <log-level> The maximum level of messages that should be logged by treefmt. [possible values:
info, warn, error] [default: debug]
SUBCOMMANDS:
--init Init a new project with a default config
help Prints this message or the help of the given subcommand(s)
```
* `file`: path to files to format. If no files are passed, format all of the
files from the current folder and down.
### Options
* `--init`: Creates a templated `treefmt.toml` in the current directory.
* `--config <path>`: Overrides the `treefmt.toml` file lookup.
* `--help`: Shows this help.
## Configuration format
`treefmt` depends on the `treefmt.toml` to map file extensions to actual code
formatters. That file is searched for recursively from the current folder and
up unless the `--config <path>` option is passed.
### `[formatters.<name>]`
### `[formatter.<name>]`
This section describes the integration between a single formatter and
`treefmt`.
* `files`: A list of glob patterns used to select files. Usually this would be
something like `[ "*.sh" ]` to select all the shell scripts. Sometimes,
full filenames can be passed. Eg: `[ "Makefile" ]`.
- `command`: A list of arguments to execute the formatter. This will be
composed with the `options` attribute during invocation. The first argument
is the name of the executable to run.
* `command`: A list of arguments to execute the formatter. This will be
composed with the `options` attribute during invocation. The first argument
is the name of the executable to run.
- `options`: A list of extra arguments to add to the command. This is typically
project-specific arguments.
* `options`: A list of extra arguments to add to the command. This is typically
project-specific arguments.
- `includes`: A list of glob patterns used to select files. Usually this would be
something like `[ "*.sh" ]` to select all the shell scripts. Sometimes,
full filenames can be passed. Eg: `[ "Makefile" ]`.
- `excludes`: A list of glob patterns to deny. If any of these patterns match,
the file will be excluded.
## Use cases
@ -91,8 +104,7 @@ TODO: not supported yet.
Editors often want to be able to format a file, before it gets written to disk.
Ideally, the editor would pipe the code in, pass the filename, and get the
formatted code out. Eg: `cat ./my_file.sh | treefmt --stdin my_file.sh >
formatted_file.sh`
formatted code out. Eg: `cat ./my_file.sh | treefmt --stdin my_file.sh > formatted_file.sh`
### CI integration
@ -128,10 +140,10 @@ usage to match that spec.
## Related projects
* [EditorConfig](https://editorconfig.org/): unifies file indentations
- [EditorConfig](https://editorconfig.org/): unifies file indentations
configuration on a per-project basis.
* [prettier](https://prettier.io/): and opinionated code formatter for a
number of languages.
- [prettier](https://prettier.io/): and opinionated code formatter for a
number of languages.
## Contributing

View File

@ -1,51 +0,0 @@
# One CLI to format the code tree - https://github.com/numtide/treefmt
[formatters.black]
files = [ "*.py" ]
command = "black"
options = [ ]
[formatters.elm-format]
files = [ "*.elm" ]
command = "elm-format"
options = [ "--yes" ]
[formatters.gofmt]
files = [ "*.go" ]
command = "gofmt"
options = [ "-w" ]
[formatters.ormolu]
files = "*.hs"
excludes = [ "haskell/" ]
command = "ormolu"
options = [
"--ghc-opt", "-XBangPatterns",
"--ghc-opt", "-XPatternSynonyms",
"--ghc-opt", "-XTypeApplications",
"--mode", "inplace",
"--check-idempotence"
]
[formatters.prettier]
files = [ "*.js" ]
command = "prettier"
options = [ "--write" ]
[formatters.rustfmt]
files = [ "*.rs" ]
includes = [ "rust/" ]
excludes = []
command = "rustfmt"
options = [ "--edition", "2018" ]
[formatters.shfmt]
files = [ "*.sh" ]
includes = [ "shell/" ]
excludes = []
command = "shfmt"
options = [
"-i", "2", # indent 2
"-s", # simplify the code
"-w", # write back to the file
]

View File

@ -1,4 +1,4 @@
use super::lookup_treefmt_toml;
use crate::config;
use crate::engine::run_treefmt;
use crate::CLOG;
use anyhow::anyhow;
@ -7,10 +7,15 @@ use std::path::Path;
use std::{env, path::PathBuf};
pub fn format_cmd(path: Option<PathBuf>) -> anyhow::Result<()> {
let cwd = env::current_dir()?;
let cfg_dir = match path {
Some(p) => p,
None => lookup_treefmt_toml(cwd)?,
None => {
let cwd = env::current_dir()?;
match config::lookup_dir(&cwd) {
Some(p) => p,
None => return Err(anyhow!("treefmt.toml could not be found in {} and up. Use the --init option to create one.", cwd.display()))
}
}
};
let treefmt_toml = cfg_dir.join("treefmt.toml");

View File

@ -15,11 +15,15 @@ pub fn init_cmd(path: Option<PathBuf>) -> anyhow::Result<()> {
&file_path,
r#"# One CLI to format the code tree - https://github.com/numtide/treefmt
[formatter.<Language>]
includes = [ "*.<language-extension>" ]
excludes = []
command = ""
[formatter.mylanguage]
# Formatter to run
command = "command-to-run"
# Command-line arguments for the command
options = []
# Glob pattern of files to include
includes = [ "*.<language-extension>" ]
# Glob patterns of files to exclude
excludes = []
"#,
)
.with_context(|| {

View File

@ -7,7 +7,6 @@ mod init;
use self::format::format_cmd;
use self::init::init_cmd;
use super::customlog::LogLevel;
use anyhow::{anyhow, Result};
use std::path::PathBuf;
use structopt::StructOpt;
@ -15,17 +14,8 @@ use structopt::StructOpt;
/// The various kinds of commands that `treefmt` can execute.
pub enum Command {
#[structopt(name = "--init")]
/// init a new project with a default config
Init {
/// path to file or folder
path: Option<PathBuf>,
},
#[structopt(name = "--config")]
/// Specify treefmt.toml file
PrjFmt {
/// path to file or folder
path: PathBuf,
},
/// Init a new project with a default config
Init {},
}
/// ✨ format all your language!
@ -46,33 +36,18 @@ pub struct Cli {
#[structopt(long = "log-level", default_value = "debug")]
/// The maximum level of messages that should be logged by treefmt. [possible values: info, warn, error]
pub log_level: LogLevel,
#[structopt(long = "config", short = "C")]
/// Specify where to look for the treefmt.toml file
pub config: Option<PathBuf>,
}
/// Run a command with the given logger
pub fn run_cli(cli: Cli) -> anyhow::Result<()> {
match cli.cmd {
Some(Command::Init { path }) => init_cmd(path)?,
Some(Command::PrjFmt { path }) => format_cmd(Some(path))?,
None => format_cmd(None)?,
Some(Command::Init {}) => init_cmd(cli.config)?,
None => format_cmd(cli.config)?,
}
Ok(())
}
/// Look up treefmt toml from current directory up into project's root
pub fn lookup_treefmt_toml(path: PathBuf) -> Result<PathBuf> {
let mut work = path;
loop {
if work.join("treefmt.toml").exists() {
return Ok(work);
}
let prev = work.clone();
work = match work.parent() {
Some(x) => x.to_path_buf(),
None => return Err(anyhow!("You already reached root directory")),
};
if prev == work {
return Err(anyhow!("treefmt.toml could not be found"));
}
}
}

55
src/config.rs Normal file
View File

@ -0,0 +1,55 @@
//! Contains the project configuration schema and parsing
use anyhow::Result;
use serde::Deserialize;
use std::collections::BTreeMap;
use std::fs::read_to_string;
use std::path::PathBuf;
/// Name of the config file
pub const FILENAME: &str = "treefmt.toml";
/// treefmt.toml structure
#[derive(Debug, Deserialize)]
pub struct Root {
/// Map of formatters into the config
pub formatter: BTreeMap<String, FmtConfig>,
}
/// Config for each formatters
#[derive(Debug, Deserialize)]
pub struct FmtConfig {
/// Command formatter to run
pub command: String,
/// Argument for formatter
#[serde(default)]
pub options: Vec<String>,
/// File or Folder that is included to be formatted
#[serde(default)]
pub includes: Vec<String>,
/// File or Folder that is excluded to be formatted
#[serde(default)]
pub excludes: Vec<String>,
}
/// Find the directory that contains the treefmt.toml file. From the current folder, and up.
pub fn lookup_dir(dir: &PathBuf) -> Option<PathBuf> {
let mut cwd = dir.clone();
loop {
if cwd.join(FILENAME).exists() {
return Some(cwd);
}
cwd = match cwd.parent() {
Some(x) => x.to_path_buf(),
// None is returned when .parent() is already the root folder. In that case we have
// exhausted the search space.
None => return None,
};
}
}
/// Loads the treefmt.toml config from the given file path.
pub fn from_path(path: &PathBuf) -> Result<Root> {
let content = read_to_string(path)?;
let ret: Root = toml::from_str(&content)?;
Ok(ret)
}

View File

@ -1,17 +1,13 @@
//! The main formatting engine logic should be in this module.
use crate::formatters::{
check::check_treefmt,
manifest::{create_manifest, read_manifest},
RootManifest,
};
use crate::{customlog, CmdContext, FileExtensions, FileMeta, Root, CLOG};
use crate::eval_cache::{check_treefmt, create_manifest, read_manifest, RootManifest};
use crate::{config, customlog, CmdContext, FileMeta, CLOG};
use anyhow::{anyhow, Error, Result};
use filetime::FileTime;
use rayon::prelude::*;
use std::collections::BTreeSet;
use std::fs::{metadata, read_to_string};
use std::iter::{IntoIterator, Iterator};
use std::fs::metadata;
use std::iter::Iterator;
use std::path::PathBuf;
use which::which;
use xshell::cmd;
@ -32,11 +28,13 @@ pub fn check_bin(command: &str) -> Result<()> {
/// Run the treefmt
pub fn run_treefmt(cwd: PathBuf, cache_dir: PathBuf) -> anyhow::Result<()> {
let treefmt_toml = cwd.join("treefmt.toml");
let treefmt_toml = cwd.join(config::FILENAME);
let project_config = config::from_path(&treefmt_toml)?;
// Once the treefmt found the $XDG_CACHE_DIR/treefmt/eval-cache/ folder,
// it will try to scan the manifest and passed it into check_treefmt function
let old_ctx = create_command_context(&treefmt_toml)?;
let old_ctx = create_command_context(&cwd, &project_config)?;
// TODO: Resolve all of the formatters paths. If missing, print an error, remove the formatters from the list and continue.
// Load the manifest if it exists, otherwise start with empty manifest
let mfst: RootManifest = read_manifest(&treefmt_toml, &cache_dir)?;
@ -87,7 +85,7 @@ pub fn run_treefmt(cwd: PathBuf, cache_dir: PathBuf) -> anyhow::Result<()> {
create_manifest(treefmt_toml, cache_dir, old_ctx)?;
} else {
// Read the current status of files and insert into the manifest.
let new_ctx = create_command_context(&treefmt_toml)?;
let new_ctx = create_command_context(&cwd, &project_config)?;
println!("Format successful");
println!("capturing formatted file's state...");
create_manifest(treefmt_toml, cache_dir, new_ctx)?;
@ -99,32 +97,19 @@ pub fn run_treefmt(cwd: PathBuf, cache_dir: PathBuf) -> anyhow::Result<()> {
/// Convert glob pattern into list of pathBuf
pub fn glob_to_path(
cwd: &PathBuf,
extensions: &FileExtensions,
includes: &Option<Vec<String>>,
excludes: &Option<Vec<String>>,
includes: &[String],
excludes: &[String],
) -> anyhow::Result<Vec<PathBuf>> {
use ignore::{overrides::OverrideBuilder, WalkBuilder};
let mut overrides_builder = OverrideBuilder::new(cwd);
if let Some(includes) = includes {
for include in includes {
// Remove trailing `/` as we add one explicitly in the override
let include = include.trim_end_matches('/');
for extension in extensions.into_iter() {
overrides_builder.add(&format!("{}/**/{}", include, extension))?;
}
}
} else {
for extension in extensions.into_iter() {
overrides_builder.add(&extension)?;
}
for include in includes {
overrides_builder.add(include)?;
}
if let Some(excludes) = excludes {
for exclude in excludes {
overrides_builder.add(&format!("!{}", exclude))?;
}
for exclude in excludes {
overrides_builder.add(&format!("!{}", exclude))?;
}
let overrides = overrides_builder.build()?;
@ -164,42 +149,18 @@ pub fn path_to_filemeta(paths: Vec<PathBuf>) -> Result<BTreeSet<FileMeta>> {
}
/// Creating command configuration based on treefmt.toml
pub fn create_command_context(treefmt_toml: &PathBuf) -> Result<Vec<CmdContext>> {
let open_treefmt = match read_to_string(treefmt_toml) {
Ok(file) => file,
Err(err) => {
return Err(anyhow!(
"cannot open {} due to {}.",
treefmt_toml.display(),
err
))
}
};
let cwd = match treefmt_toml.parent() {
Some(path) => path,
None => {
return Err(anyhow!(
"{}treefmt.toml not found, please run --init command",
customlog::ERROR
))
}
};
let toml_content: Root = toml::from_str(&open_treefmt)?;
pub fn create_command_context(
cwd: &PathBuf,
toml_content: &config::Root,
) -> Result<Vec<CmdContext>> {
let cmd_context: Vec<CmdContext> = toml_content
.formatters
.formatter
.values()
.map(|config| {
let list_files = glob_to_path(
&cwd.to_path_buf(),
&config.files,
&config.includes,
&config.excludes,
)?;
let list_files = glob_to_path(&cwd.to_path_buf(), &config.includes, &config.excludes)?;
Ok(CmdContext {
command: config.command.clone().unwrap_or_default(),
options: config.options.clone().unwrap_or_default(),
command: config.command.clone(),
options: config.options.clone(),
metadata: path_to_filemeta(list_files)?,
})
})
@ -216,11 +177,12 @@ mod tests {
#[test]
fn test_glob_to_path() -> Result<()> {
let cwd = PathBuf::from(r"examples");
let file_ext = FileExtensions::SingleFile("*.rs".to_string());
let includes = vec!["*.rs".to_string()];
let excludes: Vec<String> = vec![];
let glob_path = PathBuf::from(r"examples/rust/src/main.rs");
let mut vec_path = Vec::new();
vec_path.push(glob_path);
assert_eq!(glob_to_path(&cwd, &file_ext, &None, &None)?, vec_path);
assert_eq!(glob_to_path(&cwd, &includes, &excludes)?, vec_path);
Ok(())
}

View File

@ -1,13 +1,20 @@
use super::RootManifest;
//! Keep track of evaluations
use crate::{customlog, CmdContext, CLOG};
use anyhow::{anyhow, Result};
use anyhow::{anyhow, Error, Result};
use serde::{Deserialize, Serialize};
use sha1::{Digest, Sha1};
use std::collections::BTreeMap;
use std::fs::{read_to_string, File};
use std::io::Write;
use std::path::PathBuf;
#[derive(Debug, Deserialize, Serialize)]
/// RootManifest
pub struct RootManifest {
/// Map of manifests config based on its formatter
pub manifest: BTreeMap<String, CmdContext>,
}
/// Create <hex(hash(path-to-treefmt))>.toml and put it in $XDG_CACHE_DIR/treefmt/eval-cache/
pub fn create_manifest(
treefmt_toml: PathBuf,
@ -88,9 +95,60 @@ fn create_hash(treefmt_toml: &PathBuf) -> Result<String> {
Ok(manifest_toml)
}
/// Checking content of cache's file and current treefmt runs
pub fn check_treefmt(
treefmt_toml: &PathBuf,
cmd_context: &[CmdContext],
cache: &RootManifest,
) -> Result<Vec<CmdContext>> {
let cache_context = cache.manifest.values();
let results = cmd_context.iter().zip(cache_context);
let cache_context: Vec<CmdContext> = results
.clone()
.map(|(new, old)| {
Ok(CmdContext {
command: new.command.clone(),
options: new.options.clone(),
metadata: if new.command != old.command || new.options != old.options {
// If either the command or the options have changed, invalidate old entries
new.metadata.clone()
} else {
new.metadata.difference(&old.metadata).cloned().collect()
},
})
})
.filter(|c| match c {
Ok(x) => !x.metadata.is_empty(),
_ => false,
})
.collect::<Result<Vec<CmdContext>, Error>>()?;
if cache_context.iter().all(|f| f.metadata.is_empty()) {
CLOG.debug(&format!("No changes found in {}", treefmt_toml.display()));
return Ok(Vec::new());
}
CLOG.info("The following file has changed or newly added:");
for cmd in &cache_context {
if !cmd.metadata.is_empty() {
for p in &cmd.metadata {
CLOG.info(&format!(
" - {} last modification time: {}",
p.path.display(),
p.mtimes
));
}
}
}
// return Err(anyhow!("treefmt failed to run."));
Ok(cache_context)
}
#[cfg(test)]
mod tests {
use super::*;
use std::collections::BTreeMap;
/// Every same path produce same hash
#[test]
@ -100,4 +158,20 @@ mod tests {
assert_eq!(create_hash(&file_path)?, treefmt_hash);
Ok(())
}
/// Every same path produce same hash
#[test]
fn test_check_treefmt() -> Result<()> {
let treefmt_path = PathBuf::from(r"examples/monorepo/treefmt.toml");
let cache: RootManifest = RootManifest {
manifest: BTreeMap::new(),
};
let cmd_context: Vec<CmdContext> = Vec::new();
assert_eq!(
check_treefmt(&treefmt_path, &cmd_context, &cache)?,
cmd_context
);
Ok(())
}
}

View File

@ -1,77 +0,0 @@
use super::RootManifest;
use crate::{CmdContext, CLOG};
use anyhow::{Error, Result};
use std::path::PathBuf;
use std::vec::Vec;
/// Checking content of cache's file and current treefmt runs
pub fn check_treefmt(
treefmt_toml: &PathBuf,
cmd_context: &[CmdContext],
cache: &RootManifest,
) -> Result<Vec<CmdContext>> {
let cache_context = cache.manifest.values();
let results = cmd_context.iter().zip(cache_context);
let cache_context: Vec<CmdContext> = results
.clone()
.map(|(new, old)| {
Ok(CmdContext {
command: new.command.clone(),
options: new.options.clone(),
metadata: if new.command != old.command || new.options != old.options {
// If either the command or the options have changed, invalidate old entries
new.metadata.clone()
} else {
new.metadata.difference(&old.metadata).cloned().collect()
},
})
})
.filter(|c| match c {
Ok(x) => !x.metadata.is_empty(),
_ => false,
})
.collect::<Result<Vec<CmdContext>, Error>>()?;
if cache_context.iter().all(|f| f.metadata.is_empty()) {
CLOG.debug(&format!("No changes found in {}", treefmt_toml.display()));
return Ok(Vec::new());
}
CLOG.info("The following file has changed or newly added:");
for cmd in &cache_context {
if !cmd.metadata.is_empty() {
for p in &cmd.metadata {
CLOG.info(&format!(
" - {} last modification time: {}",
p.path.display(),
p.mtimes
));
}
}
}
// return Err(anyhow!("treefmt failed to run."));
Ok(cache_context)
}
#[cfg(test)]
mod tests {
use super::*;
use std::collections::BTreeMap;
/// Every same path produce same hash
#[test]
fn test_check_treefmt() -> Result<()> {
let treefmt_path = PathBuf::from(r"examples/monorepo/treefmt.toml");
let cache: RootManifest = RootManifest {
manifest: BTreeMap::new(),
};
let cmd_context: Vec<CmdContext> = Vec::new();
assert_eq!(
check_treefmt(&treefmt_path, &cmd_context, &cache)?,
cmd_context
);
Ok(())
}
}

View File

@ -1,18 +0,0 @@
//! Functionality related to installing prebuilt binaries
#![deny(missing_docs)]
/// File checking utility
pub mod check;
/// Manifest configuration
pub mod manifest;
use crate::CmdContext;
use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
#[derive(Debug, Deserialize, Serialize)]
/// RootManifest
pub struct RootManifest {
/// Map of manifests config based on its formatter
pub manifest: BTreeMap<String, CmdContext>,
}

View File

@ -2,64 +2,20 @@
#![deny(missing_docs)]
pub mod command;
pub mod config;
pub mod customlog;
pub mod engine;
pub mod formatters;
pub mod eval_cache;
use customlog::CustomLogOutput;
use serde::{Deserialize, Serialize};
use std::cmp::Ordering;
use std::collections::{BTreeMap, BTreeSet};
use std::collections::BTreeSet;
use std::path::PathBuf;
/// The global custom log and user-facing message output.
pub static CLOG: CustomLogOutput = CustomLogOutput::new();
/// treefmt.toml structure
#[derive(Debug, Deserialize)]
pub struct Root {
/// Map of formatters into the config
pub formatters: BTreeMap<String, FmtConfig>,
}
/// Config for each formatters
#[derive(Debug, Deserialize)]
pub struct FmtConfig {
/// File extensions that want to be formatted
pub files: FileExtensions,
/// File or Folder that is included to be formatted
pub includes: Option<Vec<String>>,
/// File or Folder that is excluded to be formatted
pub excludes: Option<Vec<String>>,
/// Command formatter to run
pub command: Option<String>,
/// Argument for formatter
pub options: Option<Vec<String>>,
}
/// File extensions can be single string (e.g. "*.hs") or
/// list of string (e.g. [ "*.hs", "*.rs" ])
#[derive(Debug, Deserialize, Clone)]
#[serde(untagged)]
pub enum FileExtensions {
/// Single file type
SingleFile(String),
/// List of file type
MultipleFile(Vec<String>),
}
impl<'a> IntoIterator for &'a FileExtensions {
type Item = &'a String;
type IntoIter = either::Either<std::iter::Once<&'a String>, std::slice::Iter<'a, String>>;
fn into_iter(self) -> Self::IntoIter {
match self {
FileExtensions::SingleFile(glob) => either::Either::Left(std::iter::once(glob)),
FileExtensions::MultipleFile(globs) => either::Either::Right(globs.iter()),
}
}
}
#[derive(Debug, Deserialize, Serialize, Clone)]
/// Each context of the formatter config
pub struct CmdContext {

52
treefmt.toml Normal file
View File

@ -0,0 +1,52 @@
# One CLI to format the code tree - https://github.com/numtide/treefmt
[formatter.python]
command = "black"
includes = ["*.py"]
[formatter.elm]
command = "elm-format"
options = ["--yes"]
includes = ["*.elm"]
[formatter.go]
command = "gofmt"
options = ["-w"]
includes = ["*.go"]
[formatter.haskell]
command = "ormolu"
options = [
"--ghc-opt", "-XBangPatterns",
"--ghc-opt", "-XPatternSynonyms",
"--ghc-opt", "-XTypeApplications",
"--mode", "inplace",
"--check-idempotence",
]
includes = ["*.hs"]
excludes = ["examples/haskell/"]
[formatter.javascript]
command = "prettier"
options = ["--write"]
includes = ["*.js"]
[formatter.markdown]
command = "prettier"
options = ["--write"]
includes = ["*.md"]
[formatter.rust]
command = "rustfmt"
options = ["--edition", "2018"]
includes = ["*.rs"]
[formatter.shell]
command = "shfmt"
options = [
"-i",
"2", # indent 2
"-s", # simplify the code
"-w", # write back to the file
]
includes = ["*.sh"]