on master help sign in
Merge pull request #6113 from commercialhaskell/commenter-stdin

Simpler `commenter` flow, allow piping stderr from `curator`

commit8b2741ace0aab27546fb568f4a630eae9e498bc9 authorChris Dornan <chris@chrisdornan.com> dateThu 29 Jul 2021, 10:59 (5 years ago) parent13cd90c3163a84
5 files changed+138−53
CURATORS.md+24−8 etc/commenter/Cargo.lock+67 etc/commenter/Cargo.toml+1 etc/commenter/README.md+2 etc/commenter/src/main.rs+44−45
CURATORS.md+24−8
@@ -173,12 +173,12 @@ This typically happens when we move to a new major GHC release or when
173173there are only a few packages waiting for updates on an upper bounds
174174issue.
175175
176−Comment out the offending packages from the "packages" section and add
177−a comment saying why it was disabled:
176+If a package needs to be disabled due to build failures: Add a `< 0`
177+bound to the package to exclude it, and add a comment stating why it
178+was disabled: `- swagger < 0 # compile failure againts aeson 1.0`
178179
179−```
180− # - swagger # bounds: aeson 1.0
181−```
180+If a package needs to be disabled due to bounds issues, see the "Large
181+scale enabling/disabling of packages" section below.
182182
183183If this causes reverse dependencies to be disabled we should notify
184184the maintainers of those packages.
@@ -413,7 +413,19 @@ errors for builds, tests and benchmarks.
413413`build-constraints.yaml`. It can only handle bounds issues,
414414compilation issues still need to be analyzed manually.
415415
416−Example usage: After disabling a few packages you get this curator output:
416+It disables all offending packages/test suites/benchmarks, so it is
417+only meant to be used when we close bounds issues and want to disable
418+packages, and when upgrading GHC.
419+
420+#### Setup
421+This is currently a rust program, You can install the rust toolchain
422+by using [rustup](https://rustup.rs/).
423+
424+Then `cd etc/commenter && cargo install --locked --path .`
425+
426+#### Example usage
427+
428+After disabling a few packages you get this curator output:
417429
418430```
419431ConfigFile (GHC 9 bounds issues, @maintainer) (not present) depended on by:
@@ -428,10 +440,14 @@ testing-feat (GHC 9 bounds issues, Grandfathered dependencies) (not present) dep
428440- [ ] dual-tree-0.2.3.0 (-any). Grandfathered dependencies. @handles. Used by: test-suite
429441```
430442
431−Copy this into `etc/commenter/comments.txt` and run `./commenter` (currently requires Rust)
443+Now run:
444+```
445+./check 2>&1 >/dev/null | commenter
446+```
432447
433448You will get this output:
434449```
450+[INFO] ...
435451LIBS + EXES
436452
437453 - xdg-desktop-entry < 0 # tried xdg-desktop-entry-0.1.1.1, but its *library* requires the disabled package: ConfigFile
@@ -448,7 +464,7 @@ TESTS have a similar section under `skipped-tests`, and BENCHMARKS under `skippe
448464
449465#### Re-enabling
450466
451−We can periodically remove all packages under the bounds sections and then re-run the disabling flow above until we get a clean plan. This way we will automatically pick up packages that have been fixed.
467+We can periodically remove all packages under the bounds sections and then re-run the disabling flow above until we get a clean plan. This will automatically pick up packages that have been fixed.
452468
453469#### Notes
454470
etc/commenter/Cargo.lock+67
@@ -1,5 +1,7 @@
11# This file is automatically @generated by Cargo.
22# It is not intended for manual editing.
3+version = 3
4+
35[[package]]
46name = "aho-corasick"
57version = "0.7.18"
@@ -13,15 +15,63 @@ dependencies = [
1315name = "commenter"
1416version = "0.1.0"
1517dependencies = [
18+ "lazy-regex",
1619 "regex",
1720]
1821
22+[[package]]
23+name = "lazy-regex"
24+version = "2.2.1"
25+source = "registry+https://github.com/rust-lang/crates.io-index"
26+checksum = "17d198f91272f6e788a5c0bd5d741cf778da4e5bc761ec67b32d5d3b0db34a54"
27+dependencies = [
28+ "lazy-regex-proc_macros",
29+ "once_cell",
30+ "regex",
31+]
32+
33+[[package]]
34+name = "lazy-regex-proc_macros"
35+version = "2.2.1"
36+source = "registry+https://github.com/rust-lang/crates.io-index"
37+checksum = "6c12938b1b92cf5be22940527e15b79fd0c7e706e34bc70816f6a72b3484f84e"
38+dependencies = [
39+ "proc-macro2",
40+ "quote",
41+ "regex",
42+ "syn",
43+]
44+
1945[[package]]
2046name = "memchr"
2147version = "2.4.0"
2248source = "registry+https://github.com/rust-lang/crates.io-index"
2349checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc"
2450
51+[[package]]
52+name = "once_cell"
53+version = "1.8.0"
54+source = "registry+https://github.com/rust-lang/crates.io-index"
55+checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56"
56+
57+[[package]]
58+name = "proc-macro2"
59+version = "1.0.27"
60+source = "registry+https://github.com/rust-lang/crates.io-index"
61+checksum = "f0d8caf72986c1a598726adc988bb5984792ef84f5ee5aa50209145ee8077038"
62+dependencies = [
63+ "unicode-xid",
64+]
65+
66+[[package]]
67+name = "quote"
68+version = "1.0.9"
69+source = "registry+https://github.com/rust-lang/crates.io-index"
70+checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7"
71+dependencies = [
72+ "proc-macro2",
73+]
74+
2575[[package]]
2676name = "regex"
2777version = "1.5.4"
@@ -38,3 +88,20 @@ name = "regex-syntax"
3888version = "0.6.25"
3989source = "registry+https://github.com/rust-lang/crates.io-index"
4090checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b"
91+
92+[[package]]
93+name = "syn"
94+version = "1.0.73"
95+source = "registry+https://github.com/rust-lang/crates.io-index"
96+checksum = "f71489ff30030d2ae598524f61326b902466f72a0fb1a8564c001cc63425bcc7"
97+dependencies = [
98+ "proc-macro2",
99+ "quote",
100+ "unicode-xid",
101+]
102+
103+[[package]]
104+name = "unicode-xid"
105+version = "0.2.2"
106+source = "registry+https://github.com/rust-lang/crates.io-index"
107+checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3"
etc/commenter/Cargo.toml+1
@@ -7,4 +7,5 @@ edition = "2018"
77# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
88
99[dependencies]
10+lazy-regex = "2.2.1"
1011regex = "1.5.4"
etc/commenter/README.md+2
@@ -1 +1,3 @@
11Helps automate mass-disabling of packages in Stackage's build-constraint.yaml.
2+
3+See CURATORS.md for usage instructions.
etc/commenter/src/main.rs+44−45
@@ -1,8 +1,7 @@
1−use std::collections::HashMap;
2−use std::fs::File;
1+use std::collections::{HashMap};
32use std::io::{self, BufRead};
4−use std::path::Path;
53
4+use lazy_regex::regex;
65use regex::Regex;
76
87type H = HashMap<Header, Vec<(String, String, String)>>;
@@ -18,43 +17,47 @@ fn main() {
1817 let mut tests: H = Default::default();
1918 let mut benches: H = Default::default();
2019 let mut last_header: Option<Header> = None;
21− let empty = Regex::new(r#"^\s*$"#).unwrap();
22− let header_versioned =
23− Regex::new(r#"^(?P<package>[a-zA-z]([a-zA-z0-9.-]*?))-(?P<version>(\d+(\.\d+)*)).+?is out of bounds for:$"#).unwrap();
24− let header_missing =
25− Regex::new(r#"^(?P<package>[a-zA-z]([a-zA-z0-9.-]*)).+?depended on by:$"#).unwrap();
26− let package =
27− Regex::new(r#"^- \[ \] (?P<package>[a-zA-z]([a-zA-z0-9.-]*?))-(?P<version>(\d+(\.\d+)*)).+?Used by: (?P<component>.+)$"#)
28− .unwrap();
2920
30− if let Ok(lines) = read_lines("./comments.txt") {
31− for line in lines {
32− if let Ok(line) = line {
33− if empty.captures(&line).is_some() {
34− } else if let Some(cap) = package.captures(&line) {
35− let root = last_header.clone().unwrap();
36− let package = cap.name("package").unwrap().as_str();
37− let version = cap.name("version").unwrap().as_str();
38− let component = cap.name("component").unwrap().as_str();
39− match component {
40− "library" | "executable" => {
41− insert(&mut lib_exes, root, package, version, component)
42− }
43− "benchmark" => insert(&mut benches, root, package, version, "benchmarks"),
44− "test-suite" => insert(&mut tests, root, package, version, component),
45− _ => panic!("Bad component: {}", component),
46− }
47− } else if let Some(cap) = header_versioned.captures(&line) {
48− let package = cap.name("package").unwrap().as_str().to_owned();
49− let version = cap.name("version").unwrap().as_str().to_owned();
50− last_header = Some(Header::Versioned { package, version });
51− } else if let Some(cap) = header_missing.captures(&line) {
52− let package = cap.name("package").unwrap().as_str().to_owned();
53− last_header = Some(Header::Missing { package });
54− } else {
55− panic!("Unhandled: {:?}", line);
21+ let header_versioned = regex!(
22+ r#"^(?P<package>[a-zA-z]([a-zA-z0-9.-]*?))-(?P<version>(\d+(\.\d+)*)).+?is out of bounds for:$"#
23+ );
24+ let header_missing = regex!(r#"^(?P<package>[a-zA-z]([a-zA-z0-9.-]*)).+?depended on by:$"#);
25+ let package = regex!(
26+ r#"^- \[ \] (?P<package>[a-zA-z]([a-zA-z0-9.-]*?))-(?P<version>(\d+(\.\d+)*)).+?Used by: (?P<component>.+)$"#
27+ );
28+
29+ // Ignore everything until the bounds issues show up.
30+ let mut process_line = false;
31+
32+ for line in io::stdin().lock().lines().flatten() {
33+ if is_reg_match(&line, regex!(r#"^\s*$"#)) {
34+ // noop
35+ } else if line == "curator: Snapshot dependency graph contains errors:" {
36+ process_line = true;
37+ } else if !process_line {
38+ println!("[INFO] {}", line);
39+ } else if let Some(cap) = package.captures(&line) {
40+ let root = last_header.clone().unwrap();
41+ let package = cap.name("package").unwrap().as_str();
42+ let version = cap.name("version").unwrap().as_str();
43+ let component = cap.name("component").unwrap().as_str();
44+ match component {
45+ "library" | "executable" => {
46+ insert(&mut lib_exes, root, package, version, component)
5647 }
48+ "benchmark" => insert(&mut benches, root, package, version, "benchmarks"),
49+ "test-suite" => insert(&mut tests, root, package, version, component),
50+ _ => panic!("Bad component: {}", component),
5751 }
52+ } else if let Some(cap) = header_versioned.captures(&line) {
53+ let package = cap.name("package").unwrap().as_str().to_owned();
54+ let version = cap.name("version").unwrap().as_str().to_owned();
55+ last_header = Some(Header::Versioned { package, version });
56+ } else if let Some(cap) = header_missing.captures(&line) {
57+ let package = cap.name("package").unwrap().as_str().to_owned();
58+ last_header = Some(Header::Missing { package });
59+ } else {
60+ panic!("Unhandled: {:?}", line);
5861 }
5962 }
6063
@@ -93,7 +96,7 @@ fn printer(
9396 version: &str,
9497 component: &str,
9598 header: &Header,
96−) -> () {
99+) {
97100 let lt0 = if lt0 { " < 0" } else { "" };
98101 println!(
99102 "{indent}- {package}{lt0} # tried {package}-{version}, but its *{component}* {cause}",
@@ -117,17 +120,13 @@ fn printer(
117120}
118121
119122fn insert(h: &mut H, header: Header, package: &str, version: &str, component: &str) {
120− (*h.entry(header).or_insert_with(|| vec![])).push((
123+ (*h.entry(header).or_insert_with(Vec::new)).push((
121124 package.to_owned(),
122125 version.to_owned(),
123126 component.to_owned(),
124127 ));
125128}
126129
127−fn read_lines<P>(filename: P) -> io::Result<io::Lines<io::BufReader<File>>>
128−where
129− P: AsRef<Path>,
130−{
131− let file = File::open(filename)?;
132− Ok(io::BufReader::new(file).lines())
130+fn is_reg_match(s: &str, r: &Regex) -> bool {
131+ r.captures(s).is_some()
133132}