git.iko.soy/stackage
on master help sign in
Add `commenter` script to help manage bounds issues
7f6003bAdam Bergmark
CURATORS.md+50
compare with master
unifiedsplit
@@ -406,6 +406,56 @@ errors for builds, tests and benchmarks.
406
406
407[tell-me-when-its-released]: https://github.com/commercialhaskell/stackage/blob/master/CURATORS.md#waiting-for-new-releases
407[tell-me-when-its-released]: https://github.com/commercialhaskell/stackage/blob/master/CURATORS.md#waiting-for-new-releases
408
408
409+### Large scale enabling/disabling of packages
410+
411+`etc/commenter` is a binary that semi-automates the translation of
412+`./check` errors into lines that can be copy pasted into
413+`build-constraints.yaml`. It can only handle bounds issues,
414+compilation issues still need to be analyzed manually.
415+
416+Example usage: After disabling a few packages you get this curator output:
417+
418+```
419+ConfigFile (GHC 9 bounds issues, @maintainer) (not present) depended on by:
420+- [ ] xdg-desktop-entry-0.1.1.1 (-any). @maintainer. Used by: library
421+
422+
423+pipes-misc (GHC 9 bounds issues, @maintainer) (not present) depended on by:
424+- [ ] pipes-fluid-0.6.0.1 (>=0.5). @handles. Used by: test-suite
425+
426+
427+testing-feat (GHC 9 bounds issues, Grandfathered dependencies) (not present) depended on by:
428+- [ ] dual-tree-0.2.3.0 (-any). Grandfathered dependencies. @handles. Used by: test-suite
429+```
430+
431+Copy this into `etc/commenter/comments.txt` and run `./commenter` (currently requires Rust)
432+
433+You will get this output:
434+```
435+LIBS + EXES
436+
437+ - xdg-desktop-entry < 0 # tried xdg-desktop-entry-0.1.1.1, but its *library* requires the disabled package: ConfigFile
438+
439+TESTS
440+
441+ - dual-tree # tried dual-tree-0.2.3.0, but its *test-suite* requires the disabled package: testing-feat
442+ - pipes-fluid # tried pipes-fluid-0.6.0.1, but its *test-suite* requires the disabled package: pipes-misc
443+```
444+
445+The lines under LIBS+EXES should be pasted in the shared section (currently called "GHC 9 bounds issues").
446+
447+TESTS have a similar section under `skipped-tests`, and BENCHMARKS under `skipped-benchmarks`.
448+
449+#### Re-enabling
450+
451+We can periodically remove all packages under the bounds sections and then re-run the disabling flow above until we get a clean plan. This way we will automatically pick up packages that have been fixed.
452+
453+#### Notes
454+
455+* Please keep these lists sorted as the diffs will be much cleaner when we re-enable packages and re-run this flow
456+* Please make sure to separate bounds issues from compilation failures/test run failures, as we cannot verify that a package builds or that tests pass without running the build!
457+
458+
409## Adding new curators
459## Adding new curators
410
460
4111. Add public ssh key to `~/.ssh/authorized_keys` on build server
4611. Add public ssh key to `~/.ssh/authorized_keys` on build server
etc/commenter/.gitignorenew file+3
@@ -0,0 +1,3 @@
1+/target
2+/comments.txt
3+/out.txt
etc/commenter/Cargo.locknew file+40
@@ -0,0 +1,40 @@
1+# This file is automatically @generated by Cargo.
2+# It is not intended for manual editing.
3+[[package]]
4+name = "aho-corasick"
5+version = "0.7.18"
6+source = "registry+https://github.com/rust-lang/crates.io-index"
7+checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f"
8+dependencies = [
9+ "memchr",
10+]
11+
12+[[package]]
13+name = "commenter"
14+version = "0.1.0"
15+dependencies = [
16+ "regex",
17+]
18+
19+[[package]]
20+name = "memchr"
21+version = "2.4.0"
22+source = "registry+https://github.com/rust-lang/crates.io-index"
23+checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc"
24+
25+[[package]]
26+name = "regex"
27+version = "1.5.4"
28+source = "registry+https://github.com/rust-lang/crates.io-index"
29+checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461"
30+dependencies = [
31+ "aho-corasick",
32+ "memchr",
33+ "regex-syntax",
34+]
35+
36+[[package]]
37+name = "regex-syntax"
38+version = "0.6.25"
39+source = "registry+https://github.com/rust-lang/crates.io-index"
40+checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b"
etc/commenter/Cargo.tomlnew file+10
@@ -0,0 +1,10 @@
1+[package]
2+name = "commenter"
3+version = "0.1.0"
4+authors = ["Adam Bergmark <adam@bergmark.nl>"]
5+edition = "2018"
6+
7+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
8+
9+[dependencies]
10+regex = "1.5.4"
etc/commenter/README.mdnew file+1
@@ -0,0 +1 @@
1+Helps automate mass-disabling of packages in Stackage's build-constraint.yaml.
etc/commenter/commenternew file+2
@@ -0,0 +1,2 @@
1+#!/bin/bash
2+cargo run -- $@
etc/commenter/src/main.rsnew file+133
@@ -0,0 +1,133 @@
1+use std::collections::HashMap;
2+use std::fs::File;
3+use std::io::{self, BufRead};
4+use std::path::Path;
5+
6+use regex::Regex;
7+
8+type H = HashMap<Header, Vec<(String, String, String)>>;
9+
10+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
11+enum Header {
12+ Versioned { package: String, version: String },
13+ Missing { package: String },
14+}
15+
16+fn main() {
17+ let mut lib_exes: H = Default::default();
18+ let mut tests: H = Default::default();
19+ let mut benches: H = Default::default();
20+ let mut last_header: Option<Header> = None;
21+ let empty = Regex::new(r#"^\s*$"#).unwrap();
22+ let header_versioned =
23+ Regex::new(r#"^(?P<package>[a-zA-z]([a-zA-z0-9.-]*?))-(?P<version>(\d+(\.\d+)*)).+?is out of bounds for:$"#).unwrap();
24+ let header_missing =
25+ Regex::new(r#"^(?P<package>[a-zA-z]([a-zA-z0-9.-]*)).+?depended on by:$"#).unwrap();
26+ let package =
27+ Regex::new(r#"^- \[ \] (?P<package>[a-zA-z]([a-zA-z0-9.-]*?))-(?P<version>(\d+(\.\d+)*)).+?Used by: (?P<component>.+)$"#)
28+ .unwrap();
29+
30+ if let Ok(lines) = read_lines("./comments.txt") {
31+ for line in lines {
32+ if let Ok(line) = line {
33+ if empty.captures(&line).is_some() {
34+ } else if let Some(cap) = package.captures(&line) {
35+ let root = last_header.clone().unwrap();
36+ let package = cap.name("package").unwrap().as_str();
37+ let version = cap.name("version").unwrap().as_str();
38+ let component = cap.name("component").unwrap().as_str();
39+ match component {
40+ "library" | "executable" => {
41+ insert(&mut lib_exes, root, package, version, component)
42+ }
43+ "benchmark" => insert(&mut benches, root, package, version, "benchmarks"),
44+ "test-suite" => insert(&mut tests, root, package, version, component),
45+ _ => panic!("Bad component: {}", component),
46+ }
47+ } else if let Some(cap) = header_versioned.captures(&line) {
48+ let package = cap.name("package").unwrap().as_str().to_owned();
49+ let version = cap.name("version").unwrap().as_str().to_owned();
50+ last_header = Some(Header::Versioned { package, version });
51+ } else if let Some(cap) = header_missing.captures(&line) {
52+ let package = cap.name("package").unwrap().as_str().to_owned();
53+ last_header = Some(Header::Missing { package });
54+ } else {
55+ panic!("Unhandled: {:?}", line);
56+ }
57+ }
58+ }
59+ }
60+
61+ if !lib_exes.is_empty() {
62+ println!("\nLIBS + EXES\n");
63+ }
64+ for (header, packages) in lib_exes {
65+ for (package, version, component) in packages {
66+ printer(" ", &package, true, &version, &component, &header);
67+ }
68+ }
69+
70+ if !tests.is_empty() {
71+ println!("\nTESTS\n");
72+ }
73+ for (header, packages) in tests {
74+ for (package, version, component) in packages {
75+ printer(" ", &package, false, &version, &component, &header);
76+ }
77+ }
78+
79+ if !benches.is_empty() {
80+ println!("\nBENCHMARKS\n");
81+ }
82+ for (header, packages) in benches {
83+ for (package, version, component) in packages {
84+ printer(" ", &package, false, &version, &component, &header);
85+ }
86+ }
87+}
88+
89+fn printer(
90+ indent: &str,
91+ package: &str,
92+ lt0: bool,
93+ version: &str,
94+ component: &str,
95+ header: &Header,
96+) -> () {
97+ let lt0 = if lt0 { " < 0" } else { "" };
98+ println!(
99+ "{indent}- {package}{lt0} # tried {package}-{version}, but its *{component}* {cause}",
100+ indent = indent,
101+ package = package,
102+ lt0 = lt0,
103+ version = version,
104+ component = component,
105+ cause = match header {
106+ Header::Versioned { package, version } => format!(
107+ "does not support: {package}-{version}",
108+ package = package,
109+ version = version
110+ ),
111+ Header::Missing { package } => format!(
112+ "requires the disabled package: {package}",
113+ package = package
114+ ),
115+ },
116+ );
117+}
118+
119+fn insert(h: &mut H, header: Header, package: &str, version: &str, component: &str) {
120+ (*h.entry(header).or_insert_with(|| vec![])).push((
121+ package.to_owned(),
122+ version.to_owned(),
123+ component.to_owned(),
124+ ));
125+}
126+
127+fn read_lines<P>(filename: P) -> io::Result<io::Lines<io::BufReader<File>>>
128+where
129+ P: AsRef<Path>,
130+{
131+ let file = File::open(filename)?;
132+ Ok(io::BufReader::new(file).lines())
133+}