git.iko.soy/stackage
on master help sign in
commenter: Insert constraints into build-constraints.yaml
commitb9cbeb27d40dc11bab46c59dfc2336f0a64f9e52 authorAdam Bergmark <adam@bergmark.nl> dateSat 4 Dec 2021, 14:46 (5 years ago) parentb7f7a68
6 files changed+378−23
compare with master
unifiedsplit
CURATORS.md+17−9 build-constraints.yaml+5−6 etc/commenter/Cargo.lock+181 etc/commenter/Cargo.toml+2−1 etc/commenter/src/lib.rs+124 etc/commenter/src/main.rs+49−7
CURATORS.md+17−9
@@ -421,10 +421,10 @@ errors for builds, tests and benchmarks.
421421
422422### Large scale enabling/disabling of packages
423423
424−`etc/commenter` is a binary that semi-automates the translation of
425−`./check` errors into lines that can be copy pasted into
426−`build-constraints.yaml`. It can only handle bounds issues,
427−compilation issues still need to be analyzed manually.
424+`etc/commenter` is a binary that mostly automates the translation of
425+`./check` errors into lines that go into `build-constraints.yaml`. It
426+can only handle bounds issues, compilation issues still need to be
427+handled manually.
428428
429429It disables all offending packages/test suites/benchmarks, so it is
430430only meant to be used when we close bounds issues and want to disable
@@ -434,7 +434,7 @@ packages, and when upgrading GHC.
434434This is currently a rust program, You can install the rust toolchain
435435by using [rustup](https://rustup.rs/).
436436
437−Then `cd etc/commenter && cargo install --locked --path .`
437+Then `cargo install --locked --path etc/commenter`
438438
439439#### Example usage
440440
@@ -455,7 +455,7 @@ testing-feat (GHC 9 bounds issues, Grandfathered dependencies) (not present) dep
455455
456456Now run:
457457```
458−./check 2>&1 >/dev/null | commenter
458+./check 2>&1 >/dev/null | commenter add
459459```
460460
461461You will get this output:
@@ -469,19 +469,27 @@ TESTS
469469
470470 - dual-tree # tried dual-tree-0.2.3.0, but its *test-suite* requires the disabled package: testing-feat
471471 - pipes-fluid # tried pipes-fluid-0.6.0.1, but its *test-suite* requires the disabled package: pipes-misc
472+
473+Adding 1 libs, 2 tests, 0 benches to build-constraints.yaml
472474```
473475
474−The lines under LIBS+EXES should be pasted in the shared section (currently called "GHC 9 bounds issues").
476+These bounds are added to build-constraints.yaml automatically.
475477
476−TESTS have a similar section under `skipped-tests`, and BENCHMARKS under `skipped-benchmarks`.
478+Re-run this command until no more packages are disabled.
477479
478480#### Re-enabling
479481
480482We can periodically remove all packages under the bounds sections and then re-run the disabling flow above until we get a clean plan. This will automatically pick up packages that have been fixed.
481483
484+```
485+commenter clear
486+./check 2>&1 >/dev/null | commenter add
487+```
488+
489+Repeat the second command until no updates are made to build-constraints.yaml.
490+
482491#### Notes
483492
484−* Please keep these lists sorted as the diffs will be much cleaner when we re-enable packages and re-run this flow
485493* Please make sure to separate bounds issues from compilation failures/test run failures, as we cannot verify that a package builds or that tests pass without running the build!
486494
487495
build-constraints.yaml+5−6
@@ -6667,6 +6667,7 @@ packages:
66676667 - ziptastic-core < 0 # tried ziptastic-core-0.2.0.3, but its *library* does not support: servant-0.18.3
66686668 - zydiskell < 0 # tried zydiskell-0.2.0.0, but its *library* does not support: base-4.15.0.0
66696669 - zydiskell < 0 # tried zydiskell-0.2.0.0, but its *library* does not support: storable-record-0.0.6
6670+ # End of Library and exe bounds failures
66706671
66716672 "Stackage upper bounds":
66726673
@@ -7039,11 +7040,9 @@ skipped-tests:
70397040 # https://github.com/clash-lang/clash-compiler/issues/1622
70407041 - clash-prelude
70417042
7042− #
7043− # Test bounds issues
7044− #
70457043 # See "Large scale enabling/disabling of packages" in CURATORS.md for how to manage this section.
70467044 #
7045+ # Test bounds issues
70477046 - ENIG # tried ENIG-0.0.1.0, but its *test-suite* requires the disabled package: test-framework-th
70487047 - IPv6DB # tried IPv6DB-0.3.2, but its *test-suite* does not support: hspec-2.8.5
70497048 - IPv6DB # tried IPv6DB-0.3.2, but its *test-suite* does not support: http-client-0.7.9
@@ -7263,6 +7262,7 @@ skipped-tests:
72637262 - wreq # tried wreq-0.5.3.3, but its *test-suite* requires the disabled package: snap-server
72647263 - xmlhtml # tried xmlhtml-0.2.5.2, but its *test-suite* does not support: hspec-2.8.5
72657264 - yesod-static-angular # tried yesod-static-angular-0.1.8, but its *test-suite* does not support: yesod-test-1.6.12
7265+ # End of Test bounds issues
72667266
72677267# end of skipped-tests
72687268
@@ -7788,11 +7788,9 @@ skipped-benchmarks:
77887788 - prettyprinter
77897789 - prettyprinter-ansi-terminal # https://github.com/commercialhaskell/stackage/issues/5560
77907790
7791− #
7792− # Benchmark bounds issues
7793− #
77947791 # See "Large scale enabling/disabling of packages" in CURATORS.md for how to manage this section.
77957792 #
7793+ # Benchmark bounds issues
77967794 - IntervalMap # tried IntervalMap-0.6.1.2, but its *benchmarks* requires the disabled package: SegmentTree
77977795 - accelerate-bignum # tried accelerate-bignum-0.3.0.0, but its *benchmarks* requires the disabled package: accelerate-io-vector
77987796 - accelerate-fourier # tried accelerate-fourier-1.0.0.5, but its *benchmarks* does not support: accelerate-llvm-native-1.3.0.0
@@ -7827,6 +7825,7 @@ skipped-benchmarks:
78277825 - tz # tried tz-0.1.3.5, but its *benchmarks* requires the disabled package: thyme
78287826 - unicode-transforms # tried unicode-transforms-0.3.8, but its *benchmarks* does not support: path-0.9.1
78297827 - xxhash-ffi # tried xxhash-ffi-0.2.0.0, but its *benchmarks* requires the disabled package: xxhash
7828+ # End of Benchmark bounds issues
78307829
78317830# end of skipped-benchmarks
78327831
etc/commenter/Cargo.lock+181
@@ -11,12 +11,72 @@ dependencies = [
1111 "memchr",
1212]
1313
14+[[package]]
15+name = "ansi_term"
16+version = "0.12.1"
17+source = "registry+https://github.com/rust-lang/crates.io-index"
18+checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2"
19+dependencies = [
20+ "winapi",
21+]
22+
23+[[package]]
24+name = "atty"
25+version = "0.2.14"
26+source = "registry+https://github.com/rust-lang/crates.io-index"
27+checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
28+dependencies = [
29+ "hermit-abi",
30+ "libc",
31+ "winapi",
32+]
33+
34+[[package]]
35+name = "bitflags"
36+version = "1.3.2"
37+source = "registry+https://github.com/rust-lang/crates.io-index"
38+checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
39+
40+[[package]]
41+name = "clap"
42+version = "2.34.0"
43+source = "registry+https://github.com/rust-lang/crates.io-index"
44+checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c"
45+dependencies = [
46+ "ansi_term",
47+ "atty",
48+ "bitflags",
49+ "strsim",
50+ "textwrap",
51+ "unicode-width",
52+ "vec_map",
53+]
54+
1455[[package]]
1556name = "commenter"
1657version = "0.1.0"
1758dependencies = [
1859 "lazy-regex",
1960 "regex",
61+ "structopt",
62+]
63+
64+[[package]]
65+name = "heck"
66+version = "0.3.3"
67+source = "registry+https://github.com/rust-lang/crates.io-index"
68+checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c"
69+dependencies = [
70+ "unicode-segmentation",
71+]
72+
73+[[package]]
74+name = "hermit-abi"
75+version = "0.1.19"
76+source = "registry+https://github.com/rust-lang/crates.io-index"
77+checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
78+dependencies = [
79+ "libc",
2080]
2181
2282[[package]]
@@ -42,6 +102,18 @@ dependencies = [
42102 "syn",
43103]
44104
105+[[package]]
106+name = "lazy_static"
107+version = "1.4.0"
108+source = "registry+https://github.com/rust-lang/crates.io-index"
109+checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
110+
111+[[package]]
112+name = "libc"
113+version = "0.2.108"
114+source = "registry+https://github.com/rust-lang/crates.io-index"
115+checksum = "8521a1b57e76b1ec69af7599e75e38e7b7fad6610f037db8c79b127201b5d119"
116+
45117[[package]]
46118name = "memchr"
47119version = "2.4.0"
@@ -54,6 +126,30 @@ version = "1.8.0"
54126source = "registry+https://github.com/rust-lang/crates.io-index"
55127checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56"
56128
129+[[package]]
130+name = "proc-macro-error"
131+version = "1.0.4"
132+source = "registry+https://github.com/rust-lang/crates.io-index"
133+checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c"
134+dependencies = [
135+ "proc-macro-error-attr",
136+ "proc-macro2",
137+ "quote",
138+ "syn",
139+ "version_check",
140+]
141+
142+[[package]]
143+name = "proc-macro-error-attr"
144+version = "1.0.4"
145+source = "registry+https://github.com/rust-lang/crates.io-index"
146+checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869"
147+dependencies = [
148+ "proc-macro2",
149+ "quote",
150+ "version_check",
151+]
152+
57153[[package]]
58154name = "proc-macro2"
59155version = "1.0.27"
@@ -89,6 +185,36 @@ version = "0.6.25"
89185source = "registry+https://github.com/rust-lang/crates.io-index"
90186checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b"
91187
188+[[package]]
189+name = "strsim"
190+version = "0.8.0"
191+source = "registry+https://github.com/rust-lang/crates.io-index"
192+checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
193+
194+[[package]]
195+name = "structopt"
196+version = "0.3.25"
197+source = "registry+https://github.com/rust-lang/crates.io-index"
198+checksum = "40b9788f4202aa75c240ecc9c15c65185e6a39ccdeb0fd5d008b98825464c87c"
199+dependencies = [
200+ "clap",
201+ "lazy_static",
202+ "structopt-derive",
203+]
204+
205+[[package]]
206+name = "structopt-derive"
207+version = "0.4.18"
208+source = "registry+https://github.com/rust-lang/crates.io-index"
209+checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0"
210+dependencies = [
211+ "heck",
212+ "proc-macro-error",
213+ "proc-macro2",
214+ "quote",
215+ "syn",
216+]
217+
92218[[package]]
93219name = "syn"
94220version = "1.0.73"
@@ -100,8 +226,63 @@ dependencies = [
100226 "unicode-xid",
101227]
102228
229+[[package]]
230+name = "textwrap"
231+version = "0.11.0"
232+source = "registry+https://github.com/rust-lang/crates.io-index"
233+checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060"
234+dependencies = [
235+ "unicode-width",
236+]
237+
238+[[package]]
239+name = "unicode-segmentation"
240+version = "1.8.0"
241+source = "registry+https://github.com/rust-lang/crates.io-index"
242+checksum = "8895849a949e7845e06bd6dc1aa51731a103c42707010a5b591c0038fb73385b"
243+
244+[[package]]
245+name = "unicode-width"
246+version = "0.1.9"
247+source = "registry+https://github.com/rust-lang/crates.io-index"
248+checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973"
249+
103250[[package]]
104251name = "unicode-xid"
105252version = "0.2.2"
106253source = "registry+https://github.com/rust-lang/crates.io-index"
107254checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3"
255+
256+[[package]]
257+name = "vec_map"
258+version = "0.8.2"
259+source = "registry+https://github.com/rust-lang/crates.io-index"
260+checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"
261+
262+[[package]]
263+name = "version_check"
264+version = "0.9.3"
265+source = "registry+https://github.com/rust-lang/crates.io-index"
266+checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe"
267+
268+[[package]]
269+name = "winapi"
270+version = "0.3.9"
271+source = "registry+https://github.com/rust-lang/crates.io-index"
272+checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
273+dependencies = [
274+ "winapi-i686-pc-windows-gnu",
275+ "winapi-x86_64-pc-windows-gnu",
276+]
277+
278+[[package]]
279+name = "winapi-i686-pc-windows-gnu"
280+version = "0.4.0"
281+source = "registry+https://github.com/rust-lang/crates.io-index"
282+checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
283+
284+[[package]]
285+name = "winapi-x86_64-pc-windows-gnu"
286+version = "0.4.0"
287+source = "registry+https://github.com/rust-lang/crates.io-index"
288+checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
etc/commenter/Cargo.toml+2−1
@@ -1,6 +1,6 @@
11[package]
22name = "commenter"
3−version = "0.1.0"
3+version = "0.2.0"
44authors = ["Adam Bergmark <adam@bergmark.nl>"]
55edition = "2018"
66
@@ -9,3 +9,4 @@ edition = "2018"
99[dependencies]
1010lazy-regex = "2.2.1"
1111regex = "1.5.4"
12+structopt = "0.3.25"
etc/commenter/src/lib.rsnew file+124
@@ -0,0 +1,124 @@
1+use std::fs::File;
2+use std::io::{BufRead, BufReader, LineWriter, Lines, Write};
3+use std::path::Path;
4+
5+pub fn clear() {
6+ handle(|loc, _lines| match loc {
7+ // Add empty array to keep yaml valid
8+ Location::Lib => vec![" []".to_owned()],
9+ Location::Test | Location::Bench => vec![],
10+ });
11+}
12+
13+pub fn add(lib: Vec<String>, test: Vec<String>, bench: Vec<String>) {
14+ handle(|loc, mut lines| {
15+ lines.extend(match loc {
16+ Location::Lib => lib.clone(),
17+ Location::Test => test.clone(),
18+ Location::Bench => bench.clone(),
19+ });
20+ lines.sort();
21+ lines
22+ })
23+}
24+
25+enum State {
26+ LookingForLibBounds,
27+ ProcessingLibBounds,
28+ LookingForTestBounds,
29+ ProcessingTestBounds,
30+ LookingForBenchBounds,
31+ ProcessingBenchBounds,
32+ Done,
33+}
34+
35+fn handle<F>(f: F)
36+where
37+ F: Fn(Location, Vec<String>) -> Vec<String>,
38+{
39+ let path = "build-constraints.yaml";
40+ let mut new_lines: Vec<String> = vec![];
41+
42+ let mut state = State::LookingForLibBounds;
43+ let mut buf = vec![];
44+ for line in read_lines(path).map(|s| s.unwrap()) {
45+ match state {
46+ State::LookingForLibBounds => {
47+ if line == r#" "Library and exe bounds failures":"# {
48+ state = State::ProcessingLibBounds;
49+ }
50+ new_lines.push(line);
51+ }
52+ State::ProcessingLibBounds => {
53+ if line == r#" # End of Library and exe bounds failures"# {
54+ new_lines.extend(f(Location::Lib, buf).into_iter());
55+ buf = vec![];
56+ new_lines.push(line);
57+ state = State::LookingForTestBounds;
58+ } else {
59+ // Remove empty section
60+ if line != " []" {
61+ buf.push(line);
62+ }
63+ }
64+ }
65+ State::LookingForTestBounds => {
66+ if line == r#" # Test bounds issues"# {
67+ state = State::ProcessingTestBounds;
68+ }
69+ new_lines.push(line);
70+ }
71+ State::ProcessingTestBounds => {
72+ if line == r#" # End of Test bounds issues"# {
73+ new_lines.extend(f(Location::Test, buf).into_iter());
74+ buf = vec![];
75+ new_lines.push(line);
76+ state = State::LookingForBenchBounds;
77+ } else {
78+ buf.push(line);
79+ }
80+ }
81+ State::LookingForBenchBounds => {
82+ if line == r#" # Benchmark bounds issues"# {
83+ state = State::ProcessingBenchBounds;
84+ }
85+ new_lines.push(line);
86+ }
87+ State::ProcessingBenchBounds => {
88+ if line == r#" # End of Benchmark bounds issues"# {
89+ new_lines.extend(f(Location::Bench, buf).into_iter());
90+ buf = vec![];
91+ new_lines.push(line);
92+ state = State::Done;
93+ } else {
94+ buf.push(line);
95+ }
96+ }
97+ State::Done => {
98+ new_lines.push(line);
99+ }
100+ }
101+ }
102+
103+ let file = File::create(path).unwrap();
104+ let mut file = LineWriter::new(file);
105+
106+ for line in new_lines {
107+ file.write_all((line + "\n").as_bytes()).unwrap();
108+ }
109+ file.flush().unwrap();
110+}
111+
112+enum Location {
113+ Lib,
114+ Test,
115+ Bench,
116+}
117+
118+fn read_lines<P>(filename: P) -> Lines<BufReader<File>>
119+where
120+ P: AsRef<Path>,
121+{
122+ let file = File::open(filename).unwrap();
123+ BufReader::new(file).lines()
124+}
etc/commenter/src/main.rs+49−7
@@ -1,8 +1,9 @@
1−use std::collections::{HashMap};
1+use std::collections::HashMap;
22use std::io::{self, BufRead};
33
44use lazy_regex::regex;
55use regex::Regex;
6+use structopt::StructOpt;
67
78type H = HashMap<Header, Vec<(String, String, String)>>;
89
@@ -12,7 +13,29 @@ enum Header {
1213 Missing { package: String },
1314}
1415
16+#[derive(Debug, StructOpt)]
17+#[structopt(
18+ name = "commenter",
19+ about = "Automates generation of bounds in build-constraints.yaml"
20+)]
21+enum Opt {
22+ Clear,
23+ Add,
24+}
25+
1526fn main() {
27+ let opt = Opt::from_args();
28+ match opt {
29+ Opt::Clear => clear(),
30+ Opt::Add => add(),
31+ }
32+}
33+
34+fn clear() {
35+ commenter::clear();
36+}
37+
38+fn add() {
1639 let mut lib_exes: H = Default::default();
1740 let mut tests: H = Default::default();
1841 let mut benches: H = Default::default();
@@ -61,12 +84,18 @@ fn main() {
6184 }
6285 }
6386
87+ let mut auto_lib_exes = vec![];
88+ let mut auto_tests = vec![];
89+ let mut auto_benches = vec![];
90+
6491 if !lib_exes.is_empty() {
6592 println!("\nLIBS + EXES\n");
6693 }
6794 for (header, packages) in lib_exes {
6895 for (package, version, component) in packages {
69− printer(" ", &package, true, &version, &component, &header);
96+ let s = printer(" ", &package, true, &version, &component, &header);
97+ println!("{}", s);
98+ auto_lib_exes.push(s);
7099 }
71100 }
72101
@@ -75,7 +104,9 @@ fn main() {
75104 }
76105 for (header, packages) in tests {
77106 for (package, version, component) in packages {
78− printer(" ", &package, false, &version, &component, &header);
107+ let s = printer(" ", &package, false, &version, &component, &header);
108+ println!("{}", s);
109+ auto_tests.push(s);
79110 }
80111 }
81112
@@ -84,9 +115,20 @@ fn main() {
84115 }
85116 for (header, packages) in benches {
86117 for (package, version, component) in packages {
87− printer(" ", &package, false, &version, &component, &header);
118+ let s = printer(" ", &package, false, &version, &component, &header);
119+ println!("{}", s);
120+ auto_benches.push(s);
88121 }
89122 }
123+
124+ println!();
125+ println!(
126+ "Adding {lib_exes} libs, {tests} tests, {benches} benches to build-constraints.yaml",
127+ lib_exes = auto_lib_exes.len(),
128+ tests = auto_tests.len(),
129+ benches = auto_benches.len()
130+ );
131+ commenter::add(auto_lib_exes, auto_tests, auto_benches);
90132}
91133
92134fn printer(
@@ -96,9 +138,9 @@ fn printer(
96138 version: &str,
97139 component: &str,
98140 header: &Header,
99−) {
141+) -> String {
100142 let lt0 = if lt0 { " < 0" } else { "" };
101− println!(
143+ format!(
102144 "{indent}- {package}{lt0} # tried {package}-{version}, but its *{component}* {cause}",
103145 indent = indent,
104146 package = package,
@@ -116,7 +158,7 @@ fn printer(
116158 package = package
117159 ),
118160 },
119− );
161+ )
120162}
121163
122164fn insert(h: &mut H, header: Header, package: &str, version: &str, component: &str) {