on master help sign in
commenter: snapshot-diff
f702991Adam Bergmark
CURATORS.md+18
@@ -512,6 +512,24 @@ sections, or that are of the form `- package < 0 # $version`.
512
512
513* Please make sure to separate bounds issues from compilation failures/test run failures, as we cannot verify that a package builds or that tests pass without running the build!
513* Please make sure to separate bounds issues from compilation failures/test run failures, as we cannot verify that a package builds or that tests pass without running the build!
514
514
515+#### Diffing snapshots / Inspecting changes
516+
517+To diff existing snapshots, or to evaluate changes before they end up
518+in a snapshot you can run:
519+
520+```
521+commenter diff-snapshot <old-snapshot.yaml> <new-snapshot.yaml>
522+```
523+
524+Existing snapshots can be retrieved from https://github.com/commercialhaskell/stackage-snapshots. Preliminary snapshots can be generated by running relevant parts of `automated/build.sh`, at the time of writing:
525+
526+```
527+TARGET=nightly-2021-01-14 \ # the date doesn't matter
528+ curator update && \
529+ curator constraints --target $TARGET && \
530+ curator snapshot-incomplete --target $TARGET && \
531+ curator snapshot
532+```
515
533
516## Adding new curators
534## Adding new curators
517
535
etc/commenter/Cargo.lock+95−18
@@ -31,6 +31,12 @@ dependencies = [
31 "winapi",
31 "winapi",
32]
32]
33
33
34+[[package]]
35+name = "autocfg"
36+version = "1.0.1"
37+source = "registry+https://github.com/rust-lang/crates.io-index"
38+checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
39+
34[[package]]
40[[package]]
35name = "bitflags"
41name = "bitflags"
36version = "1.3.2"
42version = "1.3.2"
@@ -58,9 +64,17 @@ version = "0.2.0"
58dependencies = [
64dependencies = [
59 "lazy-regex",
65 "lazy-regex",
60 "regex",
66 "regex",
67+ "serde",
68+ "serde_yaml",
61 "structopt",
69 "structopt",
62]
70]
63
71
72+[[package]]
73+name = "hashbrown"
74+version = "0.11.2"
75+source = "registry+https://github.com/rust-lang/crates.io-index"
76+checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
77+
64[[package]]
78[[package]]
65name = "heck"
79name = "heck"
66version = "0.3.3"
80version = "0.3.3"
@@ -79,11 +93,21 @@ dependencies = [
79 "libc",
93 "libc",
80]
94]
81
95
96+[[package]]
97+name = "indexmap"
98+version = "1.8.0"
99+source = "registry+https://github.com/rust-lang/crates.io-index"
100+checksum = "282a6247722caba404c065016bbfa522806e51714c34f5dfc3e4a3a46fcb4223"
101+dependencies = [
102+ "autocfg",
103+ "hashbrown",
104+]
105+
82[[package]]
106[[package]]
83name = "lazy-regex"
107name = "lazy-regex"
84−version = "2.2.1"
108+version = "2.2.2"
85source = "registry+https://github.com/rust-lang/crates.io-index"
109source = "registry+https://github.com/rust-lang/crates.io-index"
86−checksum = "17d198f91272f6e788a5c0bd5d741cf778da4e5bc761ec67b32d5d3b0db34a54"
110+checksum = "919a16773ebf2de27e95fc58460110932e55bb0780e23aa51fa5a6b59c9e2b3d"
87dependencies = [
111dependencies = [
88 "lazy-regex-proc_macros",
112 "lazy-regex-proc_macros",
89 "once_cell",
113 "once_cell",
@@ -92,9 +116,9 @@ dependencies = [
92
116
93[[package]]
117[[package]]
94name = "lazy-regex-proc_macros"
118name = "lazy-regex-proc_macros"
95−version = "2.2.1"
119+version = "2.2.2"
96source = "registry+https://github.com/rust-lang/crates.io-index"
120source = "registry+https://github.com/rust-lang/crates.io-index"
97−checksum = "6c12938b1b92cf5be22940527e15b79fd0c7e706e34bc70816f6a72b3484f84e"
121+checksum = "5fbe6bf0a04af51c07976625d5007e75ed9b8b955befc21c77b3947733496e36"
98dependencies = [
122dependencies = [
99 "proc-macro2",
123 "proc-macro2",
100 "quote",
124 "quote",
@@ -110,21 +134,27 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
110
134
111[[package]]
135[[package]]
112name = "libc"
136name = "libc"
113−version = "0.2.108"
137+version = "0.2.112"
138+source = "registry+https://github.com/rust-lang/crates.io-index"
139+checksum = "1b03d17f364a3a042d5e5d46b053bbbf82c92c9430c592dd4c064dc6ee997125"
140+
141+[[package]]
142+name = "linked-hash-map"
143+version = "0.5.4"
114source = "registry+https://github.com/rust-lang/crates.io-index"
144source = "registry+https://github.com/rust-lang/crates.io-index"
115−checksum = "8521a1b57e76b1ec69af7599e75e38e7b7fad6610f037db8c79b127201b5d119"
145+checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3"
116
146
117[[package]]
147[[package]]
118name = "memchr"
148name = "memchr"
119−version = "2.4.0"
149+version = "2.4.1"
120source = "registry+https://github.com/rust-lang/crates.io-index"
150source = "registry+https://github.com/rust-lang/crates.io-index"
121−checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc"
151+checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"
122
152
123[[package]]
153[[package]]
124name = "once_cell"
154name = "once_cell"
125−version = "1.8.0"
155+version = "1.9.0"
126source = "registry+https://github.com/rust-lang/crates.io-index"
156source = "registry+https://github.com/rust-lang/crates.io-index"
127−checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56"
157+checksum = "da32515d9f6e6e489d7bc9d84c71b060db7247dc035bbe44eac88cf87486d8d5"
128
158
129[[package]]
159[[package]]
130name = "proc-macro-error"
160name = "proc-macro-error"
@@ -152,18 +182,18 @@ dependencies = [
152
182
153[[package]]
183[[package]]
154name = "proc-macro2"
184name = "proc-macro2"
155−version = "1.0.27"
185+version = "1.0.36"
156source = "registry+https://github.com/rust-lang/crates.io-index"
186source = "registry+https://github.com/rust-lang/crates.io-index"
157−checksum = "f0d8caf72986c1a598726adc988bb5984792ef84f5ee5aa50209145ee8077038"
187+checksum = "c7342d5883fbccae1cc37a2353b09c87c9b0f3afd73f5fb9bba687a1f733b029"
158dependencies = [
188dependencies = [
159 "unicode-xid",
189 "unicode-xid",
160]
190]
161
191
162[[package]]
192[[package]]
163name = "quote"
193name = "quote"
164−version = "1.0.9"
194+version = "1.0.14"
165source = "registry+https://github.com/rust-lang/crates.io-index"
195source = "registry+https://github.com/rust-lang/crates.io-index"
166−checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7"
196+checksum = "47aa80447ce4daf1717500037052af176af5d38cc3e571d9ec1c7353fc10c87d"
167dependencies = [
197dependencies = [
168 "proc-macro2",
198 "proc-macro2",
169]
199]
@@ -185,6 +215,44 @@ version = "0.6.25"
185source = "registry+https://github.com/rust-lang/crates.io-index"
215source = "registry+https://github.com/rust-lang/crates.io-index"
186checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b"
216checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b"
187
217
218+[[package]]
219+name = "ryu"
220+version = "1.0.9"
221+source = "registry+https://github.com/rust-lang/crates.io-index"
222+checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f"
223+
224+[[package]]
225+name = "serde"
226+version = "1.0.133"
227+source = "registry+https://github.com/rust-lang/crates.io-index"
228+checksum = "97565067517b60e2d1ea8b268e59ce036de907ac523ad83a0475da04e818989a"
229+dependencies = [
230+ "serde_derive",
231+]
232+
233+[[package]]
234+name = "serde_derive"
235+version = "1.0.133"
236+source = "registry+https://github.com/rust-lang/crates.io-index"
237+checksum = "ed201699328568d8d08208fdd080e3ff594e6c422e438b6705905da01005d537"
238+dependencies = [
239+ "proc-macro2",
240+ "quote",
241+ "syn",
242+]
243+
244+[[package]]
245+name = "serde_yaml"
246+version = "0.8.23"
247+source = "registry+https://github.com/rust-lang/crates.io-index"
248+checksum = "a4a521f2940385c165a24ee286aa8599633d162077a54bdcae2a6fd5a7bfa7a0"
249+dependencies = [
250+ "indexmap",
251+ "ryu",
252+ "serde",
253+ "yaml-rust",
254+]
255+
188[[package]]
256[[package]]
189name = "strsim"
257name = "strsim"
190version = "0.8.0"
258version = "0.8.0"
@@ -217,9 +285,9 @@ dependencies = [
217
285
218[[package]]
286[[package]]
219name = "syn"
287name = "syn"
220−version = "1.0.73"
288+version = "1.0.85"
221source = "registry+https://github.com/rust-lang/crates.io-index"
289source = "registry+https://github.com/rust-lang/crates.io-index"
222−checksum = "f71489ff30030d2ae598524f61326b902466f72a0fb1a8564c001cc63425bcc7"
290+checksum = "a684ac3dcd8913827e18cd09a68384ee66c1de24157e3c556c9ab16d85695fb7"
223dependencies = [
291dependencies = [
224 "proc-macro2",
292 "proc-macro2",
225 "quote",
293 "quote",
@@ -261,9 +329,9 @@ checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"
261
329
262[[package]]
330[[package]]
263name = "version_check"
331name = "version_check"
264−version = "0.9.3"
332+version = "0.9.4"
265source = "registry+https://github.com/rust-lang/crates.io-index"
333source = "registry+https://github.com/rust-lang/crates.io-index"
266−checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe"
334+checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
267
335
268[[package]]
336[[package]]
269name = "winapi"
337name = "winapi"
@@ -286,3 +354,12 @@ name = "winapi-x86_64-pc-windows-gnu"
286version = "0.4.0"
354version = "0.4.0"
287source = "registry+https://github.com/rust-lang/crates.io-index"
355source = "registry+https://github.com/rust-lang/crates.io-index"
288checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
356checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
357+
358+[[package]]
359+name = "yaml-rust"
360+version = "0.4.5"
361+source = "registry+https://github.com/rust-lang/crates.io-index"
362+checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85"
363+dependencies = [
364+ "linked-hash-map",
365+]
etc/commenter/Cargo.toml+2
@@ -9,4 +9,6 @@ edition = "2018"
9[dependencies]
9[dependencies]
10lazy-regex = "2.2.1"
10lazy-regex = "2.2.1"
11regex = "1.5.4"
11regex = "1.5.4"
12+serde = { version = "1.0.133", features = ["derive"] }
13+serde_yaml = "0.8.23"
12structopt = "0.3.25"
14structopt = "0.3.25"
etc/commenter/src/lib.rs+130−9
@@ -1,10 +1,13 @@
1use std::collections::{BTreeMap, BTreeSet};
1use std::collections::{BTreeMap, BTreeSet};
2+use std::error::Error;
3+use std::fmt;
2use std::fs::File;
4use std::fs::File;
3use std::io::{BufRead, BufReader, LineWriter, Lines, Write};
5use std::io::{BufRead, BufReader, LineWriter, Lines, Write};
4use std::path::Path;
6use std::path::Path;
5use std::process::Command;
7use std::process::Command;
6
8
7use lazy_regex::regex;
9use lazy_regex::regex;
10+use serde::{Deserialize, Deserializer};
8
11
9pub fn clear() {
12pub fn clear() {
10 handle(true, |loc, _lines| match loc {
13 handle(true, |loc, _lines| match loc {
@@ -86,11 +89,9 @@ pub fn outdated() {
86 let latest = latest_version(&package);
89 let latest = latest_version(&package);
87 if version.version() != latest {
90 if version.version() != latest {
88 println!(
91 println!(
89− "{} mismatch, {}: {}, hackage: {}",
92+ "{package} mismatch, {tag}: {version}, hackage: {latest}",
90− package,
93+ tag = version.tag(),
91− version.tag(),
94+ version = version.version(),
92− version.version(),
93− latest
94 );
95 );
95 }
96 }
96 }
97 }
@@ -111,18 +112,17 @@ pub fn outdated() {
111 let dependents = dependents
112 let dependents = dependents
112 .into_iter()
113 .into_iter()
113 .take(max)
114 .take(max)
114− .map(|(p, v)| format!("{}-{}", p, v))
115+ .map(|(p, v)| format!("{p}-{v}"))
115 .collect::<Vec<String>>()
116 .collect::<Vec<String>>()
116 .join(", ");
117 .join(", ");
117 let dependents = if dependents_stripped > 0 {
118 let dependents = if dependents_stripped > 0 {
118− format!("{} and {} more", dependents, dependents_stripped)
119+ format!("{dependents} and {dependents_stripped} more")
119 } else {
120 } else {
120 dependents
121 dependents
121 };
122 };
122
123
123 println!(
124 println!(
124− "{} mismatch, snapshot: {}, hackage: {}, dependents: {}",
125+ "{package} mismatch, snapshot: {version}, hackage: {latest}, dependents: {dependents}"
125− package, version, latest, dependents,
126 );
126 );
127 }
127 }
128 }
128 }
@@ -295,3 +295,124 @@ where
295 let file = File::open(filename).unwrap();
295 let file = File::open(filename).unwrap();
296 BufReader::new(file).lines()
296 BufReader::new(file).lines()
297}
297}
298+
299+#[derive(Deserialize)]
300+struct SnapshotYaml {
301+ // flags: BTreeMap<PackageName, BTreeMap<PackageFlag, bool>>,
302+ // publish_time
303+ packages: Vec<SnapshotPackage>,
304+ // hidden
305+ // resolver
306+}
307+
308+#[derive(Deserialize)]
309+struct SnapshotPackage {
310+ hackage: PackageWithVersionAndSha,
311+ // pantry-tree
312+}
313+
314+#[derive(PartialOrd, Ord, PartialEq, Eq, Clone)]
315+struct PackageName(String);
316+
317+impl fmt::Display for PackageName {
318+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
319+ self.0.fmt(f)
320+ }
321+}
322+
323+#[derive(Clone, PartialOrd, Ord, PartialEq, Eq)]
324+struct Version(String);
325+
326+impl fmt::Display for Version {
327+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
328+ self.0.fmt(f)
329+ }
330+}
331+
332+// zstd-0.1.3.0@sha256:4c0a372251068eb6086b8c3a0a9f347488f08b570a7705844ffeb2c720c97223,3723
333+struct PackageWithVersionAndSha {
334+ name: PackageName,
335+ version: Version,
336+}
337+
338+impl<'de> serde::Deserialize<'de> for PackageWithVersionAndSha {
339+ fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
340+ where
341+ D: Deserializer<'de>,
342+ {
343+ let s: String = String::deserialize(deserializer)?;
344+ let r = regex!(r#"^(.+?)-([.\d]+)@sha256:[\da-z]+,\d+$"#);
345+ if let Some(caps) = r.captures(&s) {
346+ let name = PackageName(caps.get(1).unwrap().as_str().to_owned());
347+ let version = Version(caps.get(2).unwrap().as_str().to_owned());
348+ Ok(Self { name, version })
349+ } else {
350+ Err(serde::de::Error::invalid_value(
351+ serde::de::Unexpected::Other(&s),
352+ &"Invalid PackageVersionWithSha",
353+ ))
354+ }
355+ }
356+}
357+
358+fn yaml_from_file<A, P: AsRef<Path>>(path: P) -> Result<A, Box<dyn Error>>
359+where
360+ A: for<'de> Deserialize<'de>,
361+{
362+ let file = File::open(path)?;
363+ let reader = BufReader::new(file);
364+ let u = serde_yaml::from_reader(reader)?;
365+ Ok(u)
366+}
367+
368+struct Snapshot {
369+ packages: BTreeMap<PackageName, Diff<Version>>,
370+}
371+
372+#[derive(Clone, Copy)]
373+enum Diff<A> {
374+ Left(A),
375+ Right(A),
376+ Both(A, A),
377+}
378+
379+fn to_diff(a: SnapshotYaml, b: SnapshotYaml) -> Snapshot {
380+ let mut packages = BTreeMap::new();
381+ for s in a.packages {
382+ let package = s.hackage;
383+ packages.insert(package.name, Diff::Left(package.version));
384+ }
385+ for s in b.packages {
386+ let package = s.hackage;
387+ let name = package.name;
388+ let version = package.version;
389+ if let Some(a) = packages.remove(&name) {
390+ match a {
391+ Diff::Left(a) => {
392+ if a == version {
393+ packages.remove(&name);
394+ } else {
395+ packages.insert(name, Diff::Both(a, version));
396+ }
397+ }
398+ _ => unreachable!(),
399+ }
400+ } else {
401+ packages.insert(name, Diff::Right(version));
402+ }
403+ }
404+
405+ Snapshot { packages }
406+}
407+
408+pub fn diff_snapshot(a: String, b: String) {
409+ let diff = to_diff(yaml_from_file(a).unwrap(), yaml_from_file(b).unwrap());
410+ for (name, diff) in diff.packages {
411+ let s = match diff {
412+ Diff::Left(a) => format!("- {name}-{a}"),
413+ Diff::Right(b) => format!("+ {name}-{b}"),
414+ Diff::Both(a, b) => format!("~ {name}-{a} -> {b}"),
415+ };
416+ println!("{s}");
417+ }
418+}
etc/commenter/src/main.rs+8−19
@@ -22,25 +22,19 @@ enum Opt {
22 Clear,
22 Clear,
23 Add,
23 Add,
24 Outdated,
24 Outdated,
25+ DiffSnapshot { a: String, b: String },
25}
26}
26
27
27fn main() {
28fn main() {
28 let opt = Opt::from_args();
29 let opt = Opt::from_args();
29 match opt {
30 match opt {
30− Opt::Clear => clear(),
31+ Opt::Clear => commenter::clear(),
31 Opt::Add => add(),
32 Opt::Add => add(),
32− Opt::Outdated => outdated(),
33+ Opt::Outdated => commenter::outdated(),
34+ Opt::DiffSnapshot { a, b } => commenter::diff_snapshot(a, b),
33 }
35 }
34}
36}
35
37
36−fn clear() {
37− commenter::clear();
38−}
39−
40−fn outdated() {
41− commenter::outdated();
42−}
43−
44fn add() {
38fn add() {
45 let mut lib_exes: H = Default::default();
39 let mut lib_exes: H = Default::default();
46 let mut tests: H = Default::default();
40 let mut tests: H = Default::default();
@@ -64,7 +58,7 @@ fn add() {
64 } else if line == "curator: Snapshot dependency graph contains errors:" {
58 } else if line == "curator: Snapshot dependency graph contains errors:" {
65 process_line = true;
59 process_line = true;
66 } else if !process_line {
60 } else if !process_line {
67− println!("[INFO] {}", line);
61+ println!("[INFO] {line}");
68 } else if let Some(cap) = package.captures(&line) {
62 } else if let Some(cap) = package.captures(&line) {
69 let root = last_header.clone().unwrap();
63 let root = last_header.clone().unwrap();
70 let package = cap.name("package").unwrap().as_str();
64 let package = cap.name("package").unwrap().as_str();
@@ -100,7 +94,7 @@ fn add() {
100 for (header, packages) in lib_exes {
94 for (header, packages) in lib_exes {
101 for (package, version, component) in packages {
95 for (package, version, component) in packages {
102 let s = printer(" ", &package, true, &version, &component, &header);
96 let s = printer(" ", &package, true, &version, &component, &header);
103− println!("{}", s);
97+ println!("{s}");
104 auto_lib_exes.push(s);
98 auto_lib_exes.push(s);
105 }
99 }
106 }
100 }
@@ -111,7 +105,7 @@ fn add() {
111 for (header, packages) in tests {
105 for (header, packages) in tests {
112 for (package, version, component) in packages {
106 for (package, version, component) in packages {
113 let s = printer(" ", &package, false, &version, &component, &header);
107 let s = printer(" ", &package, false, &version, &component, &header);
114− println!("{}", s);
108+ println!("{s}");
115 auto_tests.push(s);
109 auto_tests.push(s);
116 }
110 }
117 }
111 }
@@ -122,7 +116,7 @@ fn add() {
122 for (header, packages) in benches {
116 for (header, packages) in benches {
123 for (package, version, component) in packages {
117 for (package, version, component) in packages {
124 let s = printer(" ", &package, false, &version, &component, &header);
118 let s = printer(" ", &package, false, &version, &component, &header);
125− println!("{}", s);
119+ println!("{s}");
126 auto_benches.push(s);
120 auto_benches.push(s);
127 }
121 }
128 }
122 }
@@ -148,11 +142,6 @@ fn printer(
148 let lt0 = if lt0 { " < 0" } else { "" };
142 let lt0 = if lt0 { " < 0" } else { "" };
149 format!(
143 format!(
150 "{indent}- {package}{lt0} # tried {package}-{version}, but its *{component}* {cause}",
144 "{indent}- {package}{lt0} # tried {package}-{version}, but its *{component}* {cause}",
151− indent = indent,
152− package = package,
153− lt0 = lt0,
154− version = version,
155− component = component,
156 cause = match header {
145 cause = match header {
157 Header::Versioned { package, version } => format!(
146 Header::Versioned { package, version } => format!(
158 "does not support: {package}-{version}",
147 "does not support: {package}-{version}",