on master help sign in
Merge pull request #6404 from commercialhaskell/snapshot-diff

commenter: snapshot-diff

commitb38fb534ddf7a1a738d4baa8a3bf1a9578ad1d18 authorAdam Bergmark <adam@bergmark.nl> dateSat 15 Jan 2022, 00:22 (5 years ago) parent7a404a3f702991
5 files changed+253−46
CURATORS.md+18 etc/commenter/Cargo.lock+95−18 etc/commenter/Cargo.toml+2 etc/commenter/src/lib.rs+130−9 etc/commenter/src/main.rs+8−19
CURATORS.md+18
@@ -512,6 +512,24 @@ sections, or that are of the form `- package < 0 # $version`.
512512
513513* Please make sure to separate bounds issues from compilation failures/test run failures, as we cannot verify that a package builds or that tests pass without running the build!
514514
515+#### Diffing snapshots / Inspecting changes
516+
517+To diff existing snapshots, or to evaluate changes before they end up
518+in a snapshot you can run:
519+
520+```
521+commenter diff-snapshot <old-snapshot.yaml> <new-snapshot.yaml>
522+```
523+
524+Existing snapshots can be retrieved from https://github.com/commercialhaskell/stackage-snapshots. Preliminary snapshots can be generated by running relevant parts of `automated/build.sh`, at the time of writing:
525+
526+```
527+TARGET=nightly-2021-01-14 \ # the date doesn't matter
528+ curator update && \
529+ curator constraints --target $TARGET && \
530+ curator snapshot-incomplete --target $TARGET && \
531+ curator snapshot
532+```
515533
516534## Adding new curators
517535
etc/commenter/Cargo.lock+95−18
@@ -31,6 +31,12 @@ dependencies = [
3131 "winapi",
3232]
3333
34+[[package]]
35+name = "autocfg"
36+version = "1.0.1"
37+source = "registry+https://github.com/rust-lang/crates.io-index"
38+checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
39+
3440[[package]]
3541name = "bitflags"
3642version = "1.3.2"
@@ -58,9 +64,17 @@ version = "0.2.0"
5864dependencies = [
5965 "lazy-regex",
6066 "regex",
67+ "serde",
68+ "serde_yaml",
6169 "structopt",
6270]
6371
72+[[package]]
73+name = "hashbrown"
74+version = "0.11.2"
75+source = "registry+https://github.com/rust-lang/crates.io-index"
76+checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
77+
6478[[package]]
6579name = "heck"
6680version = "0.3.3"
@@ -79,11 +93,21 @@ dependencies = [
7993 "libc",
8094]
8195
96+[[package]]
97+name = "indexmap"
98+version = "1.8.0"
99+source = "registry+https://github.com/rust-lang/crates.io-index"
100+checksum = "282a6247722caba404c065016bbfa522806e51714c34f5dfc3e4a3a46fcb4223"
101+dependencies = [
102+ "autocfg",
103+ "hashbrown",
104+]
105+
82106[[package]]
83107name = "lazy-regex"
84−version = "2.2.1"
108+version = "2.2.2"
85109source = "registry+https://github.com/rust-lang/crates.io-index"
86−checksum = "17d198f91272f6e788a5c0bd5d741cf778da4e5bc761ec67b32d5d3b0db34a54"
110+checksum = "919a16773ebf2de27e95fc58460110932e55bb0780e23aa51fa5a6b59c9e2b3d"
87111dependencies = [
88112 "lazy-regex-proc_macros",
89113 "once_cell",
@@ -92,9 +116,9 @@ dependencies = [
92116
93117[[package]]
94118name = "lazy-regex-proc_macros"
95−version = "2.2.1"
119+version = "2.2.2"
96120source = "registry+https://github.com/rust-lang/crates.io-index"
97−checksum = "6c12938b1b92cf5be22940527e15b79fd0c7e706e34bc70816f6a72b3484f84e"
121+checksum = "5fbe6bf0a04af51c07976625d5007e75ed9b8b955befc21c77b3947733496e36"
98122dependencies = [
99123 "proc-macro2",
100124 "quote",
@@ -110,21 +134,27 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
110134
111135[[package]]
112136name = "libc"
113−version = "0.2.108"
137+version = "0.2.112"
138+source = "registry+https://github.com/rust-lang/crates.io-index"
139+checksum = "1b03d17f364a3a042d5e5d46b053bbbf82c92c9430c592dd4c064dc6ee997125"
140+
141+[[package]]
142+name = "linked-hash-map"
143+version = "0.5.4"
114144source = "registry+https://github.com/rust-lang/crates.io-index"
115−checksum = "8521a1b57e76b1ec69af7599e75e38e7b7fad6610f037db8c79b127201b5d119"
145+checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3"
116146
117147[[package]]
118148name = "memchr"
119−version = "2.4.0"
149+version = "2.4.1"
120150source = "registry+https://github.com/rust-lang/crates.io-index"
121−checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc"
151+checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"
122152
123153[[package]]
124154name = "once_cell"
125−version = "1.8.0"
155+version = "1.9.0"
126156source = "registry+https://github.com/rust-lang/crates.io-index"
127−checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56"
157+checksum = "da32515d9f6e6e489d7bc9d84c71b060db7247dc035bbe44eac88cf87486d8d5"
128158
129159[[package]]
130160name = "proc-macro-error"
@@ -152,18 +182,18 @@ dependencies = [
152182
153183[[package]]
154184name = "proc-macro2"
155−version = "1.0.27"
185+version = "1.0.36"
156186source = "registry+https://github.com/rust-lang/crates.io-index"
157−checksum = "f0d8caf72986c1a598726adc988bb5984792ef84f5ee5aa50209145ee8077038"
187+checksum = "c7342d5883fbccae1cc37a2353b09c87c9b0f3afd73f5fb9bba687a1f733b029"
158188dependencies = [
159189 "unicode-xid",
160190]
161191
162192[[package]]
163193name = "quote"
164−version = "1.0.9"
194+version = "1.0.14"
165195source = "registry+https://github.com/rust-lang/crates.io-index"
166−checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7"
196+checksum = "47aa80447ce4daf1717500037052af176af5d38cc3e571d9ec1c7353fc10c87d"
167197dependencies = [
168198 "proc-macro2",
169199]
@@ -185,6 +215,44 @@ version = "0.6.25"
185215source = "registry+https://github.com/rust-lang/crates.io-index"
186216checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b"
187217
218+[[package]]
219+name = "ryu"
220+version = "1.0.9"
221+source = "registry+https://github.com/rust-lang/crates.io-index"
222+checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f"
223+
224+[[package]]
225+name = "serde"
226+version = "1.0.133"
227+source = "registry+https://github.com/rust-lang/crates.io-index"
228+checksum = "97565067517b60e2d1ea8b268e59ce036de907ac523ad83a0475da04e818989a"
229+dependencies = [
230+ "serde_derive",
231+]
232+
233+[[package]]
234+name = "serde_derive"
235+version = "1.0.133"
236+source = "registry+https://github.com/rust-lang/crates.io-index"
237+checksum = "ed201699328568d8d08208fdd080e3ff594e6c422e438b6705905da01005d537"
238+dependencies = [
239+ "proc-macro2",
240+ "quote",
241+ "syn",
242+]
243+
244+[[package]]
245+name = "serde_yaml"
246+version = "0.8.23"
247+source = "registry+https://github.com/rust-lang/crates.io-index"
248+checksum = "a4a521f2940385c165a24ee286aa8599633d162077a54bdcae2a6fd5a7bfa7a0"
249+dependencies = [
250+ "indexmap",
251+ "ryu",
252+ "serde",
253+ "yaml-rust",
254+]
255+
188256[[package]]
189257name = "strsim"
190258version = "0.8.0"
@@ -217,9 +285,9 @@ dependencies = [
217285
218286[[package]]
219287name = "syn"
220−version = "1.0.73"
288+version = "1.0.85"
221289source = "registry+https://github.com/rust-lang/crates.io-index"
222−checksum = "f71489ff30030d2ae598524f61326b902466f72a0fb1a8564c001cc63425bcc7"
290+checksum = "a684ac3dcd8913827e18cd09a68384ee66c1de24157e3c556c9ab16d85695fb7"
223291dependencies = [
224292 "proc-macro2",
225293 "quote",
@@ -261,9 +329,9 @@ checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"
261329
262330[[package]]
263331name = "version_check"
264−version = "0.9.3"
332+version = "0.9.4"
265333source = "registry+https://github.com/rust-lang/crates.io-index"
266−checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe"
334+checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
267335
268336[[package]]
269337name = "winapi"
@@ -286,3 +354,12 @@ name = "winapi-x86_64-pc-windows-gnu"
286354version = "0.4.0"
287355source = "registry+https://github.com/rust-lang/crates.io-index"
288356checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
357+
358+[[package]]
359+name = "yaml-rust"
360+version = "0.4.5"
361+source = "registry+https://github.com/rust-lang/crates.io-index"
362+checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85"
363+dependencies = [
364+ "linked-hash-map",
365+]
etc/commenter/Cargo.toml+2
@@ -9,4 +9,6 @@ edition = "2018"
99[dependencies]
1010lazy-regex = "2.2.1"
1111regex = "1.5.4"
12+serde = { version = "1.0.133", features = ["derive"] }
13+serde_yaml = "0.8.23"
1214structopt = "0.3.25"
etc/commenter/src/lib.rs+130−9
@@ -1,10 +1,13 @@
11use std::collections::{BTreeMap, BTreeSet};
2+use std::error::Error;
3+use std::fmt;
24use std::fs::File;
35use std::io::{BufRead, BufReader, LineWriter, Lines, Write};
46use std::path::Path;
57use std::process::Command;
68
79use lazy_regex::regex;
10+use serde::{Deserialize, Deserializer};
811
912pub fn clear() {
1013 handle(true, |loc, _lines| match loc {
@@ -86,11 +89,9 @@ pub fn outdated() {
8689 let latest = latest_version(&package);
8790 if version.version() != latest {
8891 println!(
89− "{} mismatch, {}: {}, hackage: {}",
90− package,
91− version.tag(),
92− version.version(),
93− latest
92+ "{package} mismatch, {tag}: {version}, hackage: {latest}",
93+ tag = version.tag(),
94+ version = version.version(),
9495 );
9596 }
9697 }
@@ -111,18 +112,17 @@ pub fn outdated() {
111112 let dependents = dependents
112113 .into_iter()
113114 .take(max)
114− .map(|(p, v)| format!("{}-{}", p, v))
115+ .map(|(p, v)| format!("{p}-{v}"))
115116 .collect::<Vec<String>>()
116117 .join(", ");
117118 let dependents = if dependents_stripped > 0 {
118− format!("{} and {} more", dependents, dependents_stripped)
119+ format!("{dependents} and {dependents_stripped} more")
119120 } else {
120121 dependents
121122 };
122123
123124 println!(
124− "{} mismatch, snapshot: {}, hackage: {}, dependents: {}",
125− package, version, latest, dependents,
125+ "{package} mismatch, snapshot: {version}, hackage: {latest}, dependents: {dependents}"
126126 );
127127 }
128128 }
@@ -295,3 +295,124 @@ where
295295 let file = File::open(filename).unwrap();
296296 BufReader::new(file).lines()
297297}
298+
299+#[derive(Deserialize)]
300+struct SnapshotYaml {
301+ // flags: BTreeMap<PackageName, BTreeMap<PackageFlag, bool>>,
302+ // publish_time
303+ packages: Vec<SnapshotPackage>,
304+ // hidden
305+ // resolver
306+}
307+
308+#[derive(Deserialize)]
309+struct SnapshotPackage {
310+ hackage: PackageWithVersionAndSha,
311+ // pantry-tree
312+}
313+
314+#[derive(PartialOrd, Ord, PartialEq, Eq, Clone)]
315+struct PackageName(String);
316+
317+impl fmt::Display for PackageName {
318+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
319+ self.0.fmt(f)
320+ }
321+}
322+
323+#[derive(Clone, PartialOrd, Ord, PartialEq, Eq)]
324+struct Version(String);
325+
326+impl fmt::Display for Version {
327+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
328+ self.0.fmt(f)
329+ }
330+}
331+
332+// zstd-0.1.3.0@sha256:4c0a372251068eb6086b8c3a0a9f347488f08b570a7705844ffeb2c720c97223,3723
333+struct PackageWithVersionAndSha {
334+ name: PackageName,
335+ version: Version,
336+}
337+
338+impl<'de> serde::Deserialize<'de> for PackageWithVersionAndSha {
339+ fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
340+ where
341+ D: Deserializer<'de>,
342+ {
343+ let s: String = String::deserialize(deserializer)?;
344+ let r = regex!(r#"^(.+?)-([.\d]+)@sha256:[\da-z]+,\d+$"#);
345+ if let Some(caps) = r.captures(&s) {
346+ let name = PackageName(caps.get(1).unwrap().as_str().to_owned());
347+ let version = Version(caps.get(2).unwrap().as_str().to_owned());
348+ Ok(Self { name, version })
349+ } else {
350+ Err(serde::de::Error::invalid_value(
351+ serde::de::Unexpected::Other(&s),
352+ &"Invalid PackageVersionWithSha",
353+ ))
354+ }
355+ }
356+}
357+
358+fn yaml_from_file<A, P: AsRef<Path>>(path: P) -> Result<A, Box<dyn Error>>
359+where
360+ A: for<'de> Deserialize<'de>,
361+{
362+ let file = File::open(path)?;
363+ let reader = BufReader::new(file);
364+ let u = serde_yaml::from_reader(reader)?;
365+ Ok(u)
366+}
367+
368+struct Snapshot {
369+ packages: BTreeMap<PackageName, Diff<Version>>,
370+}
371+
372+#[derive(Clone, Copy)]
373+enum Diff<A> {
374+ Left(A),
375+ Right(A),
376+ Both(A, A),
377+}
378+
379+fn to_diff(a: SnapshotYaml, b: SnapshotYaml) -> Snapshot {
380+ let mut packages = BTreeMap::new();
381+ for s in a.packages {
382+ let package = s.hackage;
383+ packages.insert(package.name, Diff::Left(package.version));
384+ }
385+ for s in b.packages {
386+ let package = s.hackage;
387+ let name = package.name;
388+ let version = package.version;
389+ if let Some(a) = packages.remove(&name) {
390+ match a {
391+ Diff::Left(a) => {
392+ if a == version {
393+ packages.remove(&name);
394+ } else {
395+ packages.insert(name, Diff::Both(a, version));
396+ }
397+ }
398+ _ => unreachable!(),
399+ }
400+ } else {
401+ packages.insert(name, Diff::Right(version));
402+ }
403+ }
404+
405+ Snapshot { packages }
406+}
407+
408+pub fn diff_snapshot(a: String, b: String) {
409+ let diff = to_diff(yaml_from_file(a).unwrap(), yaml_from_file(b).unwrap());
410+ for (name, diff) in diff.packages {
411+ let s = match diff {
412+ Diff::Left(a) => format!("- {name}-{a}"),
413+ Diff::Right(b) => format!("+ {name}-{b}"),
414+ Diff::Both(a, b) => format!("~ {name}-{a} -> {b}"),
415+ };
416+ println!("{s}");
417+ }
418+}
etc/commenter/src/main.rs+8−19
@@ -22,25 +22,19 @@ enum Opt {
2222 Clear,
2323 Add,
2424 Outdated,
25+ DiffSnapshot { a: String, b: String },
2526}
2627
2728fn main() {
2829 let opt = Opt::from_args();
2930 match opt {
30− Opt::Clear => clear(),
31+ Opt::Clear => commenter::clear(),
3132 Opt::Add => add(),
32− Opt::Outdated => outdated(),
33+ Opt::Outdated => commenter::outdated(),
34+ Opt::DiffSnapshot { a, b } => commenter::diff_snapshot(a, b),
3335 }
3436}
3537
36−fn clear() {
37− commenter::clear();
38−}
39−
40−fn outdated() {
41− commenter::outdated();
42−}
43−
4438fn add() {
4539 let mut lib_exes: H = Default::default();
4640 let mut tests: H = Default::default();
@@ -64,7 +58,7 @@ fn add() {
6458 } else if line == "curator: Snapshot dependency graph contains errors:" {
6559 process_line = true;
6660 } else if !process_line {
67− println!("[INFO] {}", line);
61+ println!("[INFO] {line}");
6862 } else if let Some(cap) = package.captures(&line) {
6963 let root = last_header.clone().unwrap();
7064 let package = cap.name("package").unwrap().as_str();
@@ -100,7 +94,7 @@ fn add() {
10094 for (header, packages) in lib_exes {
10195 for (package, version, component) in packages {
10296 let s = printer(" ", &package, true, &version, &component, &header);
103− println!("{}", s);
97+ println!("{s}");
10498 auto_lib_exes.push(s);
10599 }
106100 }
@@ -111,7 +105,7 @@ fn add() {
111105 for (header, packages) in tests {
112106 for (package, version, component) in packages {
113107 let s = printer(" ", &package, false, &version, &component, &header);
114− println!("{}", s);
108+ println!("{s}");
115109 auto_tests.push(s);
116110 }
117111 }
@@ -122,7 +116,7 @@ fn add() {
122116 for (header, packages) in benches {
123117 for (package, version, component) in packages {
124118 let s = printer(" ", &package, false, &version, &component, &header);
125− println!("{}", s);
119+ println!("{s}");
126120 auto_benches.push(s);
127121 }
128122 }
@@ -148,11 +142,6 @@ fn printer(
148142 let lt0 = if lt0 { " < 0" } else { "" };
149143 format!(
150144 "{indent}- {package}{lt0} # tried {package}-{version}, but its *{component}* {cause}",
151− indent = indent,
152− package = package,
153− lt0 = lt0,
154− version = version,
155− component = component,
156145 cause = match header {
157146 Header::Versioned { package, version } => format!(
158147 "does not support: {package}-{version}",