Extract flake.lock parsing from core flake checker logic

.editorconfig

@@ -11,3 +11,6 @@ insert_final_newline = true
 
 [*.rs]
 indent_size = 4
+
+[*.hbs]
+insert_final_newline = false

Cargo.lock

@@ -210,6 +210,7 @@ dependencies = [
  "clap",
  "handlebars",
  "is_ci",
+ "parse-flake-lock",
  "reqwest",
  "serde",
  "serde_json",

Cargo.toml

@@ -14,6 +14,7 @@ chrono = { version = "0.4.25", default-features = false, features = [ "clock" ]
 clap = { version = "4.3.0", default-features = false, features = [ "derive", "env", "std", "wrap_help" ] }
 handlebars = { version = "4.3.7", default-features = false }
 is_ci = "1.1.1"
+parse-flake-lock = { path = "./parse-flake-lock" }
 reqwest = { version = "0.11.18", default-features = false, features = [ "blocking", "rustls-tls-native-roots" ]}
 serde = { version = "1.0.163", features = [ "derive" ] }
 serde_json = { version = "1.0.96", default-features = false }

README.md

@@ -64,6 +64,36 @@ To disable diagnostic reporting, set the diagnostics URL to an empty string by p
 
 You can read the full privacy policy for [Determinate Systems][detsys], the creators of this tool and the [Determinate Nix Installer][installer], [here][privacy].
 
+## Rust library
+
+The Nix Flake Checker is written in [Rust].
+This repo exposes a [`parse-flake-lock`](./parse-flake-lock) crate that you can use to parse [`flake.lock` files][lockfile] in your own Rust projects.
+To add that dependency:
+
+```toml
+[dependencies]
+parse-flake-lock = { git = "https://github.com/DeterminateSystems/flake-checker", branch = "main" }
+```
+
+Here's an example usage:
+
+```rust
+use parse_flake_lock::{FlakeLock, ParseFlakeLockError};
+
+fn main() -> Result<(), ParseFlakeLockError> {
+    let flake_lock = FlakeLock::new(&flake_lock_path)?;
+    println!("flake.lock info:");
+    println!("version: {version}", version=flake_lock.version);
+    println!("root node: {root:?}", root=flake_lock.root);
+    println!("all nodes: {nodes:?}", nodes=flake_lock.nodes);
+
+    Ok(())
+}
+```
+
+The `parse-flake-lock` crate doesn't yet exhaustively parse all input node types, instead using a "fallthrough" mechanism that parses input types that don't yet have explicit struct definitions to a [`serde_json::value::Value`][val].
+If you'd like to help make the parser more exhaustive, [pull requests][prs] are quite welcome.
+
 [action]: https://github.com/DeterminateSystems/flake-checker-action
 [detsys]: https://determinate.systems
 [flakes]: https://zero-to-nix.com/concepts/flakes
@@ -74,4 +104,7 @@ You can read the full privacy policy for [Determinate Systems][detsys], the crea
 [nixos-org]: https://github.com/NixOS
 [nixpkgs]: https://github.com/NixOS/nixpkgs
 [privacy]: https://determinate.systems/privacy
+[prs]: /pulls
+[rust]: https://rust-lang.org
 [telemetry]: https://github.com/DeterminateSystems/nix-flake-checker/blob/main/src/telemetry.rs#L29-L43
+[val]: https://docs.rs/serde_json/latest/serde_json/value/enum.Value.html

parse-flake-lock/src/lib.rs

@@ -20,9 +20,9 @@ pub enum FlakeLockParseError {
 
 #[derive(Clone, Debug)]
 pub struct FlakeLock {
-    nodes: HashMap<String, Node>,
-    root: HashMap<String, Node>,
-    version: usize,
+    pub nodes: HashMap<String, Node>,
+    pub root: HashMap<String, Node>,
+    pub version: usize,
 }
 
 impl<'de> Deserialize<'de> for FlakeLock {

src/error.rs

@@ -2,6 +2,8 @@
 pub enum FlakeCheckerError {
     #[error("env var error: {0}")]
     EnvVar(#[from] std::env::VarError),
+    #[error("couldn't parse flake.lock: {0}")]
+    FlakeLock(#[from] parse_flake_lock::FlakeLockParseError),
     #[error("couldn't access flake.lock: {0}")]
     Io(#[from] std::io::Error),
     #[error("couldn't parse flake.lock: {0}")]

src/flake.rs

@@ -1,15 +1,12 @@
 #![allow(dead_code)]
-use std::collections::{HashMap, VecDeque};
-use std::fmt;
-use std::fs::read_to_string;
-use std::path::Path;
+
+use std::collections::HashMap;
 
 use crate::issue::{Disallowed, Issue, IssueKind, NonUpstream, Outdated};
 use crate::FlakeCheckerError;
 
 use chrono::{Duration, Utc};
-use serde::de::{self, Deserializer, MapAccess, Visitor};
-use serde::Deserialize;
+use parse_flake_lock::{FlakeLock, Node};
 
 // Update this when necessary by running the get-allowed-refs.sh script to fetch
 // the current values from monitoring.nixos.org
@@ -46,13 +43,41 @@ impl Default for FlakeCheckConfig {
     }
 }
 
+fn nixpkgs_deps(flake_lock: &FlakeLock, keys: Vec<String>) -> Result<HashMap<String, Node>, FlakeCheckerError> {
+    let mut deps: HashMap<String, Node> = HashMap::new();
+
+    for (key, node) in flake_lock.root.clone() {
+        if let Node::Repo(_) = node {
+            if keys.contains(&key) {
+                deps.insert(key, node);
+            }
+        }
+    }
+    let missing: Vec<String> = keys
+        .iter()
+        .filter(|k| !deps.contains_key(*k))
+        .map(String::from)
+        .collect();
+
+    if !missing.is_empty() {
+        let error_msg = format!(
+            "no nixpkgs dependency found for specified {}: {}",
+            if missing.len() > 1 { "keys" } else { "key" },
+            missing.join(", ")
+        );
+        return Err(FlakeCheckerError::Invalid(error_msg));
+    }
+
+    Ok(deps)
+}
+
 pub(crate) fn check_flake_lock(
     flake_lock: &FlakeLock,
     config: &FlakeCheckConfig,
 ) -> Result<Vec<Issue>, FlakeCheckerError> {
     let mut issues = vec![];
 
-    let deps = flake_lock.nixpkgs_deps(config.nixpkgs_keys.clone())?;
+    let deps = nixpkgs_deps(flake_lock, config.nixpkgs_keys.clone())?;
 
     for (name, dep) in deps {
         if let Node::Repo(repo) = dep {
@@ -99,247 +124,6 @@ pub(crate) fn check_flake_lock(
     Ok(issues)
 }
 
-#[derive(Clone, Debug)]
-pub(crate) struct FlakeLock {
-    nodes: HashMap<String, Node>,
-    root: HashMap<String, Node>,
-    version: usize,
-}
-
-impl<'de> Deserialize<'de> for FlakeLock {
-    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
-    where
-        D: Deserializer<'de>,
-    {
-        #[derive(Deserialize)]
-        #[serde(field_identifier, rename_all = "lowercase")]
-        enum Field {
-            Nodes,
-            Root,
-            Version,
-        }
-
-        struct FlakeLockVisitor;
-
-        impl<'de> Visitor<'de> for FlakeLockVisitor {
-            type Value = FlakeLock;
-
-            fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
-                formatter.write_str("struct FlakeLock")
-            }
-
-            fn visit_map<V>(self, mut map: V) -> Result<Self::Value, V::Error>
-            where
-                V: MapAccess<'de>,
-            {
-                let mut nodes = None;
-                let mut root = None;
-                let mut version = None;
-                while let Some(key) = map.next_key()? {
-                    match key {
-                        Field::Nodes => {
-                            if nodes.is_some() {
-                                return Err(de::Error::duplicate_field("nodes"));
-                            }
-                            nodes = Some(map.next_value()?);
-                        }
-                        Field::Root => {
-                            if root.is_some() {
-                                return Err(de::Error::duplicate_field("root"));
-                            }
-                            root = Some(map.next_value()?);
-                        }
-                        Field::Version => {
-                            if version.is_some() {
-                                return Err(de::Error::duplicate_field("version"));
-                            }
-                            version = Some(map.next_value()?);
-                        }
-                    }
-                }
-                let nodes: HashMap<String, Node> =
-                    nodes.ok_or_else(|| de::Error::missing_field("nodes"))?;
-                let root: String = root.ok_or_else(|| de::Error::missing_field("root"))?;
-                let version: usize = version.ok_or_else(|| de::Error::missing_field("version"))?;
-
-                let mut root_nodes = HashMap::new();
-                let root_node = &nodes[&root];
-                let Node::Root(root_node) = root_node else {
-                    return Err(de::Error::custom(format!("root node was not a Root node, but was a {} node", root_node.variant())));
-                };
-
-                for (root_name, root_input) in root_node.inputs.iter() {
-                    let inputs: VecDeque<String> = match root_input.clone() {
-                        Input::String(s) => [s].into(),
-                        Input::List(keys) => keys.into(),
-                    };
-
-                    let real_node = chase_input_node(&nodes, inputs).map_err(|e| {
-                        de::Error::custom(format!("failed to chase input {}: {:?}", root_name, e))
-                    })?;
-                    root_nodes.insert(root_name.clone(), real_node.clone());
-                }
-
-                Ok(FlakeLock {
-                    nodes,
-                    root: root_nodes,
-                    version,
-                })
-            }
-        }
-
-        deserializer.deserialize_any(FlakeLockVisitor)
-    }
-}
-
-fn chase_input_node(
-    nodes: &HashMap<String, Node>,
-    mut inputs: VecDeque<String>,
-) -> Result<&Node, FlakeCheckerError> {
-    let Some(next_input) = inputs.pop_front() else {
-        unreachable!("there should always be at least one input");
-    };
-
-    let mut node = &nodes[&next_input];
-    for input in inputs {
-        let maybe_node_inputs = match node {
-            Node::Repo(node) => node.inputs.to_owned(),
-            Node::Fallthrough(node) => match node.get("inputs") {
-                Some(node_inputs) => {
-                    serde_json::from_value(node_inputs.clone()).map_err(FlakeCheckerError::Json)?
-                }
-                None => None,
-            },
-            Node::Root(_) => None,
-        };
-
-        let node_inputs = match maybe_node_inputs {
-            Some(node_inputs) => node_inputs,
-            None => {
-                return Err(FlakeCheckerError::Invalid(format!(
-                    "lock node should have had some inputs but had none:\n{:?}",
-                    node
-                )));
-            }
-        };
-
-        let next_inputs = &node_inputs[&input];
-        node = match next_inputs {
-            Input::String(s) => &nodes[s],
-            Input::List(inputs) => chase_input_node(nodes, inputs.to_owned().into())?,
-        };
-    }
-
-    Ok(node)
-}
-
-impl FlakeLock {
-    pub(crate) fn new(path: &Path) -> Result<Self, FlakeCheckerError> {
-        let flake_lock_file = read_to_string(path)?;
-        let flake_lock: FlakeLock = serde_json::from_str(&flake_lock_file)?;
-        Ok(flake_lock)
-    }
-
-    fn nixpkgs_deps(&self, keys: Vec<String>) -> Result<HashMap<String, Node>, FlakeCheckerError> {
-        let mut deps: HashMap<String, Node> = HashMap::new();
-
-        for (key, node) in self.root.clone() {
-            if let Node::Repo(_) = node {
-                if keys.contains(&key) {
-                    deps.insert(key, node);
-                }
-            }
-        }
-        let missing: Vec<String> = keys
-            .iter()
-            .filter(|k| !deps.contains_key(*k))
-            .map(String::from)
-            .collect();
-
-        if !missing.is_empty() {
-            let error_msg = format!(
-                "no nixpkgs dependency found for specified {}: {}",
-                if missing.len() > 1 { "keys" } else { "key" },
-                missing.join(", ")
-            );
-            return Err(FlakeCheckerError::Invalid(error_msg));
-        }
-
-        Ok(deps)
-    }
-}
-
-#[derive(Clone, Debug, Deserialize)]
-#[serde(untagged)]
-pub(crate) enum Node {
-    Repo(Box<RepoNode>),
-    Root(RootNode),
-    Fallthrough(serde_json::value::Value), // Covers all other node types
-}
-
-impl Node {
-    fn variant(&self) -> &'static str {
-        match self {
-            Node::Root(_) => "Root",
-            Node::Repo(_) => "Repo",
-            Node::Fallthrough(_) => "Fallthrough", // Covers all other node types
-        }
-    }
-
-    fn is_nixpkgs(&self) -> bool {
-        match self {
-            Self::Repo(repo) => {
-                repo.locked.node_type == "github" && repo.original.repo == "nixpkgs"
-            }
-            _ => false,
-        }
-    }
-}
-
-#[derive(Clone, Debug, Deserialize)]
-#[serde(untagged)]
-pub(crate) enum Input {
-    String(String),
-    List(Vec<String>),
-}
-
-#[derive(Clone, Debug, Deserialize)]
-#[serde(deny_unknown_fields)]
-pub(crate) struct RootNode {
-    pub(crate) inputs: HashMap<String, Input>,
-}
-
-#[derive(Clone, Debug, Deserialize)]
-#[serde(deny_unknown_fields)]
-pub(crate) struct RepoNode {
-    pub(crate) inputs: Option<HashMap<String, Input>>,
-    pub(crate) locked: RepoLocked,
-    pub(crate) original: RepoOriginal,
-}
-
-#[derive(Clone, Debug, Deserialize)]
-pub(crate) struct RepoLocked {
-    #[serde(alias = "lastModified")]
-    pub(crate) last_modified: i64,
-    #[serde(alias = "narHash")]
-    pub(crate) nar_hash: String,
-    pub(crate) owner: String,
-    pub(crate) repo: String,
-    pub(crate) rev: String,
-    #[serde(alias = "type")]
-    pub(crate) node_type: String,
-}
-
-#[derive(Clone, Debug, Deserialize)]
-pub(crate) struct RepoOriginal {
-    pub(crate) owner: String,
-    pub(crate) repo: String,
-    #[serde(alias = "type")]
-    pub(crate) node_type: String,
-    #[serde(alias = "ref")]
-    pub(crate) git_ref: Option<String>,
-}
-
 #[cfg(test)]
 mod test {
     use std::path::PathBuf;

src/main.rs

@@ -1,3 +1,5 @@
+extern crate parse_flake_lock;
+
 mod error;
 mod flake;
 mod issue;
@@ -5,13 +7,14 @@ mod summary;
 mod telemetry;
 
 use error::FlakeCheckerError;
-use flake::{check_flake_lock, FlakeCheckConfig, FlakeLock};
+use flake::{check_flake_lock, FlakeCheckConfig};
 use summary::Summary;
 
 use std::path::PathBuf;
 use std::process::ExitCode;
 
 use clap::Parser;
+use parse_flake_lock::FlakeLock;
 
 /// A flake.lock checker for Nix projects.
 #[derive(Parser)]
@@ -19,7 +22,7 @@ use clap::Parser;
 struct Cli {
     /// Don't send aggregate sums of each issue type.
     ///
-    /// See: https://github.com/determinateSystems/flake-checker.
+    /// See <https://github.com/determinateSystems/flake-checker>.
     #[arg(long, env = "NIX_FLAKE_CHECKER_NO_TELEMETRY", default_value_t = false)]
     no_telemetry: bool,
 
@@ -109,6 +112,11 @@ fn main() -> Result<ExitCode, FlakeCheckerError> {
 
     let flake_lock = FlakeLock::new(&flake_lock_path)?;
 
+    println!("flake.lock info:");
+    println!("version: {version}", version=flake_lock.version);
+    println!("root node: {root:?}", root=flake_lock.root);
+    println!("all nodes: {nodes:?}", nodes=flake_lock.nodes);
+
     let flake_check_config = FlakeCheckConfig {
         check_supported,
         check_outdated,

src/templates/summary_md.hbs

@@ -110,4 +110,4 @@ While <a href="https://github.com/NixOS/nixpkgs">upstream Nixpkgs</a> isn't bull
 {{/if}}
 {{/if}}
 
-<p>Feedback? Let us know at <a href="https://github.com/DeterminateSystems/flake-checker">DeterminateSystems/flake-checker</a>.</p>
+<p>Feedback? Let us know at <a href="https://github.com/DeterminateSystems/flake-checker">DeterminateSystems/flake-checker</a>.</p>

src/templates/summary_txt.hbs

@@ -83,4 +83,4 @@ software is!) it has a wide range of security measures in place, most notably
 continuous integration testing with Hydra, that mitigate a great deal of supply
 chain risk.
 {{/if}}
-{{/if}}
+{{/if}}