rust: check github versions

rust/migrations/2023-08-12-152848_create_packages/up.sql

@@ -1,27 +1,31 @@
 CREATE TABLE packages (
     id TEXT PRIMARY KEY NOT NULL
   , attr_path TEXT NOT NULL
+  , last_update_attempt DATETIME
+  , last_update_log DATETIME
   , version_nixpkgs_master TEXT
+  , last_checked_nixpkgs_master DATETIME
   , version_nixpkgs_staging TEXT
+  , last_checked_nixpkgs_staging DATETIME
   , version_nixpkgs_staging_next TEXT
+  , last_checked_nixpkgs_staging_next DATETIME
   , version_repology TEXT
-  , version_github TEXT
-  , version_gitlab TEXT
-  , version_pypi TEXT
   , project_repology TEXT
   , nixpkgs_name_replogy TEXT
+  , last_checked_repology DATETIME
+  , version_github TEXT
   , owner_github TEXT
   , repo_github TEXT
+  , last_checked_github DATETIME
+  , version_gitlab TEXT
   , owner_gitlab TEXT
   , repo_gitlab TEXT
-  , last_checked_repology DATETIME
-  , last_checked_github DATETIME
-  , last_hecked_gitlab DATETIME
-  , last_hecked_pypi DATETIME
-  , last_checked_pending_pr DATETIME
-  , last_update_attempt DATETIME
+  , last_checked_gitlab DATETIME
+  , package_name_pypi TEXT
+  , version_pypi TEXT
+  , last_checked_pypi DATETIME
   , pending_pr INTEGER
   , pending_pr_owner TEXT
   , pending_pr_branch_name TEXT
-  , last_update_log DATETIME
+  , last_checked_pending_pr DATETIME
 )

rust/src/main.rs

@@ -1,9 +1,10 @@
+use std::process::Command;
 use diesel::prelude::*;
 use nixpkgs_update::models::*;
 use nixpkgs_update::*;
 use chrono::offset::Utc;
 
-fn fetch_repology(project_name: String) -> Result<json::JsonValue, &'static str> {
+fn fetch_repology(project_name: &String) -> Result<json::JsonValue, &'static str> {
     let body = ureq::get(&format!(
         "https://repology.org/api/v1/project/{}",
         project_name
@@ -25,6 +26,96 @@ fn fetch_repology(project_name: String) -> Result<json::JsonValue, &'static str>
     Err("Couldn't find")
 }
 
+fn fetch_github_latest_release(github: &Github) -> Result<json::JsonValue, &'static str> {
+    let body = ureq::get(&format!(
+        "https://api.github.com/repos/{}/{}/releases/latest",
+        github.owner,
+        github.repo,
+    ))
+    .call()
+    .unwrap()
+    .into_string()
+    .unwrap();
+    if let json::JsonValue::Object(response) = json::parse(&body).unwrap() {
+        return Ok(response["tag_name"].clone());
+    }
+    Err("Couldn't find")
+}
+
+fn version_in_nixpkgs_branch(branch: &str, attr_path: &String) -> String {
+    let output = Command::new("nix")
+        .arg("eval")
+        .arg("--raw")
+        .arg("--refresh")
+        .arg(&format!("github:nixos/nixpkgs/{}#{}", branch, attr_path))
+        .arg("--apply")
+        .arg("(drv: drv.version)")
+        .output()
+        .expect(&format!("Failed to execute nix eval {}", attr_path));
+    String::from_utf8_lossy(&output.stdout).to_string()
+}
+
+fn version_in_nixpkgs_master(attr_path: &String) -> String {
+    version_in_nixpkgs_branch("master", &attr_path)
+}
+
+fn version_in_nixpkgs_staging(attr_path: &String) -> String {
+    version_in_nixpkgs_branch("staging", &attr_path)
+}
+
+fn version_in_nixpkgs_staging_next(attr_path: &String) -> String {
+    version_in_nixpkgs_branch("staging-next", &attr_path)
+}
+
+struct Github {
+    owner: String,
+    repo: String,
+}
+
+fn maybe_github(attr_path: &String) -> Option<Github> {
+    let output = Command::new("nix")
+        .arg("eval")
+        .arg("--raw")
+        .arg("--refresh")
+        .arg(&format!("github:nixos/nixpkgs/master#{}", attr_path))
+        .arg("--apply")
+        .arg("(drv: drv.src.url)")
+        .output()
+        .expect(&format!("Failed to execute nix eval for {}", attr_path));
+    if !String::from_utf8_lossy(&output.stdout).contains("github") {
+      return None;
+    }
+
+    let output = Command::new("nix")
+        .arg("eval")
+        .arg("--raw")
+        .arg("--refresh")
+        .arg(&format!("github:nixos/nixpkgs/master#{}", attr_path))
+        .arg("--apply")
+        .arg("(drv: drv.src.owner)")
+        .output()
+        .expect(&format!("Failed to execute nix eval for {}", attr_path));
+
+    let owner = String::from_utf8_lossy(&output.stdout);
+
+    let output = Command::new("nix")
+        .arg("eval")
+        .arg("--raw")
+        .arg("--refresh")
+        .arg(&format!("github:nixos/nixpkgs/master#{}", attr_path))
+        .arg("--apply")
+        .arg("(drv: drv.src.repo)")
+        .output()
+        .expect(&format!("Failed to execute nix eval for {}", attr_path));
+
+    let repo = String::from_utf8_lossy(&output.stdout);
+
+    return Some(Github {
+        owner: owner.to_string(),
+        repo: repo.to_string(),
+    });
+}
+
 fn main() {
     use nixpkgs_update::schema::packages::dsl::*;
 
@@ -34,14 +125,52 @@ fn main() {
     println!("Displaying {} packages", results.len());
     for package in results {
         println!("{} {}", package.id, package.attr_path);
-        let result : String = fetch_repology(package.attr_path).unwrap().to_string();
+
+        let result : String = fetch_repology(&package.attr_path).unwrap().to_string();
         println!("newest repology version {}", result);
-        diesel::update(packages.find(package.id))
+
+        let version_in_nixpkgs_master : String = version_in_nixpkgs_master(&package.attr_path);
+        println!("nixpkgs master version {}", version_in_nixpkgs_master);
+
+        let version_in_nixpkgs_staging : String = version_in_nixpkgs_staging(&package.attr_path);
+        println!("nixpkgs staging version {}", version_in_nixpkgs_staging);
+
+        let version_in_nixpkgs_staging_next : String = version_in_nixpkgs_staging_next(&package.attr_path);
+        println!("nixpkgs staging_next version {}", version_in_nixpkgs_staging_next);
+
+        let now = Some(Utc::now().naive_utc());
+        diesel::update(packages.find(&package.id))
             .set((
+                version_nixpkgs_master.eq(Some(version_in_nixpkgs_master)),
+                last_checked_nixpkgs_master.eq(now),
+
+                version_nixpkgs_staging.eq(Some(version_in_nixpkgs_staging)),
+                last_checked_nixpkgs_staging.eq(now),
+
+                version_nixpkgs_staging_next.eq(Some(version_in_nixpkgs_staging_next)),
+                last_checked_nixpkgs_staging_next.eq(now),
+
                 version_repology.eq(Some(result)),
-                last_checked_repology.eq(Some(Utc::now().naive_utc())),
+                last_checked_repology.eq(now),
              ))
             .execute(connection)
             .unwrap();
+
+        if let Some(github) = maybe_github(&package.attr_path) {
+            println!("found github for {}", package.attr_path);
+
+            let vgithub : String = fetch_github_latest_release(&github).unwrap().to_string();
+            println!("version github {}", vgithub);
+            let now = Some(Utc::now().naive_utc());
+            diesel::update(packages.find(&package.id))
+                .set((
+                    version_github.eq(Some(vgithub)),
+                    owner_github.eq(Some(github.owner)),
+                    repo_github.eq(Some(github.repo)),
+                    last_checked_github.eq(now),
+                ))
+                .execute(connection)
+                .unwrap();
+        }
     }
 }

rust/src/models.rs

@@ -7,27 +7,31 @@ use diesel::prelude::*;
 pub struct Package {
     pub id: String,
     pub attr_path: String,
+    pub last_update_attempt: Option<NaiveDateTime>,
+    pub last_update_log: Option<NaiveDateTime>,
     pub version_nixpkgs_master: Option<String>,
+    pub last_checked_nixpkgs_master: Option<NaiveDateTime>,
     pub version_nixpkgs_staging: Option<String>,
+    pub last_checked_nixpkgs_staging: Option<NaiveDateTime>,
     pub version_nixpkgs_staging_next: Option<String>,
-    pub version_repology: Option<String>,
-    pub version_github: Option<String>,
-    pub version_gitlab: Option<String>,
-    pub version_pypi: Option<String>,
+    pub last_checked_nixpkgs_staging_next: Option<NaiveDateTime>,
     pub project_repology: Option<String>,
     pub nixpkgs_name_replogy: Option<String>,
+    pub version_repology: Option<String>,
+    pub last_checked_repology: Option<NaiveDateTime>,
     pub owner_github: Option<String>,
     pub repo_github: Option<String>,
+    pub version_github: Option<String>,
+    pub last_checked_github: Option<NaiveDateTime>,
     pub owner_gitlab: Option<String>,
     pub repo_gitlab: Option<String>,
-    pub last_checked_repology: Option<NaiveDateTime>,
-    pub last_checked_github: Option<NaiveDateTime>,
-    pub last_hecked_gitlab: Option<NaiveDateTime>,
-    pub last_hecked_pypi: Option<NaiveDateTime>,
-    pub last_checked_pending_pr: Option<NaiveDateTime>,
-    pub last_update_attempt: Option<NaiveDateTime>,
+    pub version_gitlab: Option<String>,
+    pub last_checked_gitlab: Option<NaiveDateTime>,
+    pub package_name_pypi: Option<String>,
+    pub version_pypi: Option<String>,
+    pub last_checked_pypi: Option<NaiveDateTime>,
     pub pending_pr: Option<i32>,
     pub pending_pr_owner: Option<String>,
     pub pending_pr_branch_name: Option<String>,
-    pub last_update_log: Option<NaiveDateTime>,
+    pub last_checked_pending_pr: Option<NaiveDateTime>,
 }

rust/src/schema.rs

@@ -4,28 +4,32 @@ diesel::table! {
     packages (id) {
         id -> Text,
         attr_path -> Text,
+        last_update_attempt -> Nullable<Timestamp>,
+        last_update_log -> Nullable<Timestamp>,
         version_nixpkgs_master -> Nullable<Text>,
+        last_checked_nixpkgs_master -> Nullable<Timestamp>,
         version_nixpkgs_staging -> Nullable<Text>,
+        last_checked_nixpkgs_staging -> Nullable<Timestamp>,
         version_nixpkgs_staging_next -> Nullable<Text>,
+        last_checked_nixpkgs_staging_next -> Nullable<Timestamp>,
         version_repology -> Nullable<Text>,
-        version_github -> Nullable<Text>,
-        version_gitlab -> Nullable<Text>,
-        version_pypi -> Nullable<Text>,
         project_repology -> Nullable<Text>,
         nixpkgs_name_replogy -> Nullable<Text>,
+        last_checked_repology -> Nullable<Timestamp>,
+        version_github -> Nullable<Text>,
         owner_github -> Nullable<Text>,
         repo_github -> Nullable<Text>,
+        last_checked_github -> Nullable<Timestamp>,
+        version_gitlab -> Nullable<Text>,
         owner_gitlab -> Nullable<Text>,
         repo_gitlab -> Nullable<Text>,
-        last_checked_repology -> Nullable<Timestamp>,
-        last_checked_github -> Nullable<Timestamp>,
-        last_hecked_gitlab -> Nullable<Timestamp>,
-        last_hecked_pypi -> Nullable<Timestamp>,
-        last_checked_pending_pr -> Nullable<Timestamp>,
-        last_update_attempt -> Nullable<Timestamp>,
+        last_checked_gitlab -> Nullable<Timestamp>,
+        package_name_pypi -> Nullable<Text>,
+        version_pypi -> Nullable<Text>,
+        last_checked_pypi -> Nullable<Timestamp>,
         pending_pr -> Nullable<Integer>,
         pending_pr_owner -> Nullable<Text>,
         pending_pr_branch_name -> Nullable<Text>,
-        last_update_log -> Nullable<Timestamp>,
+        last_checked_pending_pr -> Nullable<Timestamp>,
     }
 }