fix: crates-io indexer and makeOutputsForIndexers

- simplify API for makeFlakeOutputsForIndexers (see updated example)
- remove arguments `overrideOutputs` and `indexNames`
- add arguments `indexes`
- add app `ci-job-...` which is a script meant to be executed as a cron action, pushing updates to a separate `data` branch

examples/_d2n-indexer/flake.nix

@@ -7,29 +7,18 @@
     (inp.dream2nix.lib.makeFlakeOutputsForIndexes {
       source = ./.;
       systems = ["x86_64-linux"];
-      indexNames = ["libraries-io"];
+      indexes = {
+        libraries-io = {
+          platform = "npm";
+          number = 5;
+        };
+        crates-io = {};
+      };
       packageOverrides = {
         "^.*$".disable-build = {
           buildScript = ":";
         };
       };
-      overrideOutputs = {
-        mkIndexApp,
-        prevOutputs,
-        ...
-      }: {
-        apps =
-          prevOutputs.apps
-          // {
-            libraries-io = mkIndexApp {
-              name = "libraries-io";
-              input = {
-                platform = "npm";
-                number = 5;
-              };
-            };
-          };
-      };
     })
     // {
       checks = inp.self.packages;

src/indexers/crates-io/indexer/src/lib.rs

@@ -1,8 +1,15 @@
 #![deny(rust_2018_idioms)]
+use serde::{Serialize, Deserialize};
 
-use std::collections::BTreeMap;
+#[derive(Serialize, Deserialize)]
+struct Project {
+    id: String,
+    name: String,
+    version: String,
+    translator: String,
+}
 
-pub type Index = Vec<String>;
+pub type Index = Vec<Project>;
 
 #[cfg(feature = "gen")]
 pub use self::indexer::{Indexer, Modifications, Settings};
@@ -60,15 +67,15 @@ mod indexer {
             self.exclusions.iter().any(|n| n == name)
         }
     }
-    
+
     fn default_max_pages() -> u32 {
         1
     }
-    
+
     fn default_sort_by() -> String {
         "downloads".to_string()
     }
-    
+
     fn default_output_file() -> String {
         "./index.json".to_string()
     }
@@ -127,7 +134,7 @@ mod indexer {
             self.page_callback = f;
             self
         }
-        
+
         pub fn write_info(&mut self) {
             let infos = self.generate_info();
             let file =
@@ -199,9 +206,13 @@ mod indexer {
 
                 let pname = summary.name();
                 let version = summary.version().to_string();
-                let hash = summary.checksum().unwrap().to_string();
 
-                let entry = format!("crates-io:{pname}/{version}?hash={hash}");
+                let entry = Project {
+                    id: format!("{pname}-{version}"),
+                    name: pname.to_string(),
+                    version: version,
+                    translator: "crates-io".to_string(),
+                };
 
                 index.push(entry);
             }

src/lib.nix

@@ -138,8 +138,7 @@
     pkgs ? null,
     config ? {},
     source,
-    indexNames,
-    overrideOutputs ? args: {},
+    indexes,
     inject ? {},
     packageOverrides ? {},
     settings ? [],
@@ -160,8 +159,7 @@
         allOutputs = dream2nix.utils.makeOutputsForIndexes {
           inherit
             source
-            indexNames
-            overrideOutputs
+            indexes
             inject
             packageOverrides
             settings

src/utils/index.nix

@@ -5,6 +5,7 @@
   pkgs,
   apps,
   callPackageDream,
+  utils,
   ...
 } @ topArgs: let
   l = lib // builtins;
@@ -54,31 +55,28 @@ in rec {
     (l.map makePackagesForDreamLock dreamLocks);
   makeOutputsForIndexes = {
     source,
-    indexNames,
-    overrideOutputs ? args: {},
+    indexes,
     settings ? [],
     inject ? {},
     packageOverrides ? {},
     sourceOverrides ? {},
   }: let
     l = lib // builtins;
+    indexNames = l.attrNames indexes;
+
     mkApp = script: {
       type = "app";
       program = toString script;
     };
 
-    mkIndexApp = {
-      name,
-      indexerName ? name,
-      input,
-    } @ args: let
-      input = {outputFile = "${name}/index.json";} // args.input;
+    mkIndexApp = name: input: let
+      inputFinal = {outputFile = "${name}/index.json";} // input;
       script = pkgs.writers.writeBash "index" ''
         set -e
         inputJson="$(${pkgs.coreutils}/bin/mktemp)"
-        echo '${l.toJSON input}' > $inputJson
-        mkdir -p $(dirname ${input.outputFile})
-        ${apps.index}/bin/index ${indexerName} $inputJson
+        echo '${l.toJSON inputFinal}' > $inputJson
+        mkdir -p $(dirname ${inputFinal.outputFile})
+        ${apps.index}/bin/index ${name} $inputJson
       '';
     in
       mkApp script;
@@ -90,6 +88,27 @@ in rec {
             ${name}/index.json ${name}/locks
         ''
       );
+    mkCiJobApp = name: input:
+      mkApp (
+        utils.writePureShellScript
+        (with pkgs; [
+          coreutils
+          git
+          gnugrep
+        ])
+        ''
+          mainBranch=$(git branch | grep -E '(master)|(main)')
+          git branch data || :
+          git checkout data
+          # the flake should always be the one from the current main branch
+          git checkout $mainBranch flake.nix
+          git checkout $mainBranch flake.lock
+          ${(mkIndexApp name input).program}
+          ${(mkTranslateApp name).program}
+          git add .
+          git commit "automatic update - $(date --rfc-3339=seconds)"
+        ''
+      );
     translateApps = l.listToAttrs (
       l.map
       (
@@ -100,28 +119,26 @@ in rec {
       )
       indexNames
     );
-    translateAllApp = let
-      allTranslators =
-        l.concatStringsSep
-        "\n"
-        (
-          l.mapAttrsToList
-          (
-            name: translator: ''
-              echo "::translating with ${name}::"
-              ${translator.program}
-              echo "::translated with ${name}::"
-            ''
-          )
-          translateApps
-        );
-    in
-      mkApp (
-        pkgs.writers.writeBash "translate-all" ''
-          set -e
-          ${allTranslators}
-        ''
-      );
+    indexApps = l.listToAttrs (
+      l.mapAttrsToList
+      (
+        name: input:
+          l.nameValuePair
+          "index-${name}"
+          (mkIndexApp name input)
+      )
+      indexes
+    );
+    ciJobApps = l.listToAttrs (
+      l.mapAttrsToList
+      (
+        name: input:
+          l.nameValuePair
+          "ci-job-${name}"
+          (mkCiJobApp name input)
+      )
+      indexes
+    );
 
     mkIndexOutputs = name: let
       src = "${toString source}/${name}/locks";
@@ -150,15 +167,10 @@ in rec {
     outputs = {
       packages = allPackages;
       apps =
-        translateApps
-        // {
-          translate = translateAllApp;
-        };
+        indexApps
+        // translateApps
+        // ciJobApps;
     };
   in
-    outputs
-    // (callPackageDream overrideOutputs {
-      inherit mkIndexApp;
-      prevOutputs = outputs;
-    });
+    outputs;
 }