fix(indexers): make crates-io-simple indexer work again

src/indexers/crates-io-simple/default.nix

@@ -14,11 +14,17 @@
       echo "[]" > "$tmpFile"
 
       sortBy=$(jq '.sortBy' -c -r $input)
-      maxPages=$(jq '.maxPages' -c -r $input)
+      export number=$(jq '.number' -c -r $input)
 
-      for currentPage in $(seq 1 $maxPages); do
+      # calculate number of pages to query
+      # page size is always 100
+      # result will be truncated to the given $number later
+      numPages=$(($number/100 + ($number % 100 > 0)))
+
+      for currentPage in $(seq 1 $numPages); do
         url="https://crates.io/api/v1/crates?page=$currentPage&per_page=100&sort=$sortBy"
-        curl -k "$url" | python3 ${./process-result.py} > "$tmpFile"
+        echo "fetching page $currentPage"
+        curl -k "$url" | python3 ${./process-result.py} "$tmpFile"
       done
 
       mv "$tmpFile" "$(realpath $outFile)"

src/indexers/crates-io-simple/process-result.py

@@ -1,5 +1,9 @@
 import json
 import sys
+import os
+
+out_file = sys.argv[1]
+number = int(os.environ.get("number"))
 
 input = json.load(sys.stdin)
 projects = []
@@ -10,4 +14,10 @@ for package in input['crates']:
     version=package['max_stable_version'],
     translator='crates-io',
   ))
-print(json.dumps(projects, indent=2))
+
+with open(out_file) as f:
+  existing_projects = json.load(f)
+
+all_projects = (existing_projects + projects)[:number]
+with open(out_file, 'w') as f:
+  json.dump(all_projects, f, indent=2)