diff --git a/src/indexers/crates-io-simple/default.nix b/src/indexers/crates-io-simple/default.nix index 49051104..1bb44c6f 100644 --- a/src/indexers/crates-io-simple/default.nix +++ b/src/indexers/crates-io-simple/default.nix @@ -14,11 +14,17 @@ echo "[]" > "$tmpFile" sortBy=$(jq '.sortBy' -c -r $input) - maxPages=$(jq '.maxPages' -c -r $input) + export number=$(jq '.number' -c -r $input) - for currentPage in $(seq 1 $maxPages); do + # calculate number of pages to query + # page size is always 100 + # result will be truncated to the given $number later + numPages=$(($number/100 + ($number % 100 > 0))) + + for currentPage in $(seq 1 $numPages); do url="https://crates.io/api/v1/crates?page=$currentPage&per_page=100&sort=$sortBy" - curl -k "$url" | python3 ${./process-result.py} > "$tmpFile" + echo "fetching page $currentPage" + curl -k "$url" | python3 ${./process-result.py} "$tmpFile" done mv "$tmpFile" "$(realpath $outFile)" diff --git a/src/indexers/crates-io-simple/process-result.py b/src/indexers/crates-io-simple/process-result.py index 448b47c2..dc16a8a4 100644 --- a/src/indexers/crates-io-simple/process-result.py +++ b/src/indexers/crates-io-simple/process-result.py @@ -1,5 +1,9 @@ import json import sys +import os + +out_file = sys.argv[1] +number = int(os.environ.get("number")) input = json.load(sys.stdin) projects = [] @@ -10,4 +14,10 @@ for package in input['crates']: version=package['max_stable_version'], translator='crates-io', )) -print(json.dumps(projects, indent=2)) + +with open(out_file) as f: + existing_projects = json.load(f) + +all_projects = (existing_projects + projects)[:number] +with open(out_file, 'w') as f: + json.dump(all_projects, f, indent=2)