6c05e43 · typeable/nix-s3-cache-action

git.iko.soy/typeable/nix-s3-cache-action

wip

commit6c05e43f68cbe3132a8e91eb17d01904fe1fd6fa authorAnisimov Nikita <anisimoff.nikita@gmail.com> dateWed 7 Oct 2026, 19:22 (1 week ago) parent20a06c0

4 files changed+64−4

compare with main

unifiedsplit

README.md+16 action.yml+5−1 cleanup.sh+1−1 upload.sh+42−2

README.md+16

@@ -33,3 +33,19 @@ to an S3-backed binary cache.

3333| `role-duration-seconds` | no | `21600` | AWS role duration for the Nix cache upload role. |

3434| `cache-private-key` | no | | Private signing key for the Nix binary cache. |

3535| `cache-s3-store-url` | no | | Nix S3 store URL used as the `nix copy` destination. |

36+| `cache-query-jobs` | no | `16` | Parallel cache metadata queries, from `1` to `128`. Set to `1` to disable metadata prefetch and use Nix's native concurrency. |

37+

38+Before uploading, the action resolves the recorded paths' full dependency

39+closure and checks S3 metadata in parallel. These checks populate Nix's shared

40+on-disk narinfo cache, so the subsequent `nix copy` can reuse the results.

41+Missing paths are expected; other query errors fail the action. The copy remains

42+recursive and uploads dependencies before their dependents.

43+

44+This helps when the runner has few CPU cores and S3 request latency dominates:

45+Nix 2.24 limits its native S3 metadata query pool to the number of CPU cores.

46+`parallel-compression=true` only parallelizes compression, not these queries.

47+The action logs closure resolution, metadata query batches, and copy durations.

48+

49+To tune the metadata concurrency, add `cache-query-jobs: "32"` to the upload

50+step's `with` block. More workers use more memory and S3 connections; they do

51+not change the number of concurrent NAR uploads within `nix copy`.

action.yml+5−1

@@ -22,6 +22,10 @@ inputs:

2222 cache-s3-store-url:

2323 description: Nix S3 store URL used as the nix copy destination.

2424 required: false

25+ cache-query-jobs:

26+ description: Number of parallel cache metadata queries (1 disables prefetch; maximum 128).

27+ required: false

28+ default: "16"

2529

2630runs:

2731 using: composite

@@ -61,10 +65,10 @@ runs:

6165 env:

6266 NIX_CACHE_PRIVATE_KEY: ${{ inputs.cache-private-key }}

6367 NIX_CACHE_S3_STORE_URL: ${{ inputs.cache-s3-store-url }}

68+ NIX_CACHE_QUERY_JOBS: ${{ inputs.cache-query-jobs }}

6469 run: bash "$GITHUB_ACTION_PATH/upload.sh"

6570

6671 - name: Clean up Nix S3 cache files

6772 if: ${{ always() && inputs.operation == 'upload' }}

6873 shell: bash

6974 run: bash "$GITHUB_ACTION_PATH/cleanup.sh"

70−

cleanup.sh+1−1

@@ -2,7 +2,7 @@ set -euo pipefail

33rm -f "$RUNNER_TEMP/nix-cache-private-key"

44rm -f "$RUNNER_TEMP/nix-built-paths"

5+rm -f "$RUNNER_TEMP/nix-built-closure"

56rm -f /tmp/nix-record-built-paths

67rm -f /tmp/nix-built-paths

78rm -f /tmp/nix-built-paths.lock

8−

upload.sh+42−2

@@ -15,12 +15,20 @@ if [ -z "${NIX_CACHE_S3_STORE_URL:-}" ]; then

1515 exit 1

1616fi

1717

18+query_jobs="${NIX_CACHE_QUERY_JOBS:-16}"

19+if [[ ! "$query_jobs" =~ ^[1-9][0-9]{0,2}$ ]] || (( query_jobs > 128 )); then

20+ echo "NIX_CACHE_QUERY_JOBS must be an integer between 1 and 128." >&2

21+ exit 1

22+fi

23+

1824key_file="$RUNNER_TEMP/nix-cache-private-key"

1925paths_file="$RUNNER_TEMP/nix-built-paths"

26+closure_file="$RUNNER_TEMP/nix-built-closure"

2027

2128cleanup() {

2229 rm -f "$key_file"

2330 rm -f "$paths_file"

31+ rm -f "$closure_file"

2432}

2533trap cleanup EXIT

2634

@@ -54,6 +62,38 @@ append_store_param parallel-compression true

5462path_count="$(wc -l < "$paths_file" | tr -d ' ')"

5563echo "Uploading $path_count Nix paths to $NIX_CACHE_S3_STORE_URL"

5664started_at="$SECONDS"

57−xargs -r nix copy --to "$upload_url" < "$paths_file"

58−echo "Uploaded $path_count Nix paths to $NIX_CACHE_S3_STORE_URL in $((SECONDS - started_at))s"

5965

66+if (( query_jobs > 1 )); then

67+ stage_started_at="$SECONDS"

68+ echo "Resolving the dependency closure of $path_count recorded paths..."

69+ nix path-info --recursive --stdin < "$paths_file" > "$closure_file"

70+ closure_count="$(wc -l < "$closure_file" | tr -d ' ')"

71+ echo "Resolved $closure_count paths including dependencies in $((SECONDS - stage_started_at))s"

72+

73+ # Nix's S3 metadata query pool is limited to the number of CPU cores.

74+ # Separate processes share its on-disk narinfo cache, including missing

75+ # paths. Prefetch disjoint batches so the recursive copy can reuse these

76+ # checks without uploading overlapping closures in parallel.

77+ batch_size="$(((closure_count + query_jobs - 1) / query_jobs))"

78+ if (( batch_size > 32 )); then batch_size=32; fi

79+ if (( batch_size < 1 )); then batch_size=1; fi

80+ stage_started_at="$SECONDS"

81+ echo "Checking S3 cache metadata for $closure_count paths with up to $query_jobs parallel queries..."

82+ xargs -r -P "$query_jobs" -n "$batch_size" bash -c '

83+ set -euo pipefail

84+ upload_url="$1"

85+ shift

86+ # --print-invalid permits absent paths. Unlike nix path-info, this

87+ # only checks the destination, without querying other substituters.

88+ # The S3 validity check fetches and caches the narinfo metadata.

89+ nix-store --store "$upload_url" --check-validity --print-invalid "$@" > /dev/null

90+ echo "Checked cache metadata for $# paths in ${SECONDS}s"

91+ ' bash "$upload_url" < "$closure_file"

92+ echo "Checked S3 cache metadata in $((SECONDS - stage_started_at))s"

93+fi

94+

95+stage_started_at="$SECONDS"

96+echo "Copying missing paths and dependencies to S3..."

97+nix copy --to "$upload_url" --stdin < "$paths_file"

98+echo "Copied missing paths in $((SECONDS - stage_started_at))s"

99+echo "Uploaded $path_count Nix paths to $NIX_CACHE_S3_STORE_URL in $((SECONDS - started_at))s"