From 58da4c93fbc2dcd9bde273d0e0bf3f4a07d3a5c8 Mon Sep 17 00:00:00 2001 From: Daniel Bertalan Date: Sat, 27 May 2023 17:54:17 +0200 Subject: [PATCH] Toolchain+CI: Remove cache handling logic from `BuildIt.sh` Instead of manually compressing/decompressing a toolchain tarball if `TRY_USE_LOCAL_TOOLCHAIN` is set, let's use the cache action's automatic built-in compression (which is zstd, I believe). --- .github/workflows/cmake.yml | 9 +-- .../workflows/pvs-studio-static-analysis.yml | 10 +-- .../workflows/sonar-cloud-static-analysis.yml | 8 ++- Toolchain/BuildIt.sh | 65 +------------------ 4 files changed, 18 insertions(+), 74 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 51f2c1fc7f1..efa7aed1d0f 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -86,14 +86,15 @@ jobs: uses: actions/cache/restore@v3 id: toolchain-cache with: - path: ${{ github.workspace }}/Toolchain/Cache/ + path: ${{ github.workspace }}/Toolchain/Local/${{ matrix.arch }} # This assumes that *ALL* LibC and LibPthread headers have an impact on the Toolchain. # This is wrong, and causes more Toolchain rebuilds than necessary. # However, we want to avoid false cache hits at all costs. key: ${{ runner.os }}-toolchain-${{ matrix.arch }}-${{ steps.stamps.outputs.libc_headers }} - - name: Restore or regenerate Toolchain - run: TRY_USE_LOCAL_TOOLCHAIN=y ARCH="${{ matrix.arch }}" ${{ github.workspace }}/Toolchain/BuildIt.sh + - name: Build toolchain + if: ${{ !steps.toolchain-cache.outputs.cache-hit }} + run: ARCH="${{ matrix.arch }}" ${{ github.workspace }}/Toolchain/BuildIt.sh - name: Update toolchain cache uses: actions/cache/save@v3 @@ -101,7 +102,7 @@ jobs: # as it would be discarded after being merged anyway. if: ${{ github.event_name != 'pull_request' && !steps.toolchain-cache.outputs.cache-hit }} with: - path: ${{ github.workspace }}/Toolchain/Cache/ + path: ${{ github.workspace }}/Toolchain/Local/${{ matrix.arch }} key: ${{ steps.toolchain-cache.outputs.cache-primary-key }} - name: ccache(1) cache diff --git a/.github/workflows/pvs-studio-static-analysis.yml b/.github/workflows/pvs-studio-static-analysis.yml index e03afbf0bb9..148b68a7477 100644 --- a/.github/workflows/pvs-studio-static-analysis.yml +++ b/.github/workflows/pvs-studio-static-analysis.yml @@ -48,15 +48,17 @@ jobs: - name: Toolchain cache # This job should always read the cache, never populate it. uses: actions/cache/restore@v3 + id: toolchain-cache with: - path: ${{ github.workspace }}/Toolchain/Cache/ + path: ${{ github.workspace }}/Toolchain/Local/${{ env.PVS_STUDIO_ANALYSIS_ARCH }} # This assumes that *ALL* LibC and LibPthread headers have an impact on the Toolchain. # This is wrong, and causes more Toolchain rebuilds than necessary. # However, we want to avoid false cache hits at all costs. - key: ${{ runner.os }}-toolchain-${{ env.PVS_STUDIO_NALYSIS_ARCH }}-${{ steps.stamps.outputs.libc_headers }} + key: ${{ runner.os }}-toolchain-${{ env.PVS_STUDIO_ANALYSIS_ARCH }}-${{ steps.stamps.outputs.libc_headers }} - - name: Restore or regenerate Toolchain - run: TRY_USE_LOCAL_TOOLCHAIN=y ARCH="${{ env.PVS_STUDIO_ANALYSIS_ARCH }}" ${{ github.workspace }}/Toolchain/BuildIt.sh + - name: Build toolchain + if: ${{ !steps.toolchain-cache.outputs.cache-hit }} + run: ARCH="${{ env.PVS_STUDIO_ANALYSIS_ARCH }}" ${{ github.workspace }}/Toolchain/BuildIt.sh - name: Create build directory run: | diff --git a/.github/workflows/sonar-cloud-static-analysis.yml b/.github/workflows/sonar-cloud-static-analysis.yml index d94b3a45b94..a8e575a433e 100644 --- a/.github/workflows/sonar-cloud-static-analysis.yml +++ b/.github/workflows/sonar-cloud-static-analysis.yml @@ -80,15 +80,17 @@ jobs: - name: Toolchain cache # This job should always read the cache, never populate it. uses: actions/cache/restore@v3 + id: toolchain-cache with: - path: ${{ github.workspace }}/Toolchain/Cache/ + path: ${{ github.workspace }}/Toolchain/Local/${{ env.SONAR_ANALYSIS_ARCH }} # This assumes that *ALL* LibC and LibPthread headers have an impact on the Toolchain. # This is wrong, and causes more Toolchain rebuilds than necessary. # However, we want to avoid false cache hits at all costs. key: ${{ runner.os }}-toolchain-${{ env.SONAR_ANALYSIS_ARCH }}-${{ steps.stamps.outputs.libc_headers }} - - name: Restore or regenerate Toolchain - run: TRY_USE_LOCAL_TOOLCHAIN=y ARCH="${{ env.SONAR_ANALYSIS_ARCH }}" ${{ github.workspace }}/Toolchain/BuildIt.sh + - name: Build toolchain + if: ${{ !steps.toolchain-cache.outputs.cache-hit }} + run: ARCH="${{ env.SONAR_ANALYSIS_ARCH }}" ${{ github.workspace }}/Toolchain/BuildIt.sh - name: Create build directory run: | diff --git a/Toolchain/BuildIt.sh b/Toolchain/BuildIt.sh index 62f6b2a2afd..1f2f2e05100 100755 --- a/Toolchain/BuildIt.sh +++ b/Toolchain/BuildIt.sh @@ -130,51 +130,7 @@ PROGRAM done fi -# === CHECK CACHE AND REUSE === - -pushd "$DIR" - if [ "${TRY_USE_LOCAL_TOOLCHAIN}" = "y" ] ; then - # The actual logic had to be moved to .github/workflows/cmake.yml. - # Github Actions guarantees that Toolchain/Cache/ is empty on a cache - # miss, and non-empty on a cache hit. - # The following logic is correct *only* because of that. - - mkdir -p Cache - echo "Cache (before):" - ls -l Cache - CACHED_TOOLCHAIN_ARCHIVE="Cache/ToolchainBinariesGithubActions.tar.gz" - if [ -r "${CACHED_TOOLCHAIN_ARCHIVE}" ] ; then - echo "Cache at ${CACHED_TOOLCHAIN_ARCHIVE} exists!" - echo "Extracting toolchain from cache:" - if tar xzf "${CACHED_TOOLCHAIN_ARCHIVE}" ; then - echo "Done 'building' the toolchain." - echo "Cache unchanged." - exit 0 - else - echo - echo - echo - echo "Could not extract cached toolchain archive." - echo "This means the cache is broken and *should be removed*!" - echo "As Github Actions cannot update a cache, this will unnecessarily" - echo "slow down all future builds for this hash, until someone" - echo "resets the cache." - echo - echo - echo - rm -f "${CACHED_TOOLCHAIN_ARCHIVE}" - fi - else - echo "Cache at ${CACHED_TOOLCHAIN_ARCHIVE} does not exist." - echo "Will rebuild toolchain from scratch, and save the result." - fi - echo "::group::Actually building Toolchain" - fi -popd - - # === DOWNLOAD AND PATCH === - pushd "$DIR/Tarballs" md5="" if [ -e "$BINUTILS_PKG" ]; then @@ -289,7 +245,7 @@ pushd "$DIR/Build/$ARCH" --with-sysroot="$SYSROOT" \ --enable-shared \ --disable-nls \ - ${TRY_USE_LOCAL_TOOLCHAIN:+"--quiet"} || exit 1 + ${CI:+"--quiet"} || exit 1 if [ "$SYSTEM_NAME" = "Darwin" ]; then # under macOS generated makefiles are not resolving the "intl" # dependency properly to allow linking its own copy of @@ -346,7 +302,7 @@ pushd "$DIR/Build/$ARCH" --enable-threads=posix \ --enable-initfini-array \ --with-linker-hash-style=gnu \ - ${TRY_USE_LOCAL_TOOLCHAIN:+"--quiet"} || exit 1 + ${CI:+"--quiet"} || exit 1 echo "XXX build gcc and libgcc" buildstep "gcc/build" "$MAKE" -j "$MAKEJOBS" all-gcc || exit 1 @@ -365,20 +321,3 @@ popd pushd "$DIR/Local/$ARCH/$ARCH-pc-serenity/bin" buildstep "mold_symlink" ln -s ../../../mold/bin/mold ld.mold popd - - -# == SAVE TO CACHE == - -pushd "$DIR" - if [ "${TRY_USE_LOCAL_TOOLCHAIN}" = "y" ] ; then - echo "::endgroup::" - echo "Building cache tar:" - - rm -f "${CACHED_TOOLCHAIN_ARCHIVE}" # Just in case - - tar czf "${CACHED_TOOLCHAIN_ARCHIVE}" Local/ - - echo "Cache (after):" - ls -l Cache - fi -popd