From 2334b4cebdcce1104d85afa75315c2721cc2df83 Mon Sep 17 00:00:00 2001 From: Timothy Flynn Date: Fri, 25 Nov 2022 21:03:42 -0500 Subject: [PATCH] Meta: Move UCD/CLDR/TZDB downloaded artifacts to Build/caches They currently reside under Build/, meaning that they would be redownloaded for each architecture/toolchain build combo. Move them to a location that can be re-used for all builds. --- .github/workflows/cmake.yml | 13 +++++++------ .../workflows/pvs-studio-static-analysis.yml | 13 +++++++------ .../workflows/sonar-cloud-static-analysis.yml | 13 +++++++------ .github/workflows/wasm.yml | 19 +++++++++---------- Meta/Azure/Caches.yml | 8 ++++---- Meta/Azure/Lagom.yml | 2 +- Meta/Azure/Serenity.yml | 2 +- Meta/CMake/Superbuild/CMakeLists.txt | 11 +++++------ Meta/CMake/locale_data.cmake | 2 +- Meta/CMake/time_zone_data.cmake | 2 +- Meta/CMake/unicode_data.cmake | 2 +- Meta/Lagom/CMakeLists.txt | 2 ++ Meta/serenity.sh | 1 + 13 files changed, 47 insertions(+), 43 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 56575891824..f9a801dfd07 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -130,26 +130,27 @@ jobs: ccache -s - name: Create build directory run: | - mkdir -p ${{ github.workspace }}/Build/${{ matrix.arch }}/TZDB - mkdir -p ${{ github.workspace }}/Build/${{ matrix.arch }}/UCD - mkdir -p ${{ github.workspace }}/Build/${{ matrix.arch }}/CLDR + mkdir -p ${{ github.workspace }}/Build/${{ matrix.arch }} + mkdir -p ${{ github.workspace }}/Build/caches/TZDB + mkdir -p ${{ github.workspace }}/Build/caches/UCD + mkdir -p ${{ github.workspace }}/Build/caches/CLDR - name: TimeZoneData cache # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged. uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b with: - path: ${{ github.workspace }}/Build/${{ matrix.arch }}/TZDB + path: ${{ github.workspace }}/Build/caches/TZDB key: TimeZoneData-${{ hashFiles('Meta/CMake/time_zone_data.cmake') }} - name: UnicodeData cache # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged. uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b with: - path: ${{ github.workspace }}/Build/${{ matrix.arch }}/UCD + path: ${{ github.workspace }}/Build/caches/UCD key: UnicodeData-${{ hashFiles('Meta/CMake/unicode_data.cmake') }} - name: UnicodeLocale Cache # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged. uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b with: - path: ${{ github.workspace }}/Build/${{ matrix.arch }}/CLDR + path: ${{ github.workspace }}/Build/caches/CLDR key: UnicodeLocale-${{ hashFiles('Meta/CMake/locale_data.cmake') }} - name: Create build environment with extra debug options # Build the entire project with all available debug options turned on, to prevent code rot. diff --git a/.github/workflows/pvs-studio-static-analysis.yml b/.github/workflows/pvs-studio-static-analysis.yml index 4c52b23676c..967f5c6739d 100644 --- a/.github/workflows/pvs-studio-static-analysis.yml +++ b/.github/workflows/pvs-studio-static-analysis.yml @@ -64,27 +64,28 @@ jobs: - name: Create build directory run: | - mkdir -p ${{ github.workspace }}/Build/${{ env.PVS_STUDIO_ANALYSIS_ARCH }}/TZDB - mkdir -p ${{ github.workspace }}/Build/${{ env.PVS_STUDIO_ANALYSIS_ARCH }}/UCD - mkdir -p ${{ github.workspace }}/Build/${{ env.PVS_STUDIO_ANALYSIS_ARCH }}/CLDR + mkdir -p ${{ github.workspace }}/Build/${{ env.PVS_STUDIO_ANALYSIS_ARCH }} + mkdir -p ${{ github.workspace }}/Build/caches/TZDB + mkdir -p ${{ github.workspace }}/Build/caches/UCD + mkdir -p ${{ github.workspace }}/Build/caches/CLDR - name: TimeZoneData cache # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged. uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b with: - path: ${{ github.workspace }}/Build/${{ env.PVS_STUDIO_ANALYSIS_ARCH }}/TZDB + path: ${{ github.workspace }}/Build/caches/TZDB key: TimeZoneData-${{ hashFiles('Meta/CMake/time_zone_data.cmake') }} - name: UnicodeData cache # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged. uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b with: - path: ${{ github.workspace }}/Build/${{ env.PVS_STUDIO_ANALYSIS_ARCH }}/UCD + path: ${{ github.workspace }}/Build/caches/UCD key: UnicodeData-${{ hashFiles('Meta/CMake/unicode_data.cmake') }} - name: UnicodeLocale Cache # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged. uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b with: - path: ${{ github.workspace }}/Build/${{ env.PVS_STUDIO_ANALYSIS_ARCH }}/CLDR + path: ${{ github.workspace }}/Build/caches/CLDR key: UnicodeLocale-${{ hashFiles('Meta/CMake/locale_data.cmake') }} - name: Create build environment diff --git a/.github/workflows/sonar-cloud-static-analysis.yml b/.github/workflows/sonar-cloud-static-analysis.yml index 117c062ebca..bed563771f6 100644 --- a/.github/workflows/sonar-cloud-static-analysis.yml +++ b/.github/workflows/sonar-cloud-static-analysis.yml @@ -97,27 +97,28 @@ jobs: - name: Create build directory run: | - mkdir -p ${{ github.workspace }}/Build/${{ env.SONAR_ANALYSIS_ARCH }}/TZDB - mkdir -p ${{ github.workspace }}/Build/${{ env.SONAR_ANALYSIS_ARCH }}/UCD - mkdir -p ${{ github.workspace }}/Build/${{ env.SONAR_ANALYSIS_ARCH }}/CLDR + mkdir -p ${{ github.workspace }}/Build/${{ env.SONAR_ANALYSIS_ARCH }} + mkdir -p ${{ github.workspace }}/Build/caches/TZDB + mkdir -p ${{ github.workspace }}/Build/caches/UCD + mkdir -p ${{ github.workspace }}/Build/caches/CLDR - name: TimeZoneData cache # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged. uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b with: - path: ${{ github.workspace }}/Build/${{ env.SONAR_ANALYSIS_ARCH }}/TZDB + path: ${{ github.workspace }}/Build/caches/TZDB key: TimeZoneData-${{ hashFiles('Meta/CMake/time_zone_data.cmake') }} - name: UnicodeData cache # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged. uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b with: - path: ${{ github.workspace }}/Build/${{ env.SONAR_ANALYSIS_ARCH }}/UCD + path: ${{ github.workspace }}/Build/caches/UCD key: UnicodeData-${{ hashFiles('Meta/CMake/unicode_data.cmake') }} - name: UnicodeLocale Cache # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged. uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b with: - path: ${{ github.workspace }}/Build/${{ env.SONAR_ANALYSIS_ARCH }}/CLDR + path: ${{ github.workspace }}/Build/caches/CLDR key: UnicodeLocale-${{ hashFiles('Meta/CMake/locale_data.cmake') }} - name: Create build environment diff --git a/.github/workflows/wasm.yml b/.github/workflows/wasm.yml index 08de442e446..125e5d8aa3c 100644 --- a/.github/workflows/wasm.yml +++ b/.github/workflows/wasm.yml @@ -29,26 +29,23 @@ jobs: gcc --version - name: "Create build directories" run: | - mkdir -p ${{ github.workspace }}/Build/lagom-tools/TZDB - mkdir -p ${{ github.workspace }}/Build/wasm/UCD - mkdir -p ${{ github.workspace }}/Build/wasm/CLDR + mkdir -p ${{ github.workspace }}/Build/caches/TZDB + mkdir -p ${{ github.workspace }}/Build/caches/UCD + mkdir -p ${{ github.workspace }}/Build/caches/CLDR - name: "TimeZoneData cache" uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b with: - path: ${{ github.workspace }}/Build/lagom-tools/TZDB + path: ${{ github.workspace }}/Build/caches/TZDB key: TimeZoneData-${{ hashFiles('Meta/CMake/time_zone_data.cmake') }} - - name: "Copy over TZDB cache" - run: | - cp -r ${{ github.workspace }}/Build/lagom-tools/TZDB ${{ github.workspace }}/Build/wasm/TZDB - name: "UnicodeData cache" uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b with: - path: ${{ github.workspace }}/Build/wasm/UCD + path: ${{ github.workspace }}/Build/caches/UCD key: UnicodeData-${{ hashFiles('Meta/CMake/unicode_data.cmake') }} - name: "UnicodeLocale cache" uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b with: - path: ${{ github.workspace }}/Build/wasm/CLDR + path: ${{ github.workspace }}/Build/caches/CLDR key: UnicodeLocale-${{ hashFiles('Meta/CMake/locale_data.cmake') }} - name: "Build host lagom tools" run: | @@ -56,11 +53,12 @@ jobs: -B ${{ github.workspace }}/Build/lagom-tools \ -S ${{ github.workspace }}/Meta/Lagom \ -DBUILD_LAGOM=OFF \ + -DSERENITY_CACHE_DIR=${{ github.workspace }}/Build/caches \ -DCMAKE_C_COMPILER=gcc-12 \ -DCMAKE_CXX_COMPILER=g++-12 \ -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/Build/lagom-tools \ -Dpackage=LagomTools - + ninja -C ${{ github.workspace }}/Build/lagom-tools install - name: "Create wasm build environment" run: | @@ -69,6 +67,7 @@ jobs: -S ${{ github.workspace }}/Meta/Lagom \ -DLagomTools_DIR=${{ github.workspace }}/Build/lagom-tools/share/LagomTools \ -DBUILD_LAGOM=ON \ + -DSERENITY_CACHE_DIR=${{ github.workspace }}/Build/caches \ -DBUILD_SHARED_LIBS=OFF - name: "Build libjs.{js,wasm}" run: | diff --git a/Meta/Azure/Caches.yml b/Meta/Azure/Caches.yml index bd556e56e78..166070df162 100644 --- a/Meta/Azure/Caches.yml +++ b/Meta/Azure/Caches.yml @@ -3,7 +3,7 @@ parameters: arch: 'i686' toolchain: 'gcc' coverage: 'OFF' - build_directory: '' + download_cache_path: '' ccache_version: 1 # Increment this number if CI has trouble with ccache. serenity_ccache_path: '' serenity_ccache_size: '5G' @@ -64,7 +64,7 @@ steps: key: '"time_zone_data" | Meta/CMake/time_zone_data.cmake | "$(timestamp)"' restoreKeys: | "time_zone_data" | Meta/CMake/time_zone_data.cmake - path: $(Build.SourcesDirectory)/${{ parameters.build_directory }}/TZDB + path: $(Build.SourcesDirectory)/${{ parameters.download_cache_path }}/TZDB displayName: 'TimeZoneData Cache' - task: Cache@2 @@ -72,7 +72,7 @@ steps: key: '"unicode_data" | Meta/CMake/unicode_data.cmake | "$(timestamp)"' restoreKeys: | "unicode_data" | Meta/CMake/unicode_data.cmake - path: $(Build.SourcesDirectory)/${{ parameters.build_directory }}/UCD + path: $(Build.SourcesDirectory)/${{ parameters.download_cache_path }}/UCD displayName: 'UnicodeData Cache' - task: Cache@2 @@ -80,5 +80,5 @@ steps: key: '"unicode_locale" | Meta/CMake/locale_data.cmake | "$(timestamp)"' restoreKeys: | "unicode_locale" | Meta/CMake/locale_data.cmake - path: $(Build.SourcesDirectory)/${{ parameters.build_directory }}/CLDR + path: $(Build.SourcesDirectory)/${{ parameters.download_cache_path }}/CLDR displayName: 'UnicodeLocale Cache' diff --git a/Meta/Azure/Lagom.yml b/Meta/Azure/Lagom.yml index 7f7d7ed5964..bbf1f9a6099 100644 --- a/Meta/Azure/Lagom.yml +++ b/Meta/Azure/Lagom.yml @@ -39,7 +39,7 @@ jobs: os: '${{ parameters.os }}' arch: 'Lagom' toolchain: '$(toolchain)' - build_directory: 'Meta/Lagom/Build' + download_cache_path: 'Meta/Lagom/Build' serenity_ccache_path: '$(SERENITY_CCACHE_DIR)' with_remote_data_caches: true ${{ if eq(parameters.os, 'macOS') }}: diff --git a/Meta/Azure/Serenity.yml b/Meta/Azure/Serenity.yml index 0d75f001f04..9458261e3c8 100644 --- a/Meta/Azure/Serenity.yml +++ b/Meta/Azure/Serenity.yml @@ -27,7 +27,7 @@ jobs: arch: '${{ parameters.arch }}' coverage: '${{ parameters.coverage }}' toolchain: 'clang' - build_directory: 'Build/${{ parameters.arch }}clang' + download_cache_path: 'Build/caches' toolchain_ccache_path: '$(LLVM_CCACHE_DIR)' toolchain_ccache_size: '$(LLVM_CCACHE_MAXSIZE)' serenity_ccache_path: '$(SERENITY_CCACHE_DIR)' diff --git a/Meta/CMake/Superbuild/CMakeLists.txt b/Meta/CMake/Superbuild/CMakeLists.txt index 1f71b186471..cabfc00aa1f 100644 --- a/Meta/CMake/Superbuild/CMakeLists.txt +++ b/Meta/CMake/Superbuild/CMakeLists.txt @@ -44,6 +44,9 @@ if(NOT SERENITY_TOOLCHAIN STREQUAL "GNU") endif() set(SERENITY_BUILD_DIR "${PROJECT_BINARY_DIR}/../${SERENITY_ARCH}${SERENITY_BUILD_DIR_SUFFIX}") +# Location to cache artifacts downloaded during the build. +file(REAL_PATH "${PROJECT_BINARY_DIR}/../caches" SERENITY_CACHE_DIR) + # Pkgconf incorrectly discards a sysroot if it doesn't match the start of the path to the # library file. To avoid that, resolve our sysroot into an absolute and canonical path # that matches pkgconf's result for resolving the library file. @@ -97,9 +100,7 @@ ExternalProject_Add( EXCLUDE_FROM_ALL YES CMAKE_CACHE_ARGS "-DCMAKE_INSTALL_PREFIX:STRING=" - "-DTZDB_PATH:STRING=${SERENITY_BUILD_DIR}/TZDB" - "-DUCD_PATH:STRING=${SERENITY_BUILD_DIR}/UCD" - "-DCLDR_PATH:STRING=${SERENITY_BUILD_DIR}/CLDR" + "-DSERENITY_CACHE_DIR:STRING=${SERENITY_CACHE_DIR}" ${lagom_options} # Always call the build step of tools, so keeping things up-to-date is easy BUILD_ALWAYS YES @@ -126,9 +127,7 @@ ExternalProject_Add( # Tell the find_package(Lagom REQUIRED) command call where to find # the CMake package "-DCMAKE_PREFIX_PATH:STRING=${PROJECT_BINARY_DIR}/../lagom-install" - "-DTZDB_PATH:STRING=${SERENITY_BUILD_DIR}/TZDB" - "-DUCD_PATH:STRING=${SERENITY_BUILD_DIR}/UCD" - "-DCLDR_PATH:STRING=${SERENITY_BUILD_DIR}/CLDR" + "-DSERENITY_CACHE_DIR:STRING=${SERENITY_CACHE_DIR}" "-DSERENITY_ARCH:STRING=${SERENITY_ARCH}" "${SERENITY_TOOLCHAIN_FILE_ARG}" ${serenity_options} diff --git a/Meta/CMake/locale_data.cmake b/Meta/CMake/locale_data.cmake index 292247db556..93d5c5c3046 100644 --- a/Meta/CMake/locale_data.cmake +++ b/Meta/CMake/locale_data.cmake @@ -1,7 +1,7 @@ include(${CMAKE_CURRENT_LIST_DIR}/utils.cmake) set(CLDR_VERSION 42.0.0) -set(CLDR_PATH "${CMAKE_BINARY_DIR}/CLDR" CACHE PATH "Download location for CLDR files") +set(CLDR_PATH "${SERENITY_CACHE_DIR}/CLDR" CACHE PATH "Download location for CLDR files") set(CLDR_VERSION_FILE "${CLDR_PATH}/version.txt") set(CLDR_ZIP_URL "https://github.com/unicode-org/cldr-json/releases/download/${CLDR_VERSION}/cldr-${CLDR_VERSION}-json-modern.zip") diff --git a/Meta/CMake/time_zone_data.cmake b/Meta/CMake/time_zone_data.cmake index 4dd9d673229..d3ed1b6c56e 100644 --- a/Meta/CMake/time_zone_data.cmake +++ b/Meta/CMake/time_zone_data.cmake @@ -1,6 +1,6 @@ include(${CMAKE_CURRENT_LIST_DIR}/utils.cmake) -set(TZDB_PATH "${CMAKE_BINARY_DIR}/TZDB" CACHE PATH "Download location for TZDB files") +set(TZDB_PATH "${SERENITY_CACHE_DIR}/TZDB" CACHE PATH "Download location for TZDB files") set(TZDB_VERSION 2022g) set(TZDB_VERSION_FILE "${TZDB_PATH}/version.txt") diff --git a/Meta/CMake/unicode_data.cmake b/Meta/CMake/unicode_data.cmake index 5bb7e9d06fd..6914b99a11f 100644 --- a/Meta/CMake/unicode_data.cmake +++ b/Meta/CMake/unicode_data.cmake @@ -1,7 +1,7 @@ include(${CMAKE_CURRENT_LIST_DIR}/utils.cmake) set(UCD_VERSION 15.0.0) -set(UCD_PATH "${CMAKE_BINARY_DIR}/UCD" CACHE PATH "Download location for UCD files") +set(UCD_PATH "${SERENITY_CACHE_DIR}/UCD" CACHE PATH "Download location for UCD files") set(UCD_VERSION_FILE "${UCD_PATH}/version.txt") set(UCD_ZIP_URL "https://www.unicode.org/Public/${UCD_VERSION}/ucd/UCD.zip") diff --git a/Meta/Lagom/CMakeLists.txt b/Meta/Lagom/CMakeLists.txt index bfa1a6fb539..2e8dd8d46e0 100644 --- a/Meta/Lagom/CMakeLists.txt +++ b/Meta/Lagom/CMakeLists.txt @@ -25,6 +25,8 @@ get_filename_component( ) set(SerenityOS_SOURCE_DIR "${SERENITY_PROJECT_ROOT}" CACHE STRING "") +set(SERENITY_CACHE_DIR "${PROJECT_BINARY_DIR}" CACHE STRING "") + list(APPEND CMAKE_MODULE_PATH "${SERENITY_PROJECT_ROOT}/Meta/CMake") if(NOT COMMAND serenity_option) diff --git a/Meta/serenity.sh b/Meta/serenity.sh index ecdb99be8d0..a9ad19b0811 100755 --- a/Meta/serenity.sh +++ b/Meta/serenity.sh @@ -238,6 +238,7 @@ cmd_with_target() { else SUPER_BUILD_DIR="$BUILD_DIR" CMAKE_ARGS+=("-DCMAKE_INSTALL_PREFIX=$SERENITY_SOURCE_DIR/Build/lagom-install") + CMAKE_ARGS+=("-DSERENITY_CACHE_DIR=${SERENITY_SOURCE_DIR}/Build/caches") fi export PATH="$SERENITY_SOURCE_DIR/Toolchain/Local/cmake/bin":$PATH }