Improve CI performance (#12523)

* Optimized CI execution times

* Fixed a lint error

* Update GUI cache keys for linux

* Added `git status` and `git submodule` to build_xxx-installer.sh

.github/workflows/build-linux-arm64-installer.yml

@@ -114,14 +114,36 @@ jobs:
 
     - uses: chia-network/actions/activate-venv@main
 
+    - name: Prepare GUI cache
+      id: gui-ref
+      run: |
+        gui_ref=$(git submodule status chia-blockchain-gui | sed -e 's/^ //g' -e 's/ chia-blockchain-gui.*$//g')
+        echo "${gui_ref}"
+        echo "::set-output name=GUI_REF::${gui_ref}"
+        echo "rm -rf ./chia-blockchain-gui"
+        rm -rf ./chia-blockchain-gui
+
+    - name: Cache GUI
+      uses: actions/cache@v3
+      id: cache-gui
+      with:
+        path: ./chia-blockchain-gui
+        key: ${{ runner.os }}-arm64-chia-blockchain-gui-${{ steps.gui-ref.outputs.GUI_REF }}
+
+    - if: steps.cache-gui.outputs.cache-hit != 'true'
+      name: Build GUI
+      continue-on-error: false
+      run: |
+        cd ./build_scripts
+        bash build_linux_deb-1-gui.sh
+
     - name: Build arm64 .deb package
       env:
         CHIA_INSTALLER_VERSION: ${{ steps.version_number.outputs.CHIA_INSTALLER_VERSION }}
       run: |
         ldd --version
-        git -C ./chia-blockchain-gui status
         cd ./build_scripts
-        sh build_linux_deb.sh arm64
+        sh build_linux_deb-2-installer.sh arm64
 
     - name: Upload Linux artifacts
       uses: actions/upload-artifact@v3
@@ -205,3 +227,7 @@ jobs:
     - name: Clean up on self hosted runner
       run: |
         sudo rm -rf build_scripts/final_installer
+
+    - name: Remove working files to exclude from cache
+      run: |
+        rm -rf ./chia-blockchain-gui/packages/gui/daemon

.github/workflows/build-linux-installer-deb.yml

@@ -154,14 +154,36 @@ jobs:
       run: |
         sudo apt-get install -y jq
 
+    - name: Prepare GUI cache
+      id: gui-ref
+      run: |
+        gui_ref=$(git submodule status chia-blockchain-gui | sed -e 's/^ //g' -e 's/ chia-blockchain-gui.*$//g')
+        echo "${gui_ref}"
+        echo "::set-output name=GUI_REF::${gui_ref}"
+        echo "rm -rf ./chia-blockchain-gui"
+        rm -rf ./chia-blockchain-gui
+
+    - name: Cache GUI
+      uses: actions/cache@v3
+      id: cache-gui
+      with:
+        path: ./chia-blockchain-gui
+        key: ${{ runner.os }}-deb-chia-blockchain-gui-${{ steps.gui-ref.outputs.GUI_REF }}
+
+    - if: steps.cache-gui.outputs.cache-hit != 'true'
+      name: Build GUI
+      continue-on-error: false
+      run: |
+        cd ./build_scripts
+        bash build_linux_deb-1-gui.sh
+
     - name: Build .deb package
       env:
         CHIA_INSTALLER_VERSION: ${{ steps.version_number.outputs.CHIA_INSTALLER_VERSION }}
       run: |
         ldd --version
-        git -C ./chia-blockchain-gui status
         cd ./build_scripts
-        sh build_linux_deb.sh amd64
+        sh build_linux_deb-2-installer.sh amd64
 
     - name: Upload Linux artifacts
       uses: actions/upload-artifact@v3
@@ -241,3 +263,7 @@ jobs:
       if: startsWith(github.ref, 'refs/tags/')
       run: |
         curl -s -XPOST -H "Authorization: Bearer ${{ secrets.GLUE_ACCESS_TOKEN }}" --data '{"chia_ref": "${{ steps.tag-name.outputs.TAG_NAME }}"}' ${{ secrets.GLUE_API_URL }}/api/v1/${{ steps.tag-name.outputs.REPO_NAME }}/${{ steps.tag-name.outputs.TAG_NAME }}/success/build-linux-deb
+
+    - name: Remove working files to exclude from cache
+      run: |
+        rm -rf ./chia-blockchain-gui/packages/gui/daemon

.github/workflows/build-linux-installer-rpm.yml

@@ -114,14 +114,36 @@ jobs:
 
     - uses: chia-network/actions/activate-venv@main
 
+    - name: Prepare GUI cache
+      id: gui-ref
+      run: |
+        gui_ref=$(git submodule status chia-blockchain-gui | sed -e 's/^ //g' -e 's/ chia-blockchain-gui.*$//g')
+        echo "${gui_ref}"
+        echo "::set-output name=GUI_REF::${gui_ref}"
+        echo "rm -rf ./chia-blockchain-gui"
+        rm -rf ./chia-blockchain-gui
+
+    - name: Cache GUI
+      uses: actions/cache@v3
+      id: cache-gui
+      with:
+        path: ./chia-blockchain-gui
+        key: ${{ runner.os }}-rpm-chia-blockchain-gui-${{ steps.gui-ref.outputs.GUI_REF }}
+
+    - if: steps.cache-gui.outputs.cache-hit != 'true'
+      name: Build GUI
+      continue-on-error: false
+      run: |
+        cd ./build_scripts
+        bash build_linux_rpm-1-gui.sh
+
     - name: Build .rpm package
       env:
         CHIA_INSTALLER_VERSION: ${{ steps.version_number.outputs.CHIA_INSTALLER_VERSION }}
       run: |
         ldd --version
-        git -C ./chia-blockchain-gui status
         cd ./build_scripts
-        bash build_linux_rpm.sh amd64
+        bash build_linux_rpm-2-installer.sh amd64
 
     - name: Upload Linux artifacts
       uses: actions/upload-artifact@v3
@@ -201,3 +223,7 @@ jobs:
       if: startsWith(github.ref, 'refs/tags/')
       run: |
           curl -s -XPOST -H "Authorization: Bearer ${{ secrets.GLUE_ACCESS_TOKEN }}" --data '{"chia_ref": "${{ steps.tag-name.outputs.TAG_NAME }}"}' ${{ secrets.GLUE_API_URL }}/api/v1/${{ steps.tag-name.outputs.REPO_NAME }}/${{ steps.tag-name.outputs.TAG_NAME }}/success/build-linux-rpm
+
+    - name: Remove working files to exclude from cache
+      run: |
+        rm -rf ./chia-blockchain-gui/packages/gui/daemon

.github/workflows/build-macos-installer.yml

@@ -138,6 +138,29 @@ jobs:
       with:
         node-version: '16.x'
 
+    - name: Prepare GUI cache
+      id: gui-ref
+      run: |
+        gui_ref=$(git submodule status chia-blockchain-gui | sed -e 's/^ //g' -e 's/ chia-blockchain-gui.*$//g')
+        echo "${gui_ref}"
+        echo "::set-output name=GUI_REF::${gui_ref}"
+        echo "rm -rf ./chia-blockchain-gui"
+        rm -rf ./chia-blockchain-gui
+
+    - name: Cache GUI
+      uses: actions/cache@v3
+      id: cache-gui
+      with:
+        path: ./chia-blockchain-gui
+        key: ${{ runner.os }}-chia-blockchain-gui-${{ steps.gui-ref.outputs.GUI_REF }}
+
+    - if: steps.cache-gui.outputs.cache-hit != 'true'
+      name: Build GUI
+      continue-on-error: false
+      run: |
+        cd ./build_scripts
+        sh build_macos-1-gui.sh
+
     - name: Build MacOS DMG
       env:
         CHIA_INSTALLER_VERSION: ${{ steps.version_number.outputs.CHIA_INSTALLER_VERSION }}
@@ -145,9 +168,8 @@ jobs:
         APPLE_NOTARIZE_USERNAME: "${{ secrets.APPLE_NOTARIZE_USERNAME }}"
         APPLE_NOTARIZE_PASSWORD: "${{ secrets.APPLE_NOTARIZE_PASSWORD }}"
       run: |
-        git -C ./chia-blockchain-gui status
         cd ./build_scripts
-        sh build_macos.sh
+        sh build_macos-2-installer.sh
 
     - name: Upload MacOS artifacts
       uses: actions/upload-artifact@v3
@@ -212,3 +234,7 @@ jobs:
       if: startsWith(github.ref, 'refs/tags/')
       run: |
         curl -s -XPOST -H "Authorization: Bearer ${{ secrets.GLUE_ACCESS_TOKEN }}" --data '{"chia_ref": "${{ steps.tag-name.outputs.TAG_NAME }}"}' ${{ secrets.GLUE_API_URL }}/api/v1/${{ steps.tag-name.outputs.REPO_NAME }}/${{ steps.tag-name.outputs.TAG_NAME }}/success/build-macos
+
+    - name: Remove working files to exclude from cache
+      run: |
+        rm -rf ./chia-blockchain-gui/packages/gui/daemon

.github/workflows/build-macos-m1-installer.yml

@@ -112,6 +112,30 @@ jobs:
         run: |
           arch -arm64 brew install node@16
 
+      - name: Prepare GUI cache
+        id: gui-ref
+        run: |
+          gui_ref=$(git submodule status chia-blockchain-gui | sed -e 's/^ //g' -e 's/ chia-blockchain-gui.*$//g')
+          echo "${gui_ref}"
+          echo "::set-output name=GUI_REF::${gui_ref}"
+          echo "rm -rf ./chia-blockchain-gui"
+          rm -rf ./chia-blockchain-gui
+
+      - name: Cache GUI
+        uses: actions/cache@v3
+        id: cache-gui
+        with:
+          path: ./chia-blockchain-gui
+          key: ${{ runner.os }}-arm64-chia-blockchain-gui-${{ steps.gui-ref.outputs.GUI_REF }}
+
+      - if: steps.cache-gui.outputs.cache-hit != 'true'
+        name: Build GUI
+        continue-on-error: false
+        run: |
+          export PATH=$(brew --prefix node@16)/bin:$PATH
+          cd ./build_scripts
+          arch -arm64 sh build_macos_m1-1-gui.sh
+
       - name: Build MacOS DMG
         env:
           CHIA_INSTALLER_VERSION: ${{ steps.version_number.outputs.CHIA_INSTALLER_VERSION }}
@@ -120,9 +144,8 @@ jobs:
           APPLE_NOTARIZE_PASSWORD: "${{ secrets.APPLE_NOTARIZE_PASSWORD }}"
         run: |
           export PATH=$(brew --prefix node@16)/bin:$PATH
-          git -C ./chia-blockchain-gui status
           cd ./build_scripts
-          arch -arm64 sh build_macos_m1.sh
+          arch -arm64 sh build_macos_m1-2-installer.sh
 
       - name: Upload MacOS artifacts
         uses: actions/upload-artifact@v3
@@ -198,3 +221,7 @@ jobs:
         if: always()
         run:
           security delete-keychain signing_temp.keychain || true
+
+      - name: Remove working files to exclude from cache
+        run: |
+          rm -rf ./chia-blockchain-gui/packages/gui/daemon

.github/workflows/build-windows-installer.yml

@@ -95,7 +95,7 @@ jobs:
       uses: kitek/decode-base64-into-file-action@1.0
       with:
         encoded-value: ${{ secrets.WIN_CODE_SIGN_CERT }}
-        destination-file: .\chia-blockchain-gui\win_code_sign_cert.p12
+        destination-file: .\win_code_sign_cert.p12
 
     # Create our own venv outside of the git directory JUST for getting the ACTUAL version so that install can't break it
     - name: Get version number
@@ -154,7 +154,31 @@ jobs:
       run: |
         .\Install.ps1 -d
 
-    - name: Build Windows installer with build_scripts\build_windows.ps1
+    - name: Prepare GUI cache
+      id: gui-ref
+      run: |
+        $gui_ref = git submodule status chia-blockchain-gui
+        $gui_ref = $gui_ref -replace "^ | chia-blockchain-gui.*$",""
+        echo $gui_ref
+        echo "::set-output name=GUI_REF::$gui_ref"
+        echo "Remove-Item -Recurse -Force .\chia-blockchain-gui"
+        Remove-Item -Recurse -Force .\chia-blockchain-gui
+
+    - name: Cache GUI
+      uses: actions/cache@v3
+      id: cache-gui
+      with:
+        path: .\chia-blockchain-gui
+        key: ${{ runner.os }}-chia-blockchain-gui-${{ steps.gui-ref.outputs.GUI_REF }}
+
+    - if: steps.cache-gui.outputs.cache-hit != 'true'
+      name: Build GUI
+      continue-on-error: false
+      run: |
+        cd .\build_scripts
+        .\build_windows-1-gui.ps1
+
+    - name: Build Windows installer
       env:
         CHIA_INSTALLER_VERSION: ${{ steps.version_number.outputs.CHIA_INSTALLER_VERSION }}
         WIN_CODE_SIGN_PASS: ${{ secrets.WIN_CODE_SIGN_PASS }}
@@ -162,10 +186,9 @@ jobs:
       run: |
         $env:path="C:\Program` Files` (x86)\Microsoft` Visual` Studio\2019\Enterprise\SDK\ScopeCppSDK\vc15\VC\bin\;$env:path"
         $env:path="C:\Program` Files` (x86)\Windows` Kits\10\App` Certification` Kit;$env:path"
-        git -C .\chia-blockchain-gui status
         .\venv\Scripts\Activate.ps1
         cd .\build_scripts
-        .\build_windows.ps1
+        .\build_windows-2-installer.ps1
 
     - name: Upload Windows exe's to artifacts
       uses: actions/upload-artifact@v3
@@ -259,3 +282,12 @@ jobs:
         }
         $json = $data | ConvertTo-Json
         $response = Invoke-RestMethod '${{ secrets.GLUE_API_URL }}/api/v1/${{ steps.tag-name.outputs.REPO_NAME }}/${{ steps.tag-name.outputs.TAG_NAME }}/success/build-windows' -Method Post -Body $json -ContentType 'application/json' -Headers $headers
+
+    - name: Remove Windows exe and installer to exclude from cache
+      run: |
+        Remove-Item .\chia-blockchain-gui\packages\gui\Chia-win32-x64 -Recurse -Force
+        Remove-Item .\chia-blockchain-gui\packages\gui\release-builds -Recurse -Force
+        Remove-Item .\chia-blockchain-gui\packages\gui\daemon -Recurse -Force
+        Remove-Item .\chia-blockchain-gui\packages\gui\win_code_sign_cert.p12 -Force
+        Remove-Item .\chia-blockchain-gui\Chia-win32-x64 -Recurse -Force
+        Remove-Item .\chia-blockchain-gui\release-builds -Recurse -Force

build_scripts/build_linux_deb-1-gui.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+set -o errexit
+
+echo "Installing global npm packages"
+cd npm_linux_deb || exit
+npm ci
+PATH=$(npm bin):$PATH
+
+cd ../../ || exit
+git submodule update --init chia-blockchain-gui
+
+cd ./chia-blockchain-gui || exit
+
+echo "npm build"
+lerna clean -y
+npm ci
+# Audit fix does not currently work with Lerna. See https://github.com/lerna/lerna/issues/1663
+# npm audit fix
+npm run build
+LAST_EXIT_CODE=$?
+if [ "$LAST_EXIT_CODE" -ne 0 ]; then
+	echo >&2 "npm run build failed!"
+	exit $LAST_EXIT_CODE
+fi
+
+# Remove unused packages
+rm -rf node_modules
+
+# Other than `chia-blockchain-gui/package/gui`, all other packages are no longer necessary after build.
+# Since these unused packages make cache unnecessarily fat, here unused packages are removed.
+echo "Remove unused @chia packages to make cache slim"
+ls -l packages
+rm -rf packages/api
+rm -rf packages/api-react
+rm -rf packages/core
+rm -rf packages/icons
+rm -rf packages/wallets
+
+# Remove unused fat npm modules from the gui package
+cd ./packages/gui/node_modules
+echo "Remove unused node_modules in the gui package to make cache slim more"
+rm -rf electron/dist # ~186MB
+rm -rf "@mui" # ~71MB
+rm -rf typescript # ~63MB
+
+# Remove `packages/gui/node_modules/@chia` because it causes an error on later `electron-packager` command
+rm -rf "@chia"

build_scripts/build_linux_deb-2-installer.sh

@@ -14,6 +14,9 @@ else
 fi
 export PLATFORM
 
+git status
+git submodule
+
 # If the env variable NOTARIZE and the username and password variables are
 # set, this will attempt to Notarize the signed DMG
 
@@ -57,32 +60,25 @@ dpkg-deb --build --root-owner-group "dist/$CLI_DEB_BASE"
 # CLI only .deb done
 
 cp -r dist/daemon ../chia-blockchain-gui/packages/gui
-cd .. || exit
-cd chia-blockchain-gui || exit
-
-echo "npm build"
-lerna clean -y
-npm ci
-# Audit fix does not currently work with Lerna. See https://github.com/lerna/lerna/issues/1663
-# npm audit fix
-npm run build
-LAST_EXIT_CODE=$?
-if [ "$LAST_EXIT_CODE" -ne 0 ]; then
-	echo >&2 "npm run build failed!"
-	exit $LAST_EXIT_CODE
-fi
 
 # Change to the gui package
-cd packages/gui || exit
+cd ../chia-blockchain-gui/packages/gui || exit
 
 # sets the version for chia-blockchain in package.json
 cp package.json package.json.orig
 jq --arg VER "$CHIA_INSTALLER_VERSION" '.version=$VER' package.json > temp.json && mv temp.json package.json
 
+echo electron-packager
 electron-packager . chia-blockchain --asar.unpack="**/daemon/**" --platform=linux \
 --icon=src/assets/img/Chia.icns --overwrite --app-bundle-id=net.chia.blockchain \
---appVersion=$CHIA_INSTALLER_VERSION --executable-name=chia-blockchain
+--appVersion=$CHIA_INSTALLER_VERSION --executable-name=chia-blockchain \
+--no-prune --no-deref-symlinks \
+--ignore="/node_modules/(?!ws(/|$))(?!@electron(/|$))" --ignore="^/src$" --ignore="^/public$"
 LAST_EXIT_CODE=$?
+# Note: `node_modules/ws` and `node_modules/@electron/remote` are dynamic dependencies
+# which GUI calls by `window.require('...')` at runtime.
+# So `ws` and `@electron/remote` cannot be ignored at this time.
+ls -l $DIR_NAME/resources
 
 # reset the package.json to the original
 mv package.json.orig package.json

build_scripts/build_linux_rpm-1-gui.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+set -o errexit
+
+echo "Installing global npm packages"
+cd npm_linux_rpm || exit
+npm ci
+PATH=$(npm bin):$PATH
+
+cd ../../ || exit
+git submodule update --init chia-blockchain-gui
+
+cd ./chia-blockchain-gui || exit
+echo "npm build"
+lerna clean -y
+npm ci
+# Audit fix does not currently work with Lerna. See https://github.com/lerna/lerna/issues/1663
+# npm audit fix
+npm run build
+LAST_EXIT_CODE=$?
+if [ "$LAST_EXIT_CODE" -ne 0 ]; then
+	echo >&2 "npm run build failed!"
+	exit $LAST_EXIT_CODE
+fi
+
+# Remove unused packages
+rm -rf node_modules
+
+# Other than `chia-blockchain-gui/package/gui`, all other packages are no longer necessary after build.
+# Since these unused packages make cache unnecessarily fat, here unused packages are removed.
+echo "Remove unused @chia packages to make cache slim"
+ls -l packages
+rm -rf packages/api
+rm -rf packages/api-react
+rm -rf packages/core
+rm -rf packages/icons
+rm -rf packages/wallets
+
+# Remove unused fat npm modules from the gui package
+cd ./packages/gui/node_modules
+echo "Remove unused node_modules in the gui package to make cache slim more"
+rm -rf electron/dist # ~186MB
+rm -rf "@mui" # ~71MB
+rm -rf typescript # ~63MB
+
+# Remove `packages/gui/node_modules/@chia` because it causes an error on later `electron-packager` command
+rm -rf "@chia"

build_scripts/build_linux_rpm-2-installer.sh

@@ -2,6 +2,9 @@
 
 set -o errexit
 
+git status
+git submodule
+
 if [ ! "$1" ]; then
   echo "This script requires either amd64 of arm64 as an argument"
 	exit 1
@@ -69,32 +72,25 @@ fpm -s dir -t rpm \
 # CLI only rpm done
 
 cp -r dist/daemon ../chia-blockchain-gui/packages/gui
-cd .. || exit
-cd chia-blockchain-gui || exit
-
-echo "npm build"
-lerna clean -y
-npm ci
-# Audit fix does not currently work with Lerna. See https://github.com/lerna/lerna/issues/1663
-# npm audit fix
-npm run build
-LAST_EXIT_CODE=$?
-if [ "$LAST_EXIT_CODE" -ne 0 ]; then
-	echo >&2 "npm run build failed!"
-	exit $LAST_EXIT_CODE
-fi
 
 # Change to the gui package
-cd packages/gui || exit
+cd ../chia-blockchain-gui/packages/gui || exit
 
 # sets the version for chia-blockchain in package.json
 cp package.json package.json.orig
 jq --arg VER "$CHIA_INSTALLER_VERSION" '.version=$VER' package.json > temp.json && mv temp.json package.json
 
+echo electron-packager
 electron-packager . chia-blockchain --asar.unpack="**/daemon/**" --platform=linux \
 --icon=src/assets/img/Chia.icns --overwrite --app-bundle-id=net.chia.blockchain \
---appVersion=$CHIA_INSTALLER_VERSION --executable-name=chia-blockchain
+--appVersion=$CHIA_INSTALLER_VERSION --executable-name=chia-blockchain \
+--no-prune --no-deref-symlinks \
+--ignore="/node_modules/(?!ws(/|$))(?!@electron(/|$))" --ignore="^/src$" --ignore="^/public$"
 LAST_EXIT_CODE=$?
+# Note: `node_modules/ws` and `node_modules/@electron/remote` are dynamic dependencies
+# which GUI calls by `window.require('...')` at runtime.
+# So `ws` and `@electron/remote` cannot be ignored at this time.
+ls -l $DIR_NAME/resources
 
 # reset the package.json to the original
 mv package.json.orig package.json

build_scripts/build_macos-1-gui.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+set -o errexit -o nounset
+
+git status
+
+echo "Installing global npm packages"
+cd npm_macos || exit
+npm ci
+PATH=$(npm bin):$PATH
+cd ../../ || exit
+git submodule update --init chia-blockchain-gui
+
+cd ./chia-blockchain-gui || exit
+echo "npm build"
+lerna clean -y
+npm ci
+# Audit fix does not currently work with Lerna. See https://github.com/lerna/lerna/issues/1663
+# npm audit fix
+npm run build
+LAST_EXIT_CODE=$?
+if [ "$LAST_EXIT_CODE" -ne 0 ]; then
+	echo >&2 "npm run build failed!"
+	exit $LAST_EXIT_CODE
+fi
+
+# Remove unused packages
+rm -rf node_modules
+
+# Other than `chia-blockchain-gui/package/gui`, all other packages are no longer necessary after build.
+# Since these unused packages make cache unnecessarily fat, unused packages should be removed.
+echo "Remove unused @chia packages to make cache slim"
+ls -l packages
+rm -rf packages/api
+rm -rf packages/api-react
+rm -rf packages/core
+rm -rf packages/icons
+rm -rf packages/wallets
+
+# Remove unused fat npm modules from the gui package
+cd ./packages/gui/node_modules
+echo "Remove unused node_modules in the gui package to make cache slim more"
+rm -rf electron/dist # ~186MB
+rm -rf "@mui" # ~71MB
+rm -rf typescript # ~63MB
+
+# Remove `packages/gui/node_modules/@chia` because it causes an error on later `electron-packager` command
+rm -rf "@chia"

build_scripts/build_macos-2-installer.sh

@@ -2,6 +2,9 @@
 
 set -o errexit -o nounset
 
+git status
+git submodule
+
 # If the env variable NOTARIZE and the username and password variables are
 # set, this will attempt to Notarize the signed DMG.
 
@@ -30,33 +33,26 @@ if [ "$LAST_EXIT_CODE" -ne 0 ]; then
 	exit $LAST_EXIT_CODE
 fi
 cp -r dist/daemon ../chia-blockchain-gui/packages/gui
-cd .. || exit
-cd chia-blockchain-gui || exit
-
-echo "npm build"
-lerna clean -y
-npm ci
-# Audit fix does not currently work with Lerna. See https://github.com/lerna/lerna/issues/1663
-# npm audit fix
-npm run build
-LAST_EXIT_CODE=$?
-if [ "$LAST_EXIT_CODE" -ne 0 ]; then
-	echo >&2 "npm run build failed!"
-	exit $LAST_EXIT_CODE
-fi
 
 # Change to the gui package
-cd packages/gui || exit
+cd ../chia-blockchain-gui/packages/gui || exit
 
 # sets the version for chia-blockchain in package.json
 brew install jq
 cp package.json package.json.orig
 jq --arg VER "$CHIA_INSTALLER_VERSION" '.version=$VER' package.json > temp.json && mv temp.json package.json
 
+echo electron-packager
 electron-packager . Chia --asar.unpack="**/daemon/**" --platform=darwin \
 --icon=src/assets/img/Chia.icns --overwrite --app-bundle-id=net.chia.blockchain \
---appVersion=$CHIA_INSTALLER_VERSION
+--appVersion=$CHIA_INSTALLER_VERSION \
+--no-prune --no-deref-symlinks \
+--ignore="/node_modules/(?!ws(/|$))(?!@electron(/|$))" --ignore="^/src$" --ignore="^/public$"
 LAST_EXIT_CODE=$?
+# Note: `node_modules/ws` and `node_modules/@electron/remote` are dynamic dependencies
+# which GUI calls by `window.require('...')` at runtime.
+# So `ws` and `@electron/remote` cannot be ignored at this time.
+ls -l Chia-darwin-x64/Chia.app/Contents/Resources/app.asar
 
 # reset the package.json to the original
 mv package.json.orig package.json

build_scripts/build_macos_m1-1-gui.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+set -o errexit -o nounset
+
+git status
+
+echo "Installing global npm packages"
+cd npm_macos_m1 || exit
+npm ci
+PATH=$(npm bin):$PATH
+cd ../../ || exit
+git submodule update --init chia-blockchain-gui
+
+cd ./chia-blockchain-gui || exit
+
+echo "npm build"
+lerna clean -y
+npm ci
+# Audit fix does not currently work with Lerna. See https://github.com/lerna/lerna/issues/1663
+# npm audit fix
+npm run build
+LAST_EXIT_CODE=$?
+if [ "$LAST_EXIT_CODE" -ne 0 ]; then
+	echo >&2 "npm run build failed!"
+	exit $LAST_EXIT_CODE
+fi
+
+# Remove unused packages
+rm -rf node_modules
+
+# Other than `chia-blockchain-gui/package/gui`, all other packages are no longer necessary after build.
+# Since these unused packages make cache unnecessarily fat, unused packages should be removed.
+echo "Remove unused @chia packages to make cache slim"
+ls -l packages
+rm -rf packages/api
+rm -rf packages/api-react
+rm -rf packages/core
+rm -rf packages/icons
+rm -rf packages/wallets
+
+# Remove unused fat npm modules from the gui package
+cd ./packages/gui/node_modules
+echo "Remove unused node_modules in the gui package to make cache slim more"
+rm -rf electron/dist # ~186MB
+rm -rf "@mui" # ~71MB
+rm -rf typescript # ~63MB
+
+# Remove `packages/gui/node_modules/@chia` because it causes an error on later `electron-packager` command
+rm -rf "@chia"

build_scripts/build_macos_m1-2-installer.sh

@@ -2,6 +2,9 @@
 
 set -o errexit -o nounset
 
+git status
+git submodule
+
 # If the env variable NOTARIZE and the username and password variables are
 # set, this will attempt to Notarize the signed DMG.
 
@@ -30,23 +33,7 @@ if [ "$LAST_EXIT_CODE" -ne 0 ]; then
 	exit $LAST_EXIT_CODE
 fi
 cp -r dist/daemon ../chia-blockchain-gui/packages/gui
-cd .. || exit
-cd chia-blockchain-gui || exit
-
-echo "npm build"
-lerna clean -y
-npm ci
-# Audit fix does not currently work with Lerna. See https://github.com/lerna/lerna/issues/1663
-# npm audit fix
-npm run build
-LAST_EXIT_CODE=$?
-if [ "$LAST_EXIT_CODE" -ne 0 ]; then
-	echo >&2 "npm run build failed!"
-	exit $LAST_EXIT_CODE
-fi
-
-# Change to the gui package
-cd packages/gui || exit
+cd ../chia-blockchain-gui/packages/gui || exit
 
 # sets the version for chia-blockchain in package.json
 brew install jq
@@ -55,8 +42,14 @@ jq --arg VER "$CHIA_INSTALLER_VERSION" '.version=$VER' package.json > temp.json
 
 electron-packager . Chia --asar.unpack="**/daemon/**" --platform=darwin \
 --icon=src/assets/img/Chia.icns --overwrite --app-bundle-id=net.chia.blockchain \
---appVersion=$CHIA_INSTALLER_VERSION
+--appVersion=$CHIA_INSTALLER_VERSION \
+--no-prune --no-deref-symlinks \
+--ignore="/node_modules/(?!ws(/|$))(?!@electron(/|$))" --ignore="^/src$" --ignore="^/public$"
 LAST_EXIT_CODE=$?
+# Note: `node_modules/ws` and `node_modules/@electron/remote` are dynamic dependencies
+# which GUI calls by `window.require('...')` at runtime.
+# So `ws` and `@electron/remote` cannot be ignored at this time.
+ls -l Chia-darwin-arm64/Chia.app/Contents/Resources/app.asar
 
 # reset the package.json to the original
 mv package.json.orig package.json

build_scripts/build_windows-1-gui.ps1

@@ -0,0 +1,59 @@
+# $env:path should contain a path to editbin.exe and signtool.exe
+
+$ErrorActionPreference = "Stop"
+
+git status
+
+Write-Output "   ---"
+Write-Output "Setup npm packager"
+Write-Output "   ---"
+Set-Location -Path ".\npm_windows" -PassThru
+npm ci
+$Env:Path = $(npm bin) + ";" + $Env:Path
+
+Set-Location -Path "..\..\" -PassThru
+git submodule update --init chia-blockchain-gui
+
+Set-Location -Path ".\chia-blockchain-gui" -PassThru
+
+Write-Output "   ---"
+Write-Output "Build GUI npm modules"
+Write-Output "   ---"
+$Env:NODE_OPTIONS = "--max-old-space-size=3000"
+
+Write-Output "lerna clean -y"
+lerna clean -y
+Write-Output "npm ci"
+npm ci
+# Audit fix does not currently work with Lerna. See https://github.com/lerna/lerna/issues/1663
+# npm audit fix
+
+git status
+
+Write-Output "npm run build"
+npm run build
+If ($LastExitCode -gt 0){
+    Throw "npm run build failed!"
+}
+
+# Remove unused packages
+Remove-Item node_modules -Recurse -Force
+
+# Other than `chia-blockchain-gui/package/gui`, all other packages are no longer necessary after build.
+# Since these unused packages make cache unnecessarily fat, unused packages should be removed.
+Write-Output "Remove unused @chia packages to make cache slim"
+Remove-Item packages\api -Recurse -Force
+Remove-Item packages\api-react -Recurse -Force
+Remove-Item packages\core -Recurse -Force
+Remove-Item packages\icons -Recurse -Force
+Remove-Item packages\wallets -Recurse -Force
+
+# Remove unused fat npm modules from the gui package
+Set-Location -Path ".\packages\gui\node_modules" -PassThru
+Write-Output "Remove unused node_modules in the gui package to make cache slim more"
+Remove-Item electron\dist -Recurse -Force # ~186MB
+Remove-Item "@mui" -Recurse -Force # ~71MB
+Remove-Item typescript -Recurse -Force # ~63MB
+
+# Remove `packages/gui/node_modules/@chia` because it causes an error on later `electron-packager` command
+Remove-Item "@chia" -Recurse -Force

build_scripts/build_windows-2-installer.ps1

@@ -5,6 +5,7 @@ $ErrorActionPreference = "Stop"
 mkdir build_scripts\win_build
 
 git status
+git submodule
 
 if (-not (Test-Path env:CHIA_INSTALLER_VERSION)) {
   $env:CHIA_INSTALLER_VERSION = '0.0.0'
@@ -30,36 +31,18 @@ Write-Output "   ---"
 Set-Location -Path ".\npm_windows" -PassThru
 npm ci
 $Env:Path = $(npm bin) + ";" + $Env:Path
-Set-Location -Path "..\" -PassThru
 
-Set-Location -Path "..\chia-blockchain-gui" -PassThru
+Set-Location -Path "..\..\chia-blockchain-gui" -PassThru
 # We need the code sign cert in the gui subdirectory so we can actually sign the UI package
 If ($env:HAS_SECRET) {
-    Copy-Item "win_code_sign_cert.p12" -Destination "packages\gui\"
+    Copy-Item "..\win_code_sign_cert.p12" -Destination "packages\gui\"
 }
 
-git status
-
 Write-Output "   ---"
 Write-Output "Prepare Electron packager"
 Write-Output "   ---"
 $Env:NODE_OPTIONS = "--max-old-space-size=3000"
 
-lerna clean -y
-npm ci
-# Audit fix does not currently work with Lerna. See https://github.com/lerna/lerna/issues/1663
-# npm audit fix
-
-git status
-
-Write-Output "   ---"
-Write-Output "Electron package Windows Installer"
-Write-Output "   ---"
-npm run build
-If ($LastExitCode -gt 0){
-    Throw "npm run build failed!"
-}
-
 # Change to the GUI directory
 Set-Location -Path "packages\gui" -PassThru
 
@@ -85,7 +68,14 @@ Write-Output "   ---"
 
 Write-Output "   ---"
 Write-Output "electron-packager"
-electron-packager . Chia --asar.unpack="**\daemon\**" --overwrite --icon=.\src\assets\img\chia.ico --app-version=$packageVersion
+electron-packager . Chia --asar.unpack="**\daemon\**" `
+--overwrite --icon=.\src\assets\img\chia.ico --app-version=$packageVersion `
+--no-prune --no-deref-symlinks `
+--ignore="/node_modules/(?!ws(/|$))(?!@electron(/|$))" --ignore="^/src$" --ignore="^/public$"
+# Note: `node_modules/ws` and `node_modules/@electron/remote` are dynamic dependencies
+# which GUI calls by `window.require('...')` at runtime.
+# So `ws` and `@electron/remote` cannot be ignored at this time.
+Get-ChildItem Chia-win32-x64\resources
 Write-Output "   ---"
 
 Write-Output "   ---"
@@ -93,8 +83,6 @@ Write-Output "node winstaller.js"
 node winstaller.js
 Write-Output "   ---"
 
-git status
-
 If ($env:HAS_SECRET) {
    Write-Output "   ---"
    Write-Output "Add timestamp and verify signature"
@@ -105,8 +93,6 @@ If ($env:HAS_SECRET) {
    Write-Output "Skipping timestamp and verify signatures - no authorization to install certificates"
 }
 
-git status
-
 Write-Output "   ---"
 Write-Output "Moving final installers to expected location"
 Write-Output "   ---"