From cbfeac35d694e16658961f7d47e6a043d60a85e6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Donny/=EA=B0=95=EB=8F=99=EC=9C=A4?= <kdy1997.dev@gmail.com>
Date: Sun, 14 Jul 2024 13:19:15 +0900
Subject: [PATCH] chore(ci): Use dynamic matrix for `cargo-test` action (#9233)

**Description:**

We can dynamically create a matrix to make managing CI simpler.
---
 .github/workflows/CI.yml              | 400 ++------------------------
 package.json                          |   4 +-
 scripts/github/create-matrix.sh       |  39 ---
 scripts/github/get-test-matrix.mjs    |  45 +++
 scripts/github/run-cargo-hack.sh      |   2 +-
 scripts/github/tests.yml => tests.yml |   4 +
 yarn.lock                             |  39 ++-
 7 files changed, 111 insertions(+), 422 deletions(-)
 delete mode 100755 scripts/github/create-matrix.sh
 create mode 100755 scripts/github/get-test-matrix.mjs
 rename scripts/github/tests.yml => tests.yml (94%)

diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
index 982520a6ec5..84d4a00fa28 100644
--- a/.github/workflows/CI.yml
+++ b/.github/workflows/CI.yml
@@ -168,391 +168,31 @@ jobs:
         run: |
           (cd bindings/${{ matrix.pkg }} && ./scripts/test.sh)
 
-  cargo-test:
-    name: Test - ${{ matrix.settings.crate }} - ${{ matrix.settings.os }}
-    runs-on: ${{ matrix.settings.runner }}
+  list-cargo-tests:
     if: >-
       ${{ !contains(github.event.head_commit.message, 'chore: ') }}
+    name: List crates
+    runs-on: ubuntu-latest
+    outputs:
+      settings: ${{ steps.list-tests.outputs.settings }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: ./.github/actions/setup-node
+
+      - name: List crates
+        id: list-tests
+        run: echo "settings=$(npx zx ./scripts/github/get-test-matrix.mjs)" >> $GITHUB_OUTPUT
+
+  cargo-test:
+    name: Test - ${{ matrix.settings.crate }} - ${{ matrix.settings.os }}
+    runs-on: ${{ matrix.settings.os }}
+    needs:
+      - list-cargo-tests
     strategy:
       fail-fast: false
       matrix:
-        settings:
-          # Use scripts/github/create-matrix.sh to create this.
-          - crate: ast_node
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: better_scoped_tls
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: binding_macros
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: dbg-swc
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: from_variant
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: jsdoc
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: preset_env_base
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: string_enum
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc
-            os: windows-latest
-            runner: windows-latest
-          - crate: swc_atoms
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_bundler
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_cached
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_cli_impl
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_common
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_common
-            os: windows-latest
-            runner: windows-latest
-          - crate: swc_compiler_base
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_config
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_config_macro
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_core
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_css
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_css_ast
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_css_codegen
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_css_codegen
-            os: windows-latest
-            runner: windows-latest
-          - crate: swc_css_codegen_macros
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_css_compat
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_css_lints
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_css_minifier
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_css_modules
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_css_parser
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_css_parser
-            os: windows-latest
-            runner: windows-latest
-          - crate: swc_css_prefixer
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_css_utils
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_css_visit
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_ecma_ast
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_ecma_codegen
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_ecma_codegen
-            os: windows-latest
-            runner: windows-latest
-          - crate: swc_ecma_codegen_macros
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_ecma_compat_bugfixes
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_ecma_compat_common
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_ecma_compat_es2015
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_ecma_compat_es2016
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_ecma_compat_es2017
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_ecma_compat_es2018
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_ecma_compat_es2019
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_ecma_compat_es2020
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_ecma_compat_es2021
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_ecma_compat_es2022
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_ecma_compat_es3
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_ecma_ext_transforms
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_ecma_lints
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_ecma_loader
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_ecma_loader
-            os: windows-latest
-            runner: windows-latest
-          - crate: swc_ecma_minifier
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_ecma_parser
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_ecma_parser
-            os: windows-latest
-            runner: windows-latest
-          - crate: swc_ecma_preset_env
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_ecma_preset_env
-            os: windows-latest
-            runner: windows-latest
-          - crate: swc_ecma_quote
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_ecma_quote_macros
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_ecma_testing
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_ecma_transforms
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_ecma_transforms_base
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_ecma_transforms_classes
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_ecma_transforms_compat
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_ecma_transforms_macros
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_ecma_transforms_module
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_ecma_transforms_module
-            os: windows-latest
-            runner: windows-latest
-          - crate: swc_ecma_transforms_optimization
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_ecma_transforms_proposal
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_ecma_transforms_react
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_ecma_transforms_testing
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_ecma_transforms_testing
-            os: windows-latest
-            runner: windows-latest
-          - crate: swc_ecma_transforms_typescript
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_ecma_usage_analyzer
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_ecma_utils
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_ecma_visit
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_ecmascript
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_eq_ignore_macros
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_error_reporters
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_estree_ast
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_estree_compat
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_fast_graph
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_fast_ts_strip
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_graph_analyzer
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_html
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_html_ast
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_html_codegen
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_html_codegen
-            os: windows-latest
-            runner: windows-latest
-          - crate: swc_html_codegen_macros
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_html_minifier
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_html_parser
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_html_parser
-            os: windows-latest
-            runner: windows-latest
-          - crate: swc_html_utils
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_html_visit
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_macros_common
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_malloc
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_node_bundler
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_node_bundler
-            os: windows-latest
-            runner: windows-latest
-          - crate: swc_node_comments
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_nodejs_common
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_plugin
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_plugin
-            os: windows-latest
-            runner: windows-latest
-          - crate: swc_plugin_macro
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_plugin_proxy
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_plugin_runner
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_plugin_runner
-            os: macos-latest
-            runner: macos-latest
-          - crate: swc_plugin_runner
-            os: windows-latest
-            runner: windows-latest
-          - crate: swc_plugin_testing
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_plugin_testing
-            os: windows-latest
-            runner: windows-latest
-          - crate: swc_timer
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_trace_macro
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_transform_common
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_visit
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_visit_macros
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_x_optimizer
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_xml
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_xml_ast
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_xml_codegen
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_xml_codegen_macros
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_xml_parser
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_xml_visit
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: testing
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: testing_macros
-            os: ubuntu-latest
-            runner: ubuntu-latest
-          - crate: swc_typescript
-            os: ubuntu-latest
-            runner: ubuntu-latest
+        settings: ${{fromJson(needs.list-cargo-tests.outputs.settings)}}
     steps:
       - name: Handle line endings
         shell: bash
diff --git a/package.json b/package.json
index d0503467bf0..89e22190a08 100644
--- a/package.json
+++ b/package.json
@@ -68,7 +68,9 @@
         "swc-plugin-coverage-instrument": "^0.0.12",
         "terser": "^5.7.1",
         "ts-node": "^10.5.0",
-        "typescript": "^5.3.3"
+        "typescript": "^5.3.3",
+        "yaml": "^2.4.5",
+        "zx": "^8.1.4"
     },
     "lint-staged": {
         "*.toml": [
diff --git a/scripts/github/create-matrix.sh b/scripts/github/create-matrix.sh
deleted file mode 100755
index 6fab8e6098e..00000000000
--- a/scripts/github/create-matrix.sh
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/usr/bin/env bash
-set -u
-
-SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-
-function prepend() { while read line; do echo "${1}${line}"; done; }
-
-crates=$(\
-    cargo metadata --format-version=1 --all-features \
-    | jq '.packages .[] | select(.repository == "https://github.com/swc-project/swc.git") | .name' -r\
-)
-
-# yq query syntax is weird, so we have to use jq
-json_str="$(yq -o=json $SCRIPT_DIR/tests.yml)"
-
-for crate in $crates
-do
-    echo "- crate: $crate"
-    echo "  os: ubuntu-latest"
-
-    if echo $json_str | jq -e "select(.host.\"$crate\")" > /dev/null; then
-        echo "  runner: " `echo $json_str | jq -e -r ".host.\"$crate\""`
-        
-    else
-        echo "  runner: ubuntu-latest"
-    fi
-
-    if echo $json_str | jq -e "select(.os.macos | index(\"$crate\"))" > /dev/null; then
-        echo "- crate: $crate"
-        echo "  os: macos-latest"
-        echo "  runner: macos-latest"
-    fi
-
-    if echo $json_str | jq -e "select(.os.windows | index(\"$crate\"))" > /dev/null; then
-        echo "- crate: $crate"
-        echo "  os: windows-latest"
-        echo "  runner: windows-latest"
-    fi
-done
\ No newline at end of file
diff --git a/scripts/github/get-test-matrix.mjs b/scripts/github/get-test-matrix.mjs
new file mode 100755
index 00000000000..3c1904fd9de
--- /dev/null
+++ b/scripts/github/get-test-matrix.mjs
@@ -0,0 +1,45 @@
+#!/usr/bin/env zx
+import * as path from 'node:path';
+import * as fs from 'node:fs/promises';
+import { parse, } from 'yaml'
+
+const scriptDir = __dirname;
+const repoRootDir = path.resolve(scriptDir, '../../');
+const testsYmlPath = path.resolve(repoRootDir, 'tests.yml');
+const testsYml = parse(await fs.readFile(testsYmlPath, 'utf8'));
+
+process.stderr.write(`Script dir: ${scriptDir}\n`);
+process.stderr.write(`Using tests.yml at ${testsYmlPath}\n`);
+
+const rawMetadata = await $`cargo metadata --format-version=1 --all-features --no-deps`;
+const metadata = JSON.parse(rawMetadata.stdout);
+const packages = metadata.packages.map(p => p.name).filter(name => name !== 'xtask');
+
+process.stderr.write(`Crates: ${packages}\n`)
+process.stderr.write(`Test config: ${testsYml}\n`)
+
+const settings = [];
+
+for (const pkg of packages) {
+    settings.push({
+        crate: pkg,
+        os: 'ubuntu-latest',
+    });
+
+    if (testsYml.os?.windows?.includes(pkg)) {
+        settings.push({
+            crate: pkg,
+            os: 'windows-latest',
+        });
+    }
+
+    if (testsYml.os?.macos?.includes(pkg)) {
+        settings.push({
+            crate: pkg,
+            os: 'macos-latest',
+        });
+    }
+}
+
+const output = JSON.stringify(settings)
+console.log(output)
\ No newline at end of file
diff --git a/scripts/github/run-cargo-hack.sh b/scripts/github/run-cargo-hack.sh
index be3d94cffb9..0c817eb4e24 100755
--- a/scripts/github/run-cargo-hack.sh
+++ b/scripts/github/run-cargo-hack.sh
@@ -8,7 +8,7 @@ crate=$1
 echo "Running cargo hack for crate $crate"
 
 # yq query syntax is weird, so we have to use jq
-json_str="$(yq -o=json $SCRIPT_DIR/tests.yml)"
+json_str="$(yq -o=json $SCRIPT_DIR/../../tests.yml)"
 
 
 if echo $json_str | jq -e ".check.\"$crate\"" > /dev/null; then
diff --git a/scripts/github/tests.yml b/tests.yml
similarity index 94%
rename from scripts/github/tests.yml
rename to tests.yml
index 452b896f767..967e19d2094 100644
--- a/scripts/github/tests.yml
+++ b/tests.yml
@@ -1,6 +1,7 @@
 os:
   windows:
     - swc
+    - swc_allocator
     - swc_cli
     - swc_common
     - swc_css_codegen
@@ -18,6 +19,7 @@ os:
     - swc_plugin_runner
     - swc_plugin_testing
   macos:
+    - swc_allocator
     - swc_plugin_runner
 
 # host:
@@ -27,6 +29,8 @@ check:
   swc:
     # plugin feature is verified by other tasks
     - "cargo hack check --feature-powerset --no-dev-deps --exclude-features debug --exclude-features plugin --exclude-features plugin_transform_schema_v1 --exclude-features plugin_transform_schema_vtest --exclude-features plugin_transform_host_js"
+  swc_allocator:
+    - "cargo hack check --feature-powerset --no-dev-deps"
   swc_bundler:
     - "cargo hack check --feature-powerset --no-dev-deps"
   swc_common:
diff --git a/yarn.lock b/yarn.lock
index 5aca9fc529a..cf6b2b8379f 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -4781,6 +4781,8 @@ __metadata:
     terser: "npm:^5.7.1"
     ts-node: "npm:^10.5.0"
     typescript: "npm:^5.3.3"
+    yaml: "npm:^2.4.5"
+    zx: "npm:^8.1.4"
   languageName: unknown
   linkType: soft
 
@@ -4905,7 +4907,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@types/fs-extra@npm:^11.0.1":
+"@types/fs-extra@npm:>=11, @types/fs-extra@npm:^11.0.1":
   version: 11.0.4
   resolution: "@types/fs-extra@npm:11.0.4"
   dependencies:
@@ -5034,6 +5036,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@types/node@npm:>=20":
+  version: 20.14.10
+  resolution: "@types/node@npm:20.14.10"
+  dependencies:
+    undici-types: "npm:~5.26.4"
+  checksum: 672892cf94d0d95cf052f11271990686a0fd204cd1e5fe7a4ef240e5315e06711765dc47b9ec98627d3adac18b8c92bb7e2d8db21d18faa20bc3e3203a143e79
+  languageName: node
+  linkType: hard
+
 "@types/node@npm:^17.0.17":
   version: 17.0.45
   resolution: "@types/node@npm:17.0.45"
@@ -17975,6 +17986,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"yaml@npm:^2.4.5":
+  version: 2.4.5
+  resolution: "yaml@npm:2.4.5"
+  bin:
+    yaml: bin.mjs
+  checksum: b09bf5a615a65276d433d76b8e34ad6b4c0320b85eb3f1a39da132c61ae6e2ff34eff4624e6458d96d49566c93cf43408ba5e568218293a8c6541a2006883f64
+  languageName: node
+  linkType: hard
+
 "yargs-parser@npm:20.2.4":
   version: 20.2.4
   resolution: "yargs-parser@npm:20.2.4"
@@ -18143,3 +18163,20 @@ __metadata:
   checksum: 8dfecbb939cc8390707a686ccf85dcff3ac24ff69482fe5c39ec43848f438d9e84a8e946b823f8a725bf7ac606d1a6c596a3ffc2b7aaa22f9480b06b5b777ef9
   languageName: node
   linkType: hard
+
+"zx@npm:^8.1.4":
+  version: 8.1.4
+  resolution: "zx@npm:8.1.4"
+  dependencies:
+    "@types/fs-extra": "npm:>=11"
+    "@types/node": "npm:>=20"
+  dependenciesMeta:
+    "@types/fs-extra":
+      optional: true
+    "@types/node":
+      optional: true
+  bin:
+    zx: build/cli.js
+  checksum: 1ffa4c51a1edad25de0729d09667b3d1b7b4f9c8f6b4300e34d85f8f18c2e768f7e297b9bfad4d3b8a24792b3f14085f229933d0a224febba49ac2588ed155b1
+  languageName: node
+  linkType: hard