diff --git a/.github/workflows/ci-build-jit-binary.yaml b/.github/workflows/ci-build-jit-binary.yaml new file mode 100644 index 000000000..34ac0ad85 --- /dev/null +++ b/.github/workflows/ci-build-jit-binary.yaml @@ -0,0 +1,116 @@ +name: build jit binary + +on: + workflow_call: + +defaults: + run: + shell: bash + +env: + jit_src: unison-jit-src/ + jit_dist: unison-jit-dist/ + racket_version: "8.7" + +jobs: + build-jit-binary: + name: build jit binary + strategy: + fail-fast: false + matrix: + os: [ubuntu-20.04, macOS-12, windows-2019] + runs-on: ${{matrix.os}} + steps: + - name: set up environment + run: | + jit_src="$GITHUB_WORKSPACE/${{ env.jit_src }}" # scheme source + jit_exe="${jit_src}/unison-runtime" # initially built jit + jit_dist="${{ runner.temp }}/${{ env.jit_dist }}" # jit binary with libraries destination + jit_dist_exe="${jit_dist}/bin/unison-runtime" # jit binary itself + ucm="${{ runner.temp }}/unison" + + if [[ ${{runner.os}} = "Windows" ]]; then + jit_src="${jit_src//\\//}" + jit_dist="${jit_dist//\\//}" + + jit_exe="${jit_exe//\\//}.exe" + jit_dist_exe="${jit_dist//\\//}/unison-runtime.exe" + ucm="${ucm//\\//}.exe" + fi + + echo "jit_src=$jit_src" >> $GITHUB_ENV + echo "jit_exe=$jit_exe" >> $GITHUB_ENV + echo "jit_dist=$jit_dist" >> $GITHUB_ENV + echo "jit_dist_exe=$jit_dist_exe" >> $GITHUB_ENV + echo "ucm=$ucm" >> $GITHUB_ENV + + - name: get workflow files, for checking hashes + uses: actions/checkout@v4 + with: + sparse-checkout: .github + + - name: download jit source + uses: actions/download-artifact@v4 + with: + name: jit-source + path: ${{ env.jit_src }} + + - name: cache/restore jit binaries + id: cache-jit-binaries + uses: actions/cache/restore@v4 + with: + path: ${{ env.jit_dist }} + key: jit-dist_${{matrix.os}}.racket_${{env.racket_version}}.jit-src_${{hashFiles(format('{0}/**.rkt',env.jit_src),format('{0}/**.ss',env.jit_src))}}.yaml_${{hashFiles('**/ci-build-jit-binary.yaml')}} + + - name: cache racket dependencies + if: steps.cache-jit-binaries.outputs.cache-hit != 'true' + uses: actions/cache@v4 + with: + path: | + ~/.cache/racket + ~/.local/share/racket + key: ${{ matrix.os }}.racket_${{env.racket_version}}.yaml_${{hashFiles('**/ci-build-jit-binary.yaml')}} + + - name: install racket + if: steps.cache-jit-binaries.outputs.cache-hit != 'true' + uses: unisonweb/actions/racket/install@buildjet-cache + with: + version: ${{env.racket_version}} + + - name: set up redistributables (macos) + if: runner.os == 'macOS' && steps.cache-jit-binaries.outputs.cache-hit != 'true' + run: | + brew install libb2 + racket_lib_dir="$(dirname "$(readlink -f "$(which raco)")")/../lib" + ln -s "$(brew --prefix)"/lib/libb2.*.dylib "$racket_lib_dir/" + + - name: build jit binary + if: steps.cache-jit-binaries.outputs.cache-hit != 'true' + shell: bash + run: | + if [[ ${{runner.os}} = "Windows" ]]; then + raco pkg install --auto --skip-installed --scope installation x509-lib + elif [[ ${{runner.os}} = "macOS" ]]; then + raco pkg install --auto --skip-installed --scope installation x509-lib + elif [[ ${{runner.os}} = "Linux" ]]; then + sudo raco pkg install --auto --skip-installed --scope installation x509-lib + fi + raco pkg install --auto --skip-installed "$jit_src"/unison + raco exe --embed-dlls "$jit_src"/unison-runtime.rkt + raco distribute -v "$jit_dist" "$jit_exe" + + - name: cache/save jit binaries + if: steps.cache-jit-binaries.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: ${{ env.jit_dist }} + key: jit-dist_${{matrix.os}}.racket_${{env.racket_version}}.jit-src_${{hashFiles(format('{0}/**.rkt',env.jit_src),format('{0}/**.ss',env.jit_src))}}.yaml_${{hashFiles('**/ci-build-jit-binary.yaml')}} + + - name: save jit binary + uses: actions/upload-artifact@v4 + with: + name: jit-binary-${{ matrix.os }} + path: ${{ env.jit_dist }}/** + + # - name: setup tmate session + # uses: mxschmitt/action-tmate@v3 diff --git a/.github/workflows/ci-test-jit.yaml b/.github/workflows/ci-test-jit.yaml new file mode 100644 index 000000000..e5c75547a --- /dev/null +++ b/.github/workflows/ci-test-jit.yaml @@ -0,0 +1,129 @@ +name: test jit + +on: + workflow_call: + +env: + runtime_tests_version: "@unison/runtime-tests/main" + # for best results, this should match the path in ci.yaml too; but GH doesn't make it easy to share them. + runtime_tests_codebase: "~/.cache/unisonlanguage/runtime-tests.unison" + + jit_src_rel: unison-jit-src + jit_dist_rel: unison-jit-dist + jit_test_results: jit-test-results + +defaults: + run: + shell: bash + +jobs: + build-jit-binary: + name: test jit + strategy: + fail-fast: false + matrix: + os: + - ubuntu-20.04 + - macOS-12 + # - windows-2019 + runs-on: ${{matrix.os}} + steps: + - name: set up environment + run: | + jit_src="$GITHUB_WORKSPACE/${{ env.jit_src_rel }}" # scheme source, for hashing + jit_dist="$GITHUB_WORKSPACE/${{ env.jit_dist_rel }}" # jit binary with libraries destination + jit_dist_exe="${jit_dist}/bin/unison-runtime" # jit binary itself + jit_dist_rel_exe="${jit_dist_rel}/bin/unison-runtime" # jit binary itself + ucm="${{ runner.temp }}/unison" + + if [[ ${{runner.os}} = "Windows" ]]; then + jit_src="${jit_src//\\//}" + jit_dist="${jit_dist//\\//}" + + jit_dist_exe="${jit_dist//\\//}/unison-runtime.exe" + jit_dist_rel_exe="${jit_dist_rel//\\//}/unison-runtime.exe" + ucm="${ucm//\\//}.exe" + fi + + echo "jit_src=$jit_src" >> $GITHUB_ENV + echo "jit_dist=$jit_dist" >> $GITHUB_ENV + echo "jit_dist_exe=$jit_dist_exe" >> $GITHUB_ENV + echo "jit_dist_rel_exe=$jit_dist_rel_exe" >> $GITHUB_ENV + echo "ucm=$ucm" >> $GITHUB_ENV + + - uses: actions/checkout@v4 + with: + sparse-checkout: | + .github + scripts/get-share-hash.sh + unison-src/builtin-tests/jit-tests.tpl.md + unison-src/transcripts-using-base/serialized-cases/case-00.v4.ser + + - name: download jit binaries + uses: actions/download-artifact@v4 + with: + name: jit-binary-${{ matrix.os }} + path: ${{ env.jit_dist }} + + - name: look up hash for runtime tests + run: | + echo "runtime_tests_causalhash=$(scripts/get-share-hash.sh ${{ env.runtime_tests_version }})" >> $GITHUB_ENV + + - name: cache jit test results + id: cache-jit-test-results + uses: actions/cache@v4 + with: + path: ${{env.jit_test_results}} + key: jit-test-results.dist-exe_${{ hashFiles(env.jit_dist_rel_exe) }}.tests_${{ env.runtime_tests_causalhash }}.yaml_${{ hashFiles('**/ci-test-jit.yaml') }} + + - name: install libb2 (linux) + uses: awalsh128/cache-apt-pkgs-action@latest + if: runner.os == 'Linux' && steps.cache-jit-test-results.outputs.cache-hit != 'true' + with: + packages: libb2-1 + version: 1.0 # cache key version afaik + + - name: cache testing codebase + id: cache-testing-codebase + if: steps.cache-jit-test-results.outputs.cache-hit != 'true' + uses: actions/cache@v4 + with: + path: ${{ env.runtime_tests_codebase }} + key: runtime-tests-codebase-${{ matrix.os }}-${{env.runtime_tests_causalhash}} + restore-keys: runtime-tests-codebase-${{ matrix.os }}- + + - name: download ucm + if: steps.cache-jit-test-results.outputs.cache-hit != 'true' + uses: actions/download-artifact@v4 + with: + name: unison-${{ matrix.os }} + path: ${{ runner.temp }} + + - name: set ucm & runtime permissions + if: steps.cache-jit-test-results.outputs.cache-hit != 'true' + run: | + chmod +x ${{ env.ucm }} + chmod +x ${{ env.jit_dist_exe }} + if [[ ${{runner.os}} = "Linux" ]]; then + chmod +x ${{env.jit_dist}}/lib/plt/* + fi + + - name: jit integration test ${{ matrix.os }} + if: steps.cache-jit-test-results.outputs.cache-hit != 'true' + run: | + envsubst '${runtime_tests_version}' \ + < unison-src/builtin-tests/jit-tests.tpl.md \ + > unison-src/builtin-tests/jit-tests.md + ${{ env.ucm }} transcript.fork --runtime-path ${{ env.jit_dist_exe }} -C ${{env.runtime_tests_codebase}} unison-src/builtin-tests/jit-tests.md + cat unison-src/builtin-tests/jit-tests.output.md + git diff --exit-code unison-src/builtin-tests/jit-tests.output.md + + - name: mark jit tests as passing + if: steps.cache-jit-test-results.outputs.cache-hit != 'true' + run: | + echo "passing=true" >> "${{env.jit_test_results}}" + + # - name: Setup tmate session + # uses: mxschmitt/action-tmate@v3 + # if: ${{ failure() }} + # timeout-minutes: 15 diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d626b465c..9f93290f3 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -18,20 +18,18 @@ on: workflow_dispatch: env: - ormolu_version: "0.5.2.0" - racket_version: "8.7" - ucm_local_bin: "ucm-local-bin" + ormolu_version: 0.5.2.0 + ucm_local_bin: ucm-local-bin jit_version: "@unison/internal/releases/0.0.15" - jit_src_scheme: "unison-jit-src/scheme-libs/racket" - jit_dist: "unison-jit-dist" + jit_src_scheme: unison-jit-src/scheme-libs/racket + jit_dist: unison-jit-dist jit_generator_os: ubuntu-20.04 runtime_tests_version: "@unison/runtime-tests/main" runtime_tests_codebase: "~/.cache/unisonlanguage/runtime-tests.unison" # locations of some files that will indicate whether we need to re-run certain steps - transcript_test_results: "transcript-test-results" - interpreter_test_results: "interpreter-test-results" - jit_test_results: "jit-test-results" + transcript_test_results: transcript-test-results + interpreter_test_results: interpreter-test-results jobs: ormolu: @@ -417,173 +415,11 @@ jobs: if-no-files-found: error build-jit-binary: - if: always() && needs.generate-jit-source.result == 'success' name: build jit binary needs: generate-jit-source - runs-on: ${{ matrix.os }} - strategy: - # Run each build to completion, regardless of if any have failed - fail-fast: false - matrix: - os: - # While iterating on this file, you can disable one or more of these to speed things up - - ubuntu-20.04 - - macOS-12 - - windows-2019 - steps: - - name: set up environment - id: checks - run: | - jit_src_scheme="${{ runner.temp }}/${{ env.jit_src_scheme }}" # scheme source - jit_exe="${jit_src_scheme}/unison-runtime" # initially built jit - jit_dist="${{ runner.temp }}/${{ env.jit_dist }}" # jit binary with libraries destination - jit_dist_exe="${jit_dist}/bin/unison-runtime" # jit binary itself - ucm="${{ runner.temp }}/unison" + uses: ./.github/workflows/ci-build-jit-binary.yaml - if [[ ${{runner.os}} = "Windows" ]]; then - jit_src_scheme="${jit_src_scheme//\\//}" - jit_dist="${jit_dist//\\//}" - - jit_exe="${jit_exe//\\//}.exe" - jit_dist_exe="${jit_dist//\\//}/unison-runtime.exe" - ucm="${ucm//\\//}.exe" - fi - - echo "jit_src_scheme=$jit_src_scheme" >> $GITHUB_ENV - echo "jit_exe=$jit_exe" >> $GITHUB_ENV - echo "jit_dist=$jit_dist" >> $GITHUB_ENV - echo "jit_dist_exe=$jit_dist_exe" >> $GITHUB_ENV - echo "ucm=$ucm" >> $GITHUB_ENV - - - uses: actions/checkout@v4 - with: - sparse-checkout: | - scripts/get-share-hash.sh - scheme-libs - unison-src/builtin-tests/jit-tests.tpl.md - unison-src/transcripts-using-base/serialized-cases/case-00.v4.ser - - - name: cache/restore jit binaries - id: cache-jit-binaries - uses: actions/cache/restore@v4 - with: - path: ${{ env.jit_dist }} - key: jit_dist-${{ matrix.os }}.racket_${{ env.racket_version }}.jit_${{ env.jit_version }}-${{hashFiles('**/scheme-libs/**')}} - - - name: Cache Racket dependencies - if: steps.cache-jit-binaries.outputs.cache-hit != 'true' - uses: actions/cache@v4 - with: - path: | - ~/.cache/racket - ~/.local/share/racket - key: ${{ matrix.os }}-racket-${{env.racket_version}} - - - uses: Bogdanp/setup-racket@v1.11 - if: steps.cache-jit-binaries.outputs.cache-hit != 'true' - with: - architecture: x64 - distribution: full - variant: CS - version: ${{env.racket_version}} - - - name: look up hash for runtime tests - run: | - echo "runtime_tests_causalhash=$(scripts/get-share-hash.sh ${{env.runtime_tests_version}})" >> $GITHUB_ENV - - - name: cache jit test results - id: cache-jit-test-results - uses: actions/cache@v4 - with: - path: ${{env.jit_test_results}} - key: jit-test-results.${{ matrix.os }}.racket_${{ env.racket_version }}.jit_${{ env.jit_version }}-${{hashFiles('**/scheme-libs/**')}}.tests_${{env.runtime_tests_causalhash}} - - - name: download jit source - if: steps.cache-jit-binaries.outputs.cache-hit != 'true' - uses: actions/download-artifact@v4 - with: - name: jit-source - path: ${{ env.jit_src_scheme }} - - - name: install libb2 (linux) - uses: awalsh128/cache-apt-pkgs-action@latest - if: | - runner.os == 'Linux' - && steps.cache-jit-test-results.outputs.cache-hit != 'true' - # read this if a package isn't installing correctly - # https://github.com/awalsh128/cache-apt-pkgs-action#caveats - with: - packages: libb2-dev - version: 1.0 # cache key version afaik - - - name: install libb2 (macos) - if: | - runner.os == 'macOS' - && (steps.cache-jit-binaries.outputs.cache-hit != 'true' - || steps.cache-jit-test-results.outputs.cache-hit != 'true') - run: | - brew install libb2 - ln -s "$(brew --prefix)"/lib/libb2.*.dylib \ - "$(dirname "$(readlink -f "$(which raco)")")"/../lib/ - - - name: build jit binary - if: steps.cache-jit-binaries.outputs.cache-hit != 'true' - shell: bash - run: | - cp -R scheme-libs/racket/* "$jit_src_scheme" - raco pkg install --auto --skip-installed "$jit_src_scheme"/unison - raco exe --embed-dlls "$jit_src_scheme"/unison-runtime.rkt - raco distribute "$jit_dist" "$jit_exe" - - - name: cache/save jit binaries - if: steps.cache-jit-binaries.outputs.cache-hit != 'true' - uses: actions/cache/save@v4 - with: - path: ${{ env.jit_dist }} - key: jit_dist-${{ matrix.os }}.racket_${{ env.racket_version }}.jit_${{ env.jit_version }}-${{hashFiles('**/scheme-libs/**')}} - - - name: save jit binary - uses: actions/upload-artifact@v4 - with: - name: jit-binary-${{ matrix.os }} - path: ${{ env.jit_dist }}/** - - - name: cache testing codebase - id: cache-testing-codebase - if: steps.cache-jit-test-results.outputs.cache-hit != 'true' - uses: actions/cache@v4 - with: - path: ${{ env.runtime_tests_codebase }} - key: runtime-tests-codebase-${{ matrix.os }}-${{env.runtime_tests_causalhash}} - restore-keys: runtime-tests-codebase-${{ matrix.os }}- - - - name: download ucm - if: steps.cache-jit-test-results.outputs.cache-hit != 'true' - uses: actions/download-artifact@v4 - with: - name: unison-${{ matrix.os }} - path: ${{ runner.temp }} - - - name: set ucm permissions - if: steps.cache-jit-test-results.outputs.cache-hit != 'true' - run: chmod +x ${{ env.ucm }} - - - name: jit integration test ${{ matrix.os }} - if: runner.os != 'Windows' && steps.cache-jit-test-results.outputs.cache-hit != 'true' - run: | - envsubst '${runtime_tests_version}' \ - < unison-src/builtin-tests/jit-tests.tpl.md \ - > unison-src/builtin-tests/jit-tests.md - ${{ env.ucm }} transcript.fork --runtime-path ${{ env.jit_dist_exe }} -C ${{env.runtime_tests_codebase}} unison-src/builtin-tests/jit-tests.md - cat unison-src/builtin-tests/jit-tests.output.md - git diff --exit-code unison-src/builtin-tests/jit-tests.output.md - - - name: mark jit tests as passing - if: runner.os != 'Windows' && steps.cache-jit-test-results.outputs.cache-hit != 'true' - run: | - echo "passing=true" >> "${{env.jit_test_results}}" - - # - name: Setup tmate session - # uses: mxschmitt/action-tmate@v3 - # if: ${{ failure() }} - # timeout-minutes: 15 + test-jit: + name: test jit + needs: build-jit-binary + uses: ./.github/workflows/ci-test-jit.yaml diff --git a/docs/github-actions-help.md b/docs/github-actions-help.md index c454232cc..4889c7c62 100644 --- a/docs/github-actions-help.md +++ b/docs/github-actions-help.md @@ -26,6 +26,9 @@ e.g. It's not clear to me when to use `$GITHUB_OUTPUT` vs `$GITHUB_ENV`, but I have been favoring `$GITHUB_ENV` because it requires fewer characters to access. However, it seems a little wrong. +### `hashFiles()` +`hashFiles()` can only access files inside of and relative to `$GITHUB_WORKSPACE`. + ### `if:` Although the type rules don't totally make sense in Github Actions, `if:` takes a Boolean.