Merge pull request #3041 from rtfeldman/build-web-repl-on-netlify

Build web REPL on Netlify

Earthfile

@@ -105,7 +105,8 @@ test-rust:
     # RUN echo "4" | cargo run --locked --release --features="target-x86" -- --target=x86_32 examples/benchmarks/NQueens.roc
     # RUN --mount=type=cache,target=$SCCACHE_DIR \
     #    cargo test --locked --release --features with_sound --test cli_run i386 --features="i386-cli-run" && sccache --show-stats
-    # make sure doc generation works (that is, make sure build.sh returns status code 0)
+    # make sure website deployment works (that is, make sure build.sh returns status code 0)
+    ENV REPL_DEBUG=1
     RUN bash www/build.sh
 
 

compiler/builtins/build.rs

@@ -20,15 +20,6 @@ fn zig_executable() -> String {
 fn main() {
     println!("cargo:rerun-if-changed=build.rs");
 
-    // When we build on Netlify, zig is not installed (but also not used,
-    // since all we're doing is generating docs), so we can skip the steps
-    // that require having zig installed.
-    if env::var_os("NO_ZIG_INSTALLED").is_some() {
-        // We still need to do the other things before this point, because
-        // setting the env vars is needed for other parts of the build.
-        return;
-    }
-
     // "." is relative to where "build.rs" is
     // dunce can be removed once ziglang/zig#5109 is fixed
     let build_script_dir_path = dunce::canonicalize(Path::new(".")).unwrap();

repl_wasm/Cargo.toml

@@ -29,5 +29,7 @@ roc_types = {path = "../compiler/types"}
 [features]
 wasmer = ["futures"]
 
+# Tell wasm-pack not to run wasm-opt automatically. We run it explicitly when we need to.
+# (Workaround for a CI install issue with wasm-pack https://github.com/rustwasm/wasm-pack/issues/864)
 [package.metadata.wasm-pack.profile.profiling]
-wasm-opt = ['-Os', '-g'] # `wasm-pack --profiling` is supposed to have debug info `-g` by default, but doesn't
+wasm-opt = false

repl_wasm/build.rs

@@ -13,15 +13,6 @@ fn main() {
     let source_path = format!("src/{}.c", PLATFORM_FILENAME);
     println!("cargo:rerun-if-changed={}", source_path);
 
-    // When we build on Netlify, zig is not installed (but also not used,
-    // since all we're doing is generating docs), so we can skip the steps
-    // that require having zig installed.
-    if env::var_os("NO_ZIG_INSTALLED").is_some() {
-        // We still need to do the other things before this point, because
-        // setting the env vars is needed for other parts of the build.
-        return;
-    }
-
     std::fs::create_dir_all("./data").unwrap();
 
     // Zig can produce *either* an object containing relocations OR an object containing libc code

repl_www/build.sh

@@ -18,18 +18,19 @@ fi
 WWW_ROOT="${1:-repl_www/public}"
 mkdir -p $WWW_ROOT
 
-# When debugging the REPL, use `REPL_DEBUG=1 repl_www/build.sh`
-if [ -n "${REPL_DEBUG:-}" ]
-then
-    # Leave out wasm-opt since it takes too long when debugging, and provide some debug options
-    cargo build --target wasm32-unknown-unknown -p roc_repl_wasm --release --features console_error_panic_hook
-    wasm-bindgen --target web --keep-debug target/wasm32-unknown-unknown/release/roc_repl_wasm.wasm --out-dir repl_wasm/pkg/
-else
-    # A `--profiling` build is optimized and has debug info, so we get stack traces for compiler `todo!()`
-    wasm-pack build --profiling --target web repl_wasm -- --features console_error_panic_hook
-fi
+# We want a release build, but with debug info (to get stack traces for Wasm backend `todo!()`)
+# This configuration is called `--profiling`
+wasm-pack build --profiling --target web repl_wasm -- --features console_error_panic_hook -v
+cp -v repl_wasm/pkg/roc_repl_wasm.js $WWW_ROOT
 
-cp repl_wasm/pkg/*.wasm $WWW_ROOT
+# To disable optimizations while debugging, run `REPL_DEBUG=1 repl_www/build.sh`
+if [ "${REPL_DEBUG:-}" == "" ] && which wasm-opt
+then
+    wasm-opt -Os --debuginfo repl_wasm/pkg/roc_repl_wasm_bg.wasm -o $WWW_ROOT/roc_repl_wasm_bg.wasm
+else
+    echo "wasm-opt is not installed. Skipping .wasm optimization."
+    cp -v repl_wasm/pkg/roc_repl_wasm_bg.wasm $WWW_ROOT
+fi
 
 # Copy the JS from wasm_bindgen, replacing its invalid `import` statement with a `var`.
 # The JS import from the invalid path 'env', seems to be generated when there are unresolved symbols.

www/build.sh

@@ -11,21 +11,9 @@ cd $SCRIPT_RELATIVE_DIR
 rm -rf build/
 cp -r public/ build/
 
-pushd build
-
 # grab the source code and copy it to Netlify's server; if it's not there, fail the build.
+pushd build
 wget https://github.com/rtfeldman/elm-css/files/8037422/roc-source-code.zip
-
-# Copy REPL webpage source files
-cp -r ../../repl_www/public/* .
-
-# grab the pre-compiled REPL and copy it to Netlify's server; if it's not there, fail the build.
-wget https://github.com/brian-carroll/mock-repl/archive/refs/heads/deploy.zip
-unzip deploy.zip
-mv mock-repl-deploy/* .
-rmdir mock-repl-deploy
-rm deploy.zip
-
 popd
 
 pushd ..
@@ -33,24 +21,22 @@ echo 'Generating docs...'
 cargo --version
 rustc --version
 
-# We set RUSTFLAGS to -Awarnings to ignore warnings during this build,
-# because when building without "the" llvm feature (which is only ever done
-# for this exact use case), the result is lots of "unused" warnings!
-RUSTFLAGS=-Awarnings
+# removing `-C link-arg=-fuse-ld=lld` from RUSTFLAGS because this causes an error when compiling `roc_repl_wasm`
+RUSTFLAGS="-C target-cpu=native"
 
 # We set ROC_DOCS_ROOT_DIR=builtins so that links will be generated relative to
 # "/builtins/" rather than "/" - which is what we want based on how the server
 # is set up to serve them.
 export ROC_DOCS_URL_ROOT=/builtins
 
-# These just need to be defined so that some env! macros don't fail.
-BUILTINS_WASM32_O=""
-BUILTINS_HOST_O=""
-
-# We run the CLI with --no-default-features because that way we don't have the
-# "llvm" feature and therefore don't depend on LLVM being installed on the
-# system. (Netlify's build servers have Rust installed, but not LLVM.)
 cargo run --bin roc-docs compiler/builtins/roc/*.roc
 mv generated-docs/*.* www/build # move all the .js, .css, etc. files to build/
 mv generated-docs/ www/build/builtins # move all the folders to build/builtins/
+
+echo "Building Web REPL..."
+repl_www/build.sh www/build
+
+echo "Asset sizes:"
+ls -lh www/build/*
+
 popd

www/netlify.sh

@@ -6,11 +6,19 @@ set -euxo pipefail
 
 rustup update
 rustup default stable
+rustup target add wasm32-unknown-unknown wasm32-wasi
 
-# TODO remove this once we actually build the web repl!
-SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-REPL_WASM_DATA=${SCRIPT_DIR}/../repl_wasm/data/
-mkdir -p ${REPL_WASM_DATA}
-touch ${REPL_WASM_DATA}/pre_linked_binary.o
+ZIG_DIRNAME="zig-linux-x86_64-0.9.1"
+wget https://ziglang.org/download/0.9.1/${ZIG_DIRNAME}.tar.xz
+tar --extract --xz --file=${ZIG_DIRNAME}.tar.xz
+PATH="$(pwd)/${ZIG_DIRNAME}:${PATH}"
 
+# Work around an issue with wasm-pack where it fails to install wasm-opt (from binaryen) on some CI systems
+# https://github.com/rustwasm/wasm-pack/issues/864
+BINARYEN_DIRNAME="binaryen-version_108-x86_64-linux"
+wget https://github.com/WebAssembly/binaryen/releases/download/version_108/${BINARYEN_DIRNAME}.tar.gz
+tar --extract --gzip --file=${BINARYEN_DIRNAME}.tar.gz
+PATH="$(pwd)/${BINARYEN_DIRNAME}/bin:${PATH}"
+
+export PATH
 bash build.sh