Add example with WASM audio worklet (#3017)

.github/workflows/main.yml

@@ -232,7 +232,7 @@ jobs:
         ln -snf `pwd`/target/debug/wasm-bindgen $(dirname `which cargo`)/wasm-bindgen
     - run: mv _package.json package.json && npm install && rm package.json
     - run: |
-        for dir in `ls examples | grep -v README | grep -v raytrace | grep -v deno`; do
+        for dir in `ls examples | grep -v README | grep -v raytrace | grep -v deno | grep -v wasm-audio-worklet`; do
           (cd examples/$dir &&
             (npm run build -- --output-path ../../exbuild/$dir ||
               (./build.sh && mkdir -p ../../exbuild/$dir && cp -r ./* ../../exbuild/$dir))
@@ -245,7 +245,7 @@ jobs:
         name: examples1
         path: exbuild
 
-  build_raytrace:
+  build_nightly:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
@@ -253,9 +253,11 @@ jobs:
     - run: rustup target add wasm32-unknown-unknown
     - run: rustup component add rust-src
     - run: |
-        (cd examples/raytrace-parallel && ./build.sh)
-        mkdir exbuild
-        cp examples/raytrace-parallel/*.{js,html,wasm} exbuild
+        for dir in raytrace-parallel wasm-audio-worklet; do
+          (cd examples/$dir &&
+            ./build.sh && mkdir -p ../../exbuild/$dir && cp -r ./* ../../exbuild/$dir
+          ) || exit 1;
+        done
     - uses: actions/upload-artifact@v2
       with:
         name: examples2
@@ -264,7 +266,7 @@ jobs:
   test_examples:
     needs:
     - build_examples
-    - build_raytrace
+    - build_nightly
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
@@ -275,7 +277,7 @@ jobs:
     - uses: actions/download-artifact@v3
       with:
         name: examples2
-        path: exbuild/raytrace-parallel
+        path: exbuild
     - run: rustup update --no-self-update stable && rustup default stable
     - run: cargo test -p example-tests
       env:
@@ -376,7 +378,7 @@ jobs:
       - dist_macos
       - dist_windows
       - build_examples
-      - build_raytrace
+      - build_nightly
       - build_benchmarks
     runs-on: ubuntu-latest
     steps:

Cargo.toml

@@ -78,6 +78,7 @@ members = [
   "examples/raytrace-parallel",
   "examples/request-animation-frame",
   "examples/todomvc",
+  "examples/wasm-audio-worklet",
   "examples/wasm-in-wasm",
   "examples/wasm-in-wasm-imports",
   "examples/wasm-in-web-worker",

examples/wasm-audio-worklet/Cargo.toml

@@ -0,0 +1,30 @@
+[package]
+name = "wasm-audio-worklet"
+version = "0.1.0"
+edition = "2021"
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+console_log = "0.2.0"
+js-sys = "0.3.59"
+wasm-bindgen = "0.2.82"
+wasm-bindgen-futures = "0.4.32"
+
+[dependencies.web-sys]
+version = "0.3.59"
+features = [
+  "AudioContext",
+  "AudioDestinationNode",
+  "AudioWorklet",
+  "AudioWorkletNode",
+  "AudioWorkletNodeOptions",
+  "Blob",
+  "BlobPropertyBag",
+  "Document",
+  "HtmlInputElement",
+  "HtmlLabelElement",
+  "Url",
+  "Window",
+]

examples/wasm-audio-worklet/README.md

@@ -0,0 +1,17 @@
+# wasm-audio-worklet
+
+[View documentation for this example online][dox] or [View compiled example
+online][compiled]
+
+[dox]: https://rustwasm.github.io/docs/wasm-bindgen/examples/wasm-audio-worklet.html
+[compiled]: https://wasm-bindgen.netlify.app/exbuild/wasm-audio-worklet/
+
+You can build the example locally with:
+
+```
+$ ./run.sh
+```
+
+(or running the commands on Windows manually)
+
+and then visiting http://localhost:8080 in a browser should run the example!

examples/wasm-audio-worklet/build.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+
+set -ex
+
+# A couple of steps are necessary to get this build working which makes it slightly
+# nonstandard compared to most other builds.
+#
+# * First, the Rust standard library needs to be recompiled with atomics
+#   enabled. to do that we use Cargo's unstable `-Zbuild-std` feature.
+#
+# * Next we need to compile everything with the `atomics` and `bulk-memory`
+#   features enabled, ensuring that LLVM will generate atomic instructions,
+#   shared memory, passive segments, etc.
+
+RUSTFLAGS='-C target-feature=+atomics,+bulk-memory,+mutable-globals' \
+  cargo build --target wasm32-unknown-unknown --release -Z build-std=std,panic_abort
+
+cargo run -p wasm-bindgen-cli -- \
+  ../../target/wasm32-unknown-unknown/release/wasm_audio_worklet.wasm \
+  --out-dir . \
+  --target web

examples/wasm-audio-worklet/index.html

@@ -0,0 +1,16 @@
+<!DOCTYPE html>
+<html>
+    <head>
+        <title>WASM audio worklet</title>
+    </head>
+    <body>
+        <script type="module">
+            import init, {web_main} from "./wasm_audio_worklet.js";
+            async function run() {
+                await init();
+                web_main();
+            }
+            run();
+        </script>
+    </body>
+</html>

examples/wasm-audio-worklet/run.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -ex
+
+./build.sh
+
+python3 server.py

examples/wasm-audio-worklet/server.py

@@ -0,0 +1,12 @@
+#!/usr/bin/env python3
+from http.server import HTTPServer, SimpleHTTPRequestHandler, test
+import sys
+
+class RequestHandler(SimpleHTTPRequestHandler):
+    def end_headers(self):
+        self.send_header('Cross-Origin-Opener-Policy', 'same-origin')
+        self.send_header('Cross-Origin-Embedder-Policy', 'require-corp')
+        SimpleHTTPRequestHandler.end_headers(self)
+
+if __name__ == '__main__':
+    test(RequestHandler, HTTPServer, port=int(sys.argv[1]) if len(sys.argv) > 1 else 8000)

examples/wasm-audio-worklet/src/dependent_module.rs

@@ -0,0 +1,45 @@
+use js_sys::{Array, JsString};
+use wasm_bindgen::prelude::*;
+use web_sys::{Blob, BlobPropertyBag, Url};
+
+// This is a not-so-clean approach to get the current bindgen ES module URL
+// in Rust. This will fail at run time on bindgen targets not using ES modules.
+#[wasm_bindgen]
+extern "C" {
+    #[wasm_bindgen]
+    type ImportMeta;
+
+    #[wasm_bindgen(method, getter)]
+    fn url(this: &ImportMeta) -> JsString;
+
+    #[wasm_bindgen(js_namespace = import, js_name = meta)]
+    static IMPORT_META: ImportMeta;
+}
+
+pub fn on_the_fly(code: &str) -> Result<String, JsValue> {
+    // Generate the import of the bindgen ES module, assuming `--target web`:
+    let header = format!(
+        "import init, * as bindgen from '{}';\n\n",
+        IMPORT_META.url(),
+    );
+
+    Url::create_object_url_with_blob(&Blob::new_with_str_sequence_and_options(
+        &Array::of2(&JsValue::from(header.as_str()), &JsValue::from(code)),
+        &BlobPropertyBag::new().type_("text/javascript"),
+    )?)
+}
+
+// dependent_module! takes a local file name to a JS module as input and
+// returns a URL to a slightly modified module in run time. This modified module
+// has an additional import statement in the header that imports the current
+// bindgen JS module under the `bindgen` alias, and the separate init function.
+// How this URL is produced does not matter for the macro user. on_the_fly
+// creates a blob URL in run time. A better, more sophisticated solution
+// would add wasm_bindgen support to put such a module in pkg/ during build time
+// and return a URL to this file instead (described in #3019).
+#[macro_export]
+macro_rules! dependent_module {
+    ($file_name:expr) => {
+        crate::dependent_module::on_the_fly(include_str!($file_name))
+    };
+}

examples/wasm-audio-worklet/src/gui.rs

@@ -0,0 +1,39 @@
+use crate::oscillator::Params;
+use wasm_bindgen::{closure::Closure, JsCast, JsValue};
+use web_sys::{AudioContext, HtmlInputElement, HtmlLabelElement};
+
+pub fn create_gui(params: &'static Params, ctx: AudioContext) {
+    let window = web_sys::window().unwrap();
+    let document = window.document().unwrap();
+    let body = document.body().unwrap();
+
+    let volume = add_slider(&document, &body, "Volume:").unwrap();
+    let frequency = add_slider(&document, &body, "Frequency:").unwrap();
+    volume.set_value("0");
+    frequency.set_min("20");
+    frequency.set_value("60");
+
+    let listener = Closure::<dyn FnMut(_)>::new(move |_: web_sys::Event| {
+        params.set_frequency(frequency.value().parse().unwrap());
+        params.set_volume(volume.value().parse().unwrap());
+        ctx.resume().unwrap();
+    })
+    .into_js_value();
+
+    body.add_event_listener_with_callback("input", listener.as_ref().unchecked_ref())
+        .unwrap();
+}
+
+fn add_slider(
+    document: &web_sys::Document,
+    body: &web_sys::HtmlElement,
+    name: &str,
+) -> Result<HtmlInputElement, JsValue> {
+    let input: HtmlInputElement = document.create_element("input")?.unchecked_into();
+    let label: HtmlLabelElement = document.create_element("label")?.unchecked_into();
+    input.set_type("range");
+    label.set_text_content(Some(name));
+    label.append_child(&input)?;
+    body.append_child(&label)?;
+    Ok(input)
+}

examples/wasm-audio-worklet/src/lib.rs

@@ -0,0 +1,20 @@
+mod dependent_module;
+mod gui;
+mod oscillator;
+mod wasm_audio;
+
+use gui::create_gui;
+use oscillator::{Oscillator, Params};
+use wasm_audio::wasm_audio;
+use wasm_bindgen::prelude::*;
+
+#[wasm_bindgen]
+pub async fn web_main() {
+    // On the application level, audio worklet internals are abstracted by wasm_audio:
+    let params: &'static Params = Box::leak(Box::new(Params::default()));
+    let mut osc = Oscillator::new(&params);
+    let ctx = wasm_audio(Box::new(move |buf| osc.process(buf)))
+        .await
+        .unwrap();
+    create_gui(params, ctx);
+}

examples/wasm-audio-worklet/src/oscillator.rs

@@ -0,0 +1,54 @@
+// WASM audio processors can be implemented in Rust without knowing
+// about audio worklets.
+
+use std::sync::atomic::{AtomicU8, Ordering};
+
+// Let's implement a simple sine oscillator with variable frequency and volume.
+pub struct Oscillator {
+    params: &'static Params,
+    accumulator: u32,
+}
+
+impl Oscillator {
+    pub fn new(params: &'static Params) -> Self {
+        Self {
+            params,
+            accumulator: 0,
+        }
+    }
+}
+
+impl Oscillator {
+    pub fn process(&mut self, output: &mut [f32]) -> bool {
+        // This method is called in the audio process thread.
+        // All imports are set, so host functionality available in worklets
+        // (for example, logging) can be used:
+        // `web_sys::console::log_1(&JsValue::from(output.len()));`
+        // Note that currently TextEncoder and TextDecoder are stubs, so passing
+        // strings may not work in this thread.
+        for a in output {
+            let frequency = self.params.frequency.load(Ordering::Relaxed);
+            let volume = self.params.volume.load(Ordering::Relaxed);
+            self.accumulator += u32::from(frequency);
+            *a = (self.accumulator as f32 / 512.).sin() * (volume as f32 / 100.);
+        }
+        true
+    }
+}
+
+#[derive(Default)]
+pub struct Params {
+    // Use atomics for parameters so they can be set in the main thread and
+    // fetched by the audio process thread without further synchronization.
+    frequency: AtomicU8,
+    volume: AtomicU8,
+}
+
+impl Params {
+    pub fn set_frequency(&self, frequency: u8) {
+        self.frequency.store(frequency, Ordering::Relaxed);
+    }
+    pub fn set_volume(&self, volume: u8) {
+        self.volume.store(volume, Ordering::Relaxed);
+    }
+}

examples/wasm-audio-worklet/src/polyfill.js

@@ -0,0 +1,26 @@
+if (!globalThis.TextDecoder) {
+    globalThis.TextDecoder = class TextDecoder {
+        decode(arg) {
+            if (typeof arg !== 'undefined') {
+                throw Error('TextDecoder stub called');
+            } else {
+                return '';
+            }
+        }
+    };
+}
+
+if (!globalThis.TextEncoder) {
+    globalThis.TextEncoder = class TextEncoder {
+        encode(arg) {
+            if (typeof arg !== 'undefined') {
+                throw Error('TextEncoder stub called');
+            } else {
+                return new Uint8Array(0);
+            }
+        }
+    };
+}
+
+export function nop() {
+}

examples/wasm-audio-worklet/src/wasm_audio.rs

@@ -0,0 +1,68 @@
+use crate::dependent_module;
+use wasm_bindgen::prelude::*;
+use wasm_bindgen::JsValue;
+use wasm_bindgen_futures::JsFuture;
+use web_sys::{AudioContext, AudioWorkletNode, AudioWorkletNodeOptions};
+
+#[wasm_bindgen]
+pub struct WasmAudioProcessor(Box<dyn FnMut(&mut [f32]) -> bool>);
+
+#[wasm_bindgen]
+impl WasmAudioProcessor {
+    pub fn process(&mut self, buf: &mut [f32]) -> bool {
+        self.0(buf)
+    }
+    pub fn pack(self) -> usize {
+        Box::into_raw(Box::new(self)) as usize
+    }
+    pub unsafe fn unpack(val: usize) -> Self {
+        *Box::from_raw(val as *mut _)
+    }
+}
+
+// Use wasm_audio if you have a single wasm audio processor in your application
+// whose samples should be played directly. Ideally, call wasm_audio based on
+// user interaction. Otherwise, resume the context on user interaction, so
+// playback starts reliably on all browsers.
+pub async fn wasm_audio(
+    process: Box<dyn FnMut(&mut [f32]) -> bool>,
+) -> Result<AudioContext, JsValue> {
+    let ctx = AudioContext::new()?;
+    prepare_wasm_audio(&ctx).await?;
+    let node = wasm_audio_node(&ctx, process)?;
+    node.connect_with_audio_node(&ctx.destination())?;
+    Ok(ctx)
+}
+
+// wasm_audio_node creates an AudioWorkletNode running a wasm audio processor.
+// Remember to call prepare_wasm_audio once on your context before calling
+// this function.
+pub fn wasm_audio_node(
+    ctx: &AudioContext,
+    process: Box<dyn FnMut(&mut [f32]) -> bool>,
+) -> Result<AudioWorkletNode, JsValue> {
+    AudioWorkletNode::new_with_options(
+        &ctx,
+        "WasmProcessor",
+        &AudioWorkletNodeOptions::new().processor_options(Some(&js_sys::Array::of3(
+            &wasm_bindgen::module(),
+            &wasm_bindgen::memory(),
+            &WasmAudioProcessor(process).pack().into(),
+        ))),
+    )
+}
+
+pub async fn prepare_wasm_audio(ctx: &AudioContext) -> Result<(), JsValue> {
+    nop();
+    let mod_url = dependent_module!("worklet.js")?;
+    JsFuture::from(ctx.audio_worklet()?.add_module(&mod_url)?).await?;
+    Ok(())
+}
+
+// TextEncoder and TextDecoder are not available in Audio Worklets, but there
+// is a dirty workaround: Import polyfill.js to install stub implementations
+// of these classes in globalThis.
+#[wasm_bindgen(module = "/src/polyfill.js")]
+extern "C" {
+    fn nop();
+}

examples/wasm-audio-worklet/src/worklet.js

@@ -0,0 +1,11 @@
+registerProcessor("WasmProcessor", class WasmProcessor extends AudioWorkletProcessor {
+    constructor(options) {
+        super();
+        let [module, memory, handle] = options.processorOptions;
+        bindgen.initSync(module, memory);
+        this.processor = bindgen.WasmAudioProcessor.unpack(handle);
+    }
+    process(inputs, outputs) {
+        return this.processor.process(outputs[0][0]);
+    }
+});

guide/src/examples/raytrace.md

@@ -16,12 +16,14 @@ and wasm with Rust on the web.
 One of the major gotchas with threaded WebAssembly is that Rust does not ship a
 precompiled target (e.g. standard library) which has threading support enabled.
 This means that you'll need to recompile the standard library with the
-appropriate rustc flags, namely `-C target-feature=+atomics,+bulk-memory`.
+appropriate rustc flags, namely
+`-C target-feature=+atomics,+bulk-memory,+mutable-globals`.
+Note that this requires a nightly Rust toolchain.
 
 To do this you can use the `RUSTFLAGS` environment variable that Cargo reads:
 
 ```sh
-export RUSTFLAGS='-C target-feature=+atomics,+bulk-memory'
+export RUSTFLAGS='-C target-feature=+atomics,+bulk-memory,+mutable-globals'
 ```
 
 To recompile the standard library it's recommended to use Cargo's
@@ -40,7 +42,7 @@ build-std = ['std', 'panic_abort']
 
 [build]
 target = "wasm32-unknown-unknown"
-rustflags = '-Ctarget-feature=+atomics,+bulk-memory'
+rustflags = '-Ctarget-feature=+atomics,+bulk-memory,+mutable-globals'
 ```
 
 After this `cargo build` should produce a WebAssembly file with threading

guide/src/examples/wasm-audio-worklet.md

@@ -0,0 +1,45 @@
+# WASM audio worklet
+
+[View full source code][code] or [view the compiled example online][online]
+
+[online]: https://wasm-bindgen.netlify.app/exbuild/wasm-audio-worklet/
+[code]: https://github.com/rustwasm/wasm-bindgen/tree/master/examples/wasm-audio-worklet
+
+This is an example of using threads inside specific worklets with WebAssembly,
+Rust, and `wasm-bindgen`, culminating in an oscillator demo. This demo should
+complement the [parallel-raytrace][parallel-raytrace] example by
+demonstrating an alternative approach using ES modules with on-the-fly module
+creation.
+
+[parallel-raytrace]: https://rustwasm.github.io/docs/wasm-bindgen/examples/raytrace.html
+
+### Building the demo
+
+One of the major gotchas with threaded WebAssembly is that Rust does not ship a
+precompiled target (e.g. standard library) which has threading support enabled.
+This means that you'll need to recompile the standard library with the
+appropriate rustc flags, namely
+`-C target-feature=+atomics,+bulk-memory,+mutable-globals`.
+Note that this requires a nightly Rust toolchain. See the [more detailed
+instructions][build] of the parallel-raytrace example.
+
+[build]: https://rustwasm.github.io/docs/wasm-bindgen/examples/raytrace.html#building-the-demo
+
+### Caveats
+
+This example shares most of its [caveats][caveats] with the parallel-raytrace
+example. However, it tries to encapsulate worklet creation in a Rust module, so
+the application developer does not need to maintain custom JS code.
+
+[caveats]: https://rustwasm.github.io/docs/wasm-bindgen/examples/raytrace.html#caveats
+
+### Browser Requirements
+
+This demo should work in the latest Chrome and Safari versions at this time.
+Firefox [does not support][firefox-worklet-import] imports in worklet modules,
+which are difficult to avoid in this example, as `importScripts` is unavailable
+in worklets. Note that this example requires HTTP headers to be set like in
+[parallel-raytrace][headers].
+
+[firefox-worklet-import]: https://bugzilla.mozilla.org/show_bug.cgi?id=1572644
+[headers]: https://rustwasm.github.io/docs/wasm-bindgen/examples/raytrace.html#browser-requirements