Test examples in CI (#3015)

* Add a test that examples don't throw any errors

TODO:
- run all the tests, not just the ones which use webpack (also an issue with CI)
- fix webxr test
- run in CI
- share WebDriver instance between tests
- maybe ditch async, since we don't really need it here and it adds a bunch of dependencies and build time.

* Disable testing WebXR example

It isn't supported in Firefox yet, which is where we're running our tests.

* Test examples that aren't built with webpack

* Remove `WEBDRIVER` environment variable

It wouldn't have worked anyway because at least for the moment, I'm using one WebDriver session per test, and Firefox at least only allows one session to be connected.

I would like to make them share a session, in which case I could add this back, but I can't right now because Firefox hasn't implemented `LogEntry.source` yet, which is needed to figure out which log entries should fail which tests.

* Run in CI

* Use `Once` instead of `Mutex`

* Build `webxr` and `synchronous-instantiation` in CI

Although we can't test them, we can still build them.

* Add missing '`'

* Fix running of tests

* Only include dev deps when not compiling for wasm

* oops, those are the native tests

* Create build dirs before copying to them

* Install binaryen

* decompress

* Follow redirects

* Set `PATH` properly

* Use an absolute path

* Don't symlink `node_modules` and fix artifact download

* Enable `web_sys_unstable_apis`

This is needed for the `webxr` example.

* Increase timeout to 10s

* Increase timeout to 20s

This seems excessive but 10s is still sometimes failing.

* Disable testing the webgl example

* Add binaryen to PATH directly after installing

* Properly download the raytrace example artifacts

* Disable example tests instead of enabling everything else

* Move to a separate `example-tests` crate
This commit is contained in:
Liam Murphy 2022-08-06 01:51:41 +10:00 committed by GitHub
parent 8e19dcfe53
commit 643a773429
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 648 additions and 12 deletions

View File

@ -223,16 +223,23 @@ jobs:
- run: rustup update --no-self-update stable && rustup default stable
- run: rustup target add wasm32-unknown-unknown
- run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh -s -- -f
- run: |
curl -L https://github.com/WebAssembly/binaryen/releases/download/version_109/binaryen-version_109-x86_64-linux.tar.gz -sSf > binaryen-version_109-x86_64-linux.tar.gz
tar -xz -f binaryen-version_109-x86_64-linux.tar.gz
echo "$PWD/binaryen-version_109/bin" >> $GITHUB_PATH
- run: |
cargo build -p wasm-bindgen-cli
ln -snf `pwd`/target/debug/wasm-bindgen $(dirname `which cargo`)/wasm-bindgen
- run: mv _package.json package.json && npm install && rm package.json
- run: |
for dir in `ls examples | grep -v README | grep -v asm.js | grep -v raytrace | grep -v without-a-bundler | grep -v wasm-in-web-worker | grep -v websockets | grep -v webxr | grep -v deno | grep -v synchronous-instantiation`; do
for dir in `ls examples | grep -v README | grep -v raytrace | grep -v deno`; do
(cd examples/$dir &&
ln -fs ../../node_modules . &&
npm run build -- --output-path ../../exbuild/$dir) || exit 1;
(npm run build -- --output-path ../../exbuild/$dir ||
(./build.sh && mkdir -p ../../exbuild/$dir && cp -r ./* ../../exbuild/$dir))
) || exit 1;
done
env:
RUSTFLAGS: --cfg=web_sys_unstable_apis
- uses: actions/upload-artifact@v2
with:
name: examples1
@ -246,7 +253,6 @@ jobs:
- run: rustup target add wasm32-unknown-unknown
- run: rustup component add rust-src
- run: |
sed -i 's/python/#python/' examples/raytrace-parallel/build.sh
(cd examples/raytrace-parallel && ./build.sh)
mkdir exbuild
cp examples/raytrace-parallel/*.{js,html,wasm} exbuild
@ -255,6 +261,26 @@ jobs:
name: examples2
path: exbuild
test_examples:
needs:
- build_examples
- build_raytrace
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/download-artifact@v3
with:
name: examples1
path: exbuild
- uses: actions/download-artifact@v3
with:
name: examples2
path: exbuild/raytrace-parallel
- run: rustup update --no-self-update stable && rustup default stable
- run: cargo test -p example-tests
env:
EXBUILD: exbuild
build_benchmarks:
runs-on: ubuntu-latest
steps:

View File

@ -55,6 +55,7 @@ members = [
"crates/js-sys",
"crates/test",
"crates/test/sample",
"crates/example-tests",
"crates/typescript-tests",
"crates/web-sys",
"crates/webidl",

View File

@ -0,0 +1,18 @@
[package]
name = "example-tests"
version = "0.1.0"
authors = ["The wasm-bindgen Developers"]
edition = "2018"
[dependencies]
anyhow = "1.0.58"
futures-util = { version = "0.3.21", features = ["sink"] }
hyper = { version = "0.14.20", features = ["server", "tcp", "http1"] }
mozprofile = "0.8.0"
mozrunner = "0.14.0"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
tokio = { version = "1.20.0", features = ["macros", "time"] }
tokio-tungstenite = "0.17.2"
tower = { version = "0.4.13", features = ["make"] }
tower-http = { version = "0.3.4", features = ["fs"] }

View File

@ -0,0 +1 @@
../../LICENSE-APACHE

View File

@ -0,0 +1 @@
../../LICENSE-MIT

View File

@ -0,0 +1,6 @@
# example-tests
Tests that none of our examples are broken, by opening them in a browser
and checking that no errors get logged to the console.
This currently only supports Firefox.

View File

@ -0,0 +1,424 @@
use std::collections::VecDeque;
use std::fmt::{self, Display, Formatter, Write};
use std::net::TcpListener;
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::time::{Duration, Instant};
use std::{env, str};
use anyhow::{bail, Context};
use futures_util::{future, SinkExt, StreamExt};
use mozprofile::profile::Profile;
use mozrunner::firefox_default_path;
use mozrunner::runner::{FirefoxProcess, FirefoxRunner, Runner, RunnerProcess};
use serde::de::DeserializeOwned;
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
use tokio::net::TcpStream;
use tokio::sync::oneshot;
use tokio::time::timeout;
use tokio_tungstenite::tungstenite::{self, Message};
use tokio_tungstenite::{MaybeTlsStream, WebSocketStream};
use tower::make::Shared;
use tower_http::services::ServeDir;
/// A command sent from the client to the server.
#[derive(Serialize)]
struct BidiCommand<'a, T> {
id: u64,
method: &'a str,
params: T,
}
/// A message sent from the server to the client.
#[derive(Deserialize)]
#[serde(untagged)]
enum BidiMessage<R> {
CommandResponse {
id: u64,
#[serde(flatten)]
payload: CommandResult<R>,
},
Event(Event),
}
#[derive(Deserialize)]
#[serde(untagged)]
enum CommandResult<R> {
Ok { result: R },
Err(CommandError),
}
impl<R> From<CommandResult<R>> for Result<R, CommandError> {
fn from(res: CommandResult<R>) -> Self {
match res {
CommandResult::Ok { result } => Ok(result),
CommandResult::Err(e) => Err(e),
}
}
}
/// An error that occured while running a command.
#[derive(Serialize, Deserialize, Debug, Clone)]
struct CommandError {
/// The kind of error that occurred.
error: BidiErrorKind,
/// The message associated with the error.
message: String,
/// The stack trace associated with the error, if any.
stacktrace: Option<String>,
}
impl Display for CommandError {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
write!(f, "{}: {}", self.error, self.message)?;
if f.alternate() {
// Show the stack trace.
if let Some(stacktrace) = &self.stacktrace {
write!(f, "\n\nStack trace:\n{stacktrace}")?;
}
}
Ok(())
}
}
impl std::error::Error for CommandError {}
/// A kind of error that can occur while running a command.
#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)]
enum BidiErrorKind {
#[serde(rename = "unknown command")]
/// An unknown command was issued.
UnknownCommand,
/// An invalid argument was passed for a command.
#[serde(rename = "invalid argument")]
InvalidArgument,
/// Some other kind of error occured.
#[serde(rename = "unknown error")]
UnknownError,
}
impl Display for BidiErrorKind {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
match self {
BidiErrorKind::UnknownCommand => f.pad("unknown command"),
BidiErrorKind::InvalidArgument => f.pad("invalid argument"),
BidiErrorKind::UnknownError => f.pad("unknown error"),
}
}
}
/// An event sent from the server to the client.
#[derive(Deserialize)]
pub struct Event {
/// The name of the event.
method: String,
/// The payload of the event.
params: Value,
}
/// A connection to a WebDriver BiDi session.
struct WebDriver {
/// The WebSocket we're connected to the WebDriver implementation with.
ws: WebSocketStream<MaybeTlsStream<TcpStream>>,
/// The WebDriver process.
process: FirefoxProcess,
/// The ID that will be used for the next command.
next_id: u64,
/// Unyielded events.
events: VecDeque<Event>,
}
impl Drop for WebDriver {
fn drop(&mut self) {
self.process.kill().unwrap();
}
}
impl WebDriver {
async fn new() -> anyhow::Result<Self> {
// Make the OS assign us a random port by asking for port 0.
let driver_addr = TcpListener::bind("127.0.0.1:0")?.local_addr()?;
// For the moment, we're only supporting Firefox here.
let mut builder = FirefoxRunner::new(
&firefox_default_path().context("failed to find Firefox installation")?,
Some(Profile::new()?),
);
builder
.arg("--remote-debugging-port")
.arg(driver_addr.port().to_string())
.arg("--headless")
.stdout(Stdio::null())
.stderr(Stdio::null());
let process = builder
.start()
// `mozprofile` doesn't guarantee that its errors are `Send + Sync`,
// which means that they can't be converted to `anyhow::Error`.
// So, convert them to strings as a workaround.
.map_err(|e| anyhow::Error::msg(e.to_string()))?;
// Connect to the Firefox instance.
let start = Instant::now();
let ws = loop {
match tokio_tungstenite::connect_async(format!("ws://{driver_addr}/session")).await {
Ok((ws, _)) => break ws,
Err(e) => {
if start.elapsed() > Duration::from_secs(20) {
return Err(e).context("failed to connect to Firefox (after 20s)");
}
}
}
};
let mut this = WebDriver {
ws,
process,
next_id: 0,
events: VecDeque::new(),
};
// Start the session.
let _: Value = this
.issue_cmd(
"session.new",
json!({ "capabilities": { "unhandledPromptBehavior": "dismiss" } }),
)
.await?;
Ok(this)
}
async fn issue_cmd<T: Serialize, R: DeserializeOwned>(
&mut self,
method: &str,
params: T,
) -> anyhow::Result<R> {
let id = self.next_id;
self.next_id += 1;
let json = serde_json::to_string(&BidiCommand { id, method, params })
.context("failed to serialize message")?;
self.ws.send(Message::Text(json)).await?;
loop {
let msg = self
.ws
.next()
.await
.unwrap_or(Err(tungstenite::Error::AlreadyClosed))?;
let message: BidiMessage<R> = serde_json::from_str(&msg.into_text()?)?;
match message {
BidiMessage::CommandResponse {
id: response_id,
payload,
} => {
if response_id != id {
bail!("unexpected response to command {response_id} after sending command {id}")
}
return Result::from(payload).map_err(anyhow::Error::from);
}
BidiMessage::Event(event) => self.events.push_back(event),
}
}
}
async fn next_event(&mut self) -> anyhow::Result<Event> {
if let Some(event) = self.events.pop_front() {
Ok(event)
} else {
loop {
let msg = self
.ws
.next()
.await
.unwrap_or(Err(tungstenite::Error::AlreadyClosed))?;
let message: BidiMessage<Value> = serde_json::from_str(&msg.into_text()?)?;
match message {
BidiMessage::CommandResponse { .. } => bail!("unexpected command response"),
BidiMessage::Event(event) => return Ok(event),
}
}
}
}
}
/// Run a single example with the passed name, using the passed closure to
/// build it if prebuilt examples weren't provided.
pub async fn test_example(
name: &str,
build: impl FnOnce() -> anyhow::Result<PathBuf>,
) -> anyhow::Result<()> {
let path = if let Some(value) = env::var_os("EXBUILD") {
Path::new(&value).join(name)
} else {
build()?
};
let mut driver = WebDriver::new().await?;
// Serve the path.
let server = hyper::Server::try_bind(&"127.0.0.1:0".parse().unwrap())?
.serve(Shared::new(ServeDir::new(path)));
let addr = server.local_addr();
let (tx, rx) = oneshot::channel();
let (server_result, result) = future::join(
server.with_graceful_shutdown(async move {
let _ = rx.await;
}),
async {
#[derive(Deserialize)]
struct BrowsingContextCreateResult {
context: String,
}
let BrowsingContextCreateResult { context } = driver
.issue_cmd("browsingContext.create", json!({ "type": "tab" }))
.await?;
let _: Value = driver
.issue_cmd(
"session.subscribe",
json!({
"events": ["log.entryAdded"],
"contexts": [&context],
}),
)
.await?;
let _: Value = driver
.issue_cmd(
"browsingContext.navigate",
json!({
"context": &context,
"url": format!("http://{addr}"),
}),
)
.await?;
let start = Instant::now();
// Wait 5 seconds for any errors to occur.
const WAIT_DURATION: Duration = Duration::from_secs(5);
while start.elapsed() < WAIT_DURATION {
match timeout(WAIT_DURATION - start.elapsed(), driver.next_event()).await {
Ok(event) => {
let event = event?;
if event.method == "log.entryAdded" {
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
struct LogEntry {
level: LogLevel,
// source: Source,
text: Option<String>,
// timestamp: i64,
stack_trace: Option<StackTrace>,
// kind: LogEntryKind,
}
#[derive(Deserialize, Debug, PartialEq, Eq, Clone, Copy)]
#[serde(rename_all = "lowercase")]
enum LogLevel {
Debug,
Info,
Warning,
Error,
}
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
struct StackTrace {
call_frames: Vec<StackFrame>,
}
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
struct StackFrame {
column_number: i64,
function_name: String,
line_number: i64,
url: String,
}
impl Display for StackFrame {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
write!(
f,
"{} (at {}:{}:{})",
self.function_name,
self.url,
self.line_number,
self.column_number
)
}
}
let entry: LogEntry = serde_json::from_value(event.params)
.context("invalid log entry received")?;
if entry.level == LogLevel::Error {
if let Some(text) = entry.text {
let mut msg = format!("An error occured: {text}");
if let Some(stack_trace) = entry.stack_trace {
write!(msg, "\n\nStack trace:").unwrap();
for frame in stack_trace.call_frames {
write!(msg, "\n{frame}").unwrap();
}
}
bail!("{msg}")
} else {
bail!("An error occured")
}
}
}
}
Err(_) => break,
}
}
tx.send(()).unwrap();
Ok(())
},
)
.await;
server_result.context("error running file server")?;
result
}
pub fn run(command: &mut Command) -> anyhow::Result<()> {
// Format the command to use in errors.
let mut cmdline = command.get_program().to_string_lossy().to_string();
for arg in command.get_args().map(|arg| arg.to_string_lossy()) {
cmdline += " ";
cmdline += &arg;
}
let status = command.status()?;
if !status.success() {
bail!("`{cmdline}` failed with {status}");
}
Ok(())
}
/// Returns the path of root `wasm-bindgen` folder.
pub fn manifest_dir() -> &'static Path {
Path::new(env!("CARGO_MANIFEST_DIR"))
.parent()
.unwrap()
.parent()
.unwrap()
}
/// Returns the path of the example with the passed name.
pub fn example_dir(name: &str) -> PathBuf {
[manifest_dir(), "examples".as_ref(), name.as_ref()]
.iter()
.collect()
}

View File

@ -0,0 +1,56 @@
// Since these run on shell scripts, they won't work outside Unix-based OSes.
#![cfg(unix)]
use std::process::Command;
use std::str;
use example_tests::{example_dir, run, test_example};
async fn test_shell_example(name: &str) -> anyhow::Result<()> {
test_example(name, || {
let path = example_dir(name);
run(Command::new(path.join("build.sh")).current_dir(&path))?;
Ok(path)
})
.await
}
macro_rules! shell_tests {
($(
$(#[$attr:meta])*
$test:ident = $name:literal,
)*) => {
$(
$(#[$attr])*
#[tokio::test]
async fn $test() -> anyhow::Result<()> {
test_shell_example($name).await
}
)*
};
}
shell_tests! {
#[ignore = "This requires module workers, which Firefox doesn't support yet."]
synchronous_instantiation = "synchronous-instantiation",
wasm2js = "wasm2js",
wasm_in_web_worker = "wasm-in-web-worker",
websockets = "websockets",
without_a_bundler = "without-a-bundler",
without_a_bundler_no_modules = "without-a-bundler-no-modules",
}
#[tokio::test]
async fn raytrace_parallel() -> anyhow::Result<()> {
test_example("raytrace-parallel", || {
let path = example_dir("raytrace-parallel");
run(Command::new(path.join("build.sh"))
.current_dir(&path)
// This example requires nightly.
.env("RUSTUP_TOOLCHAIN", "nightly"))?;
Ok(path)
})
.await
}

View File

@ -0,0 +1,92 @@
use std::fs;
use std::io::ErrorKind;
use std::process::Command;
use std::sync::Once;
use std::{io, str};
use example_tests::{example_dir, manifest_dir, run, test_example};
async fn test_webpack_example(name: &str) -> anyhow::Result<()> {
test_example(name, || {
let manifest_dir = manifest_dir();
let path = example_dir(name);
fn allow_already_exists(e: io::Error) -> io::Result<()> {
if e.kind() == ErrorKind::AlreadyExists {
Ok(())
} else {
Err(e)
}
}
// All of the examples have the same dependencies, so we can just install
// to the root `node_modules` once, since Node resolves packages from any
// outer directories as well as the one containing the `package.json`.
static INSTALL: Once = Once::new();
INSTALL.call_once(|| {
fs::copy(
manifest_dir.join("_package.json"),
manifest_dir.join("package.json"),
)
.map(|_| ())
.or_else(allow_already_exists)
.unwrap();
run(Command::new("npm").arg("install").current_dir(manifest_dir)).unwrap();
fs::remove_file(manifest_dir.join("package.json")).unwrap();
});
// Build the example.
run(Command::new("npm")
.arg("run")
.arg("build")
.current_dir(&path))?;
Ok(path.join("dist"))
})
.await
}
macro_rules! webpack_tests {
($(
$(#[$attr:meta])*
$test:ident = $name:literal,
)*) => {
$(
$(#[$attr])*
#[tokio::test]
async fn $test() -> anyhow::Result<()> {
test_webpack_example($name).await
}
)*
};
}
webpack_tests! {
add = "add",
canvas = "canvas",
char = "char",
closures = "closures",
console_log = "console_log",
dom = "dom",
duck_typed_interfaces = "duck-typed-interfaces",
fetch = "fetch",
guide_supported_types_examples = "guide-supported-types-examples",
hello_world = "hello_world",
import_js = "import_js",
julia_set = "julia_set",
paint = "paint",
performance = "performance",
request_animation_frame = "request-animation-frame",
todomvc = "todomvc",
wasm_in_wasm_imports = "wasm-in-wasm-imports",
wasm_in_wasm = "wasm-in-wasm",
weather_report = "weather_report",
webaudio = "webaudio",
#[ignore = "The CI virtual machines don't have GPUs, so this doesn't work there."]
webgl = "webgl",
webrtc_datachannel = "webrtc_datachannel",
#[ignore = "WebXR isn't supported in Firefox yet"]
webxr = "webxr",
}

View File

@ -9,7 +9,7 @@ online][compiled]
You can build the example locally with:
```
$ ./build.sh
$ ./run.sh
```
(or running the commands on Windows manually)

View File

@ -21,5 +21,3 @@ cargo run -p wasm-bindgen-cli -- \
../../target/wasm32-unknown-unknown/release/raytrace_parallel.wasm \
--out-dir . \
--target no-modules
python3 server.py

View File

@ -0,0 +1,7 @@
#!/bin/sh
set -ex
./build.sh
python3 server.py

View File

@ -3,4 +3,3 @@
set -ex
wasm-pack build --target web
python3 -m http.server

View File

@ -4,4 +4,4 @@ set -ex
# This example requires to *not* create ES modules, therefore we pass the flag
# `--target no-modules`
wasm-pack build --out-dir www/pkg --target no-modules
wasm-pack build --target no-modules

View File

@ -11,5 +11,3 @@ wasm2js pkg/wasm2js_bg.wasm -o pkg/wasm2js_bg.wasm.js
# Update our JS shim to require the JS file instead
sed -i 's/wasm2js_bg.wasm/wasm2js_bg.wasm.js/' pkg/wasm2js.js
sed -i 's/wasm2js_bg.wasm/wasm2js_bg.wasm.js/' pkg/wasm2js_bg.js
http

5
examples/websockets/build.sh Executable file
View File

@ -0,0 +1,5 @@
#!/bin/sh
set -ex
wasm-pack build --target web

View File

@ -0,0 +1,5 @@
#!/bin/sh
set -ex
wasm-pack build --target no-modules

View File

@ -3,4 +3,3 @@
set -ex
wasm-pack build --target web
python3 -m http.server