Headless browser testing infrastructure (#371)

* tests: Add newlines between impl methods for Project

* WIP headless browser testing with geckodriver and selenium

* Get some more of headless testing working

* Extract `console.log` invocations and print them from the console
* Ship the error message from an exception from the browser back to the command
  line
* Cleanup some "if headless" and `else` branches
* Fix killing `webpack-dev-server` in the background with `--watch-stdin`

* Fix path appending logic for Windows

* Always log logs/errors in headless mode

* Install Firefox on Travis

* Don't duplicate full test suite with `yarn`

No need to run that many tests, we should be able to get by with a smoke test
that it just works.

* headless tests: Move `run-headless.js` to its own file and `include_str!` it

* Run `rustfmt` on `tests/all/main.rs`

* guide: Add note about headless browser tests and configuration

* test: Log WASM_BINDGEN_FIREFOX_BIN_PATH in run-headless.js

* TEMP only run add_headless test in CI

* Add more logging to headless testing

* Allow headless tests to run for 60 seconds before timeout

* TEMP add logging to add_headless test

* Fix headless browser tests

* Another attempt to fix Travis

* More attempts at debugging

* Fix more merge conflicts

* Touch up an error message

* Fixup travis again

* Enable all travis tests again

* Test everything on AppVeyor

.travis.yml

@@ -41,6 +41,8 @@ matrix:
         - cargo test
         # Check JS output from all tests against eslint
         - ./node_modules/.bin/eslint ./target/generated-tests/*/out*js
+      addons:
+        firefox: latest
 
     # All examples work
     - rust: nightly

Cargo.toml

@@ -33,6 +33,7 @@ serde = { version = "1.0", optional = true }
 serde_json = { version = "1.0", optional = true }
 
 [dev-dependencies]
+lazy_static = "1"
 wasm-bindgen-cli-support = { path = "crates/cli-support", version = '=0.2.11' }
 
 [workspace]

guide/src/contributing.md

@@ -42,3 +42,16 @@ Finally, you can run the tests with `cargo`:
 ```shell
 cargo test
 ```
+
+### Headless Browser Tests
+
+Some tests are configured to run in a headless Firefox instance. To run these
+tests, you must have Firefox installed. If you have Firefox installed in a
+non-default, custom location you can set the `WASM_BINDGEN_FIREFOX_BIN_PATH`
+environment variable to the path to your `firefox-bin`.
+
+For example:
+
+```shell
+WASM_BINDGEN_FIREFOX_BIN_PATH=/home/fitzgen/firefox/firefox-bin cargo test
+```

package.json

@@ -1,15 +1,20 @@
 {
   "license": "MIT",
   "scripts": {
-    "run-webpack": "webpack"
+    "run-webpack": "webpack",
+    "run-webpack-dev-server": "webpack-dev-server",
+    "run-geckodriver": "geckodriver"
   },
   "devDependencies": {
     "@types/node": "^9.4.6",
+    "eslint": "^5.0.1",
+    "geckodriver": "^1.11.0",
+    "selenium-webdriver": "^4.0.0-alpha.1",
     "ts-loader": "^4.0.1",
     "typescript": "^2.7.2",
     "webpack": "^4.11.1",
     "webpack-cli": "^2.0.10",
-    "babel-eslint": "^8.2.5",
+    "webpack-dev-server": "^3.1.4",
-    "eslint": "^5.0.1"
+    "babel-eslint": "^8.2.5"
   }
 }

tests/all/main.rs

@@ -1,3 +1,5 @@
+#[macro_use]
+extern crate lazy_static;
 extern crate wasm_bindgen_cli_support;
 
 mod project_builder;

tests/all/project_builder.rs

@@ -2,12 +2,14 @@ use wasm_bindgen_cli_support as cli;
 
 use std::env;
 use std::fs::{self, File};
+use std::thread;
 use std::io::{self, Read, Write};
 use std::path::{Path, PathBuf};
-use std::process::{Command, Stdio};
+use std::process::{Command, Stdio, Child, ChildStdin};
+use std::net::TcpStream;
 use std::sync::atomic::*;
-use std::sync::{Once, ONCE_INIT};
+use std::sync::{Once, ONCE_INIT, Mutex};
-use std::time::Instant;
+use std::time::{Duration, Instant};
 
 static CNT: AtomicUsize = ATOMIC_USIZE_INIT;
 thread_local!(static IDX: usize = CNT.fetch_add(1, Ordering::SeqCst));
@@ -23,6 +25,7 @@ pub struct Project {
     webpack: bool,
     node_args: Vec<String>,
     deps: Vec<String>,
+    headless: bool,
 }
 
 pub fn project() -> Project {
@@ -40,6 +43,7 @@ pub fn project() -> Project {
         webpack: false,
         serde: false,
         rlib: false,
+        headless: false,
         deps: Vec::new(),
         node_args: Vec::new(),
         files: vec![
@@ -164,9 +168,18 @@ impl Project {
         self
     }
 
+    /// This test requires a headless web browser
+    pub fn headless(&mut self, headless: bool) -> &mut Project {
+        self.headless = headless;
+        self
+    }
+
     /// Write this project to the filesystem, ensuring all files are ready to
     /// go.
     pub fn build(&mut self) -> (PathBuf, PathBuf) {
+        if self.headless {
+            self.webpack = true;
+        }
         if self.webpack {
             self.node = false;
             self.nodejs_experimental_modules = false;
@@ -175,6 +188,11 @@ impl Project {
         self.ensure_webpack_config();
         self.ensure_test_entry();
 
+        if self.headless {
+            self.ensure_index_html();
+            self.ensure_run_headless_js();
+        }
+
         let webidl_modules = self.generate_webidl_bindings();
         self.generate_js_entry(webidl_modules);
 
@@ -262,6 +280,7 @@ impl Project {
             ");
             extensions.push_str(", '.ts'");
         }
+        let target = if self.headless { "web" } else { "node" };
         self.files.push((
             "webpack.config.js".to_string(),
             format!(r#"
@@ -276,14 +295,16 @@ impl Project {
                 // This reads the directories in `node_modules`
                 // and give that to externals and webpack ignores
                 // to bundle the modules listed as external.
-                fs.readdirSync('node_modules')
+                if ('{2}' == 'node') {{
-                    .filter(module => module !== '.bin')
+                    fs.readdirSync('node_modules')
-                    .forEach(mod => {{
+                        .filter(module => module !== '.bin')
-                        // External however,expects browser environment.
+                        .forEach(mod => {{
-                        // To make it work in `node` target we
+                            // External however,expects browser environment.
-                        // prefix commonjs here.
+                            // To make it work in `node` target we
-                        nodeModules[mod] = 'commonjs ' + mod;
+                            // prefix commonjs here.
-                    }});
+                            nodeModules[mod] = 'commonjs ' + mod;
+                        }});
+                }}
 
                 module.exports = {{
                   entry: './run.js',
@@ -299,10 +320,10 @@ impl Project {
                     filename: 'bundle.js',
                     path: path.resolve(__dirname, '.')
                   }},
-                  target: 'node',
+                  target: '{}',
                   externals: nodeModules
                 }};
-            "#, rules, extensions)
+            "#, rules, extensions, target)
         ));
         if needs_typescript {
             self.files.push((
@@ -342,6 +363,27 @@ impl Project {
         }
     }
 
+    fn ensure_index_html(&mut self) {
+        self.file(
+            "index.html",
+            r#"
+                <!DOCTYPE html>
+                <html>
+                    <body>
+                        <div id="error"></div>
+                        <div id="logs"></div>
+                        <div id="status"></div>
+                        <script src="bundle.js"></script>
+                    </body>
+                </html>
+            "#,
+        );
+    }
+
+    fn ensure_run_headless_js(&mut self) {
+        self.file("run-headless.js", include_str!("run-headless.js"));
+    }
+
     fn generate_webidl_bindings(&mut self) -> Vec<PathBuf> {
         let mut res = Vec::new();
         let mut origpaths = Vec::new();
@@ -413,7 +455,20 @@ impl Project {
         let mut runjs = String::new();
         let esm_imports = self.webpack || !self.node || self.nodejs_experimental_modules;
 
-        if esm_imports {
+
+        if self.headless {
+            runjs.push_str(
+                r#"
+                    window.document.body.innerHTML += "\nTEST_START\n";
+                    console.log = function(...args) {
+                        const logs = document.getElementById('logs');
+                        for (let msg of args) {
+                            logs.innerHTML += `${msg}<br/>\n`;
+                        }
+                    };
+                "#,
+            );
+        } else if esm_imports {
             runjs.push_str("import * as process from 'process';\n");
         } else {
             runjs.push_str("const process = require('process');\n");
@@ -428,14 +483,27 @@ impl Project {
                 if (wasm.assertSlabEmpty)
                     wasm.assertSlabEmpty();
             }
-
-            function onerror(error) {
-                console.error(error);
-                process.exit(1);
-            }
         ");
 
+        if self.headless {
+            runjs.push_str("
+                function onerror(error) {
+                    const errors = document.getElementById('error');
+                    let content = `exception: ${e.message}\\nstack: ${e.stack}`;
+                    errors.innerHTML = `<pre>${content}</pre>`;
+                }
+            ");
+        } else {
+            runjs.push_str("
+                function onerror(error) {
+                    console.error(error);
+                    process.exit(1);
+                }
+            ");
+        }
+
         if esm_imports {
+            runjs.push_str("console.log('importing modules...');\n");
             runjs.push_str("const modules = [];\n");
             for module in modules.iter() {
                 runjs.push_str(&format!("modules.push(import('./{}'))",
@@ -453,8 +521,22 @@ impl Project {
                         return Promise.all([import('./test'), {}])
                     }})
                     .then(result => run(result[0], result[1]))
-                    .catch(onerror);
             ", import_wasm));
+
+            if self.headless {
+                runjs.push_str(".then(() => {
+                    document.getElementById('status').innerHTML = 'good';
+                })");
+            }
+            runjs.push_str(".catch(onerror)\n");
+
+            if self.headless {
+                runjs.push_str("
+                    .finally(() => {
+                        window.document.body.innerHTML += \"\\nTEST_DONE\";
+                    })
+                ");
+            }
         } else {
             assert!(!self.debug);
             assert!(modules.is_empty());
@@ -561,15 +643,12 @@ impl Project {
         let cwd = env::current_dir().unwrap();
         symlink_dir(&cwd.join("node_modules"), &root.join("node_modules")).unwrap();
 
+        if self.headless {
+            return self.test_headless(&root)
+        }
+
         // Execute webpack to generate a bundle
-        let mut cmd = if cfg!(windows) {
+        let mut cmd = self.npm();
-            let mut c = Command::new("cmd");
-            c.arg("/c");
-            c.arg("npm");
-            c
-        } else {
-            Command::new("npm")
-        };
         cmd.arg("run").arg("run-webpack").current_dir(&root);
         run(&mut cmd, "npm");
 
@@ -579,6 +658,54 @@ impl Project {
         run(&mut cmd, "node");
     }
 
+    fn npm(&self) -> Command {
+        if cfg!(windows) {
+            let mut c = Command::new("cmd");
+            c.arg("/c");
+            c.arg("npm");
+            c
+        } else {
+            Command::new("npm")
+        }
+    }
+
+    fn test_headless(&mut self, root: &Path) {
+        // Serialize all headless tests since they require starting
+        // webpack-dev-server on the same port.
+        lazy_static! {
+            static ref MUTEX: Mutex<()> = Mutex::new(());
+        }
+        let _lock = MUTEX.lock().unwrap();
+
+        let mut cmd = self.npm();
+        cmd.arg("run")
+            .arg("run-webpack-dev-server")
+            .arg("--")
+            .arg("--quiet")
+            .arg("--watch-stdin")
+            .current_dir(&root);
+        let _server = run_in_background(&mut cmd, "webpack-dev-server".into());
+
+        // wait for webpack-dev-server to come online and bind its port
+        loop {
+            if TcpStream::connect("127.0.0.1:8080").is_ok() {
+                break;
+            }
+            thread::sleep(Duration::from_millis(100));
+        }
+
+        let path = env::var_os("PATH").unwrap_or_default();
+        let mut path = env::split_paths(&path).collect::<Vec<_>>();
+        path.push(root.join("node_modules/geckodriver"));
+
+        let mut cmd = Command::new("node");
+        cmd.args(&self.node_args)
+            .arg(root.join("run-headless.js"))
+            .current_dir(&root)
+            .env("PATH", env::join_paths(&path).unwrap());
+        run(&mut cmd, "node");
+    }
+
     /// Reads JS generated by `wasm-bindgen` to a string.
     pub fn read_js(&self) -> String {
         let path = root().join(if self.nodejs_experimental_modules {
@@ -638,3 +765,65 @@ pub fn run(cmd: &mut Command, program: &str) {
     assert!(output.status.success());
 }
 
+struct BackgroundChild {
+    name: String,
+    child: Child,
+    stdin: Option<ChildStdin>,
+    stdout: Option<thread::JoinHandle<io::Result<String>>>,
+    stderr: Option<thread::JoinHandle<io::Result<String>>>,
+}
+
+impl Drop for BackgroundChild {
+    fn drop(&mut self) {
+        drop(self.stdin.take());
+        let status = self.child.wait().expect("failed to wait on child");
+        let stdout = self
+            .stdout
+            .take()
+            .unwrap()
+            .join()
+            .unwrap()
+            .expect("failed to read stdout");
+        let stderr = self
+            .stderr
+            .take()
+            .unwrap()
+            .join()
+            .unwrap()
+            .expect("failed to read stderr");
+
+        println!("···················································");
+        println!("background {}", self.name);
+        println!("status: {}", status);
+        println!("stdout ---\n{}", stdout);
+        println!("stderr ---\n{}", stderr);
+    }
+}
+
+fn run_in_background(cmd: &mut Command, name: String) -> BackgroundChild {
+    cmd.stdout(Stdio::piped());
+    cmd.stderr(Stdio::piped());
+    cmd.stdin(Stdio::piped());
+    let mut child = cmd.spawn().expect(&format!("should spawn {} OK", name));
+    let mut stdout = child.stdout.take().unwrap();
+    let mut stderr = child.stderr.take().unwrap();
+    let stdin = child.stdin.take().unwrap();
+    let stdout = thread::spawn(move || {
+        let mut t = String::new();
+        stdout.read_to_string(&mut t)?;
+        Ok(t)
+    });
+    let stderr = thread::spawn(move || {
+        let mut t = String::new();
+        stderr.read_to_string(&mut t)?;
+        Ok(t)
+    });
+    BackgroundChild {
+        name,
+        child,
+        stdout: Some(stdout),
+        stderr: Some(stderr),
+        stdin: Some(stdin),
+    }
+}
+

tests/all/run-headless.js

@@ -0,0 +1,128 @@
+const process = require("process");
+const { promisify } = require("util");
+const { Builder, By, Key, logging, promise, until } = require("selenium-webdriver");
+const firefox = require("selenium-webdriver/firefox");
+
+promise.USE_PROMISE_MANAGER = false;
+
+const prefs = new logging.Preferences();
+prefs.setLevel(logging.Type.BROWSER, logging.Level.DEBUG);
+
+const opts = new firefox.Options();
+opts.headless();
+if (process.env.WASM_BINDGEN_FIREFOX_BIN_PATH) {
+  console.log("Using custom firefox-bin: $WASM_BINDGEN_FIREFOX_BIN_PATH =",
+              process.env.WASM_BINDGEN_FIREFOX_BIN_PATH);
+  opts.setBinary(process.env.WASM_BINDGEN_FIREFOX_BIN_PATH);
+}
+
+console.log("Using Firefox options:", opts);
+
+const driver = new Builder()
+  .forBrowser("firefox")
+  .setFirefoxOptions(opts)
+  .build();
+
+const SECONDS = 1000;
+const MINUTES = 60 * SECONDS;
+
+const start = Date.now();
+const timeSinceStart = () => {
+  const elapsed = Date.now() - start;
+  const minutes = Math.floor(elapsed / MINUTES);
+  const seconds = elapsed % MINUTES / SECONDS;
+  return `${minutes}m${seconds.toFixed(3)}s`;
+};
+
+async function logged(msg, promise) {
+  console.log(`${timeSinceStart()}: START: ${msg}`);
+  try {
+    const value = await promise;
+    console.log(`${timeSinceStart()}: END: ${msg}`);
+    return value;
+  } catch (e) {
+    console.log(`${timeSinceStart()}: ERROR: ${msg}: ${e}\n\n${e.stack}`);
+    throw e;
+  }
+}
+
+async function main() {
+  let body;
+  try {
+    await logged(
+      "load http://localhost:8080/index.html",
+      driver.get("http://localhost:8080/index.html")
+    );
+
+    body = driver.findElement(By.tagName("body"));
+
+    await logged(
+      "Waiting for <body> to include text 'TEST_START'",
+      driver.wait(
+        until.elementTextContains(body, "TEST_START"),
+        1 * MINUTES
+      )
+    );
+
+    await logged(
+      "Waiting for <body> to include text 'TEST_DONE'",
+      driver.wait(
+        until.elementTextContains(body, "TEST_DONE"),
+        1 * MINUTES
+      )
+    );
+
+    const status = await logged(
+      "get #status text",
+      body.findElement(By.id("status")).getText()
+    );
+
+    console.log(`Test status is: "${status}"`);
+    if (status != "good") {
+      throw new Error(`test failed with status = ${status}`);
+    }
+  } finally {
+    const logs = await logged(
+      "getting browser logs",
+      body.findElement(By.id("logs")).getText()
+    );
+
+    if (logs.length > 0) {
+      console.log("logs:");
+      logs.split("\n").forEach(line => {
+        console.log(`    ${line}`);
+      });
+    }
+
+    const errors = await logged(
+      "getting browser errors",
+      body.findElement(By.id("error")).getText()
+    );
+
+    if (errors.length > 0) {
+      console.log("errors:");
+      errors.split("\n").forEach(line => {
+        console.log(`    ${line}`);
+      });
+    }
+
+    const bodyText = await logged(
+      "getting browser body",
+      body.getText()
+    );
+
+    if (bodyText.length > 0) {
+      console.log("body:");
+      bodyText.split("\n").forEach(line => {
+        console.log(`    ${line}`);
+      });
+    }
+  }
+}
+
+main()
+  .finally(() => driver.quit())
+  .catch(e => {
+    console.error(`Got an error: ${e}\n\nStack: ${e.stack}`);
+    process.exit(1);
+  });

tests/all/simple.rs

@@ -59,6 +59,39 @@ fn add() {
         .test();
 }
 
+#[test]
+fn add_headless() {
+    project()
+        .headless(true)
+        .file(
+            "src/lib.rs",
+            r#"
+                #![feature(proc_macro, wasm_custom_section)]
+                extern crate wasm_bindgen;
+                use wasm_bindgen::prelude::*;
+
+                #[wasm_bindgen]
+                pub fn add(a: u32, b: u32) -> u32 {
+                    a + b
+                }
+            "#,
+        )
+        .file(
+            "test.js",
+            r#"
+                import * as assert from "assert";
+                import * as wasm from "./out";
+
+                export function test() {
+                    console.log("start `add_headless` test");
+                    assert.strictEqual(wasm.add(1, 2), 3);
+                    console.log("end `add_headless` test");
+               }
+            "#,
+        )
+        .test();
+}
+
 #[test]
 fn string_arguments() {
     project()