Gate web-sys APIs on activated features (#790)

* Gate `web-sys` APIs on activated features

Currently the compile times of `web-sys` are unfortunately prohibitive,
increasing the barrier to using it. This commit updates the crate to instead
have all APIs gated by a set of Cargo features which affect what bindings are
generated at compile time (and which are then compiled by rustc). It's
significantly faster to activate only a handful of features vs all thousand of
them!

A magical env var is added to print the list of all features that should be
generated, and then necessary logic is added to ferry features from the build
script to the webidl crate which then uses that as a filter to remove items
after parsing. Currently parsing is pretty speedy so we'll unconditionally parse
all WebIDL files, but this may change in the future!

For now this will make the `web-sys` crate a bit less ergonomic to use as lots
of features will need to be specified, but it should make it much more
approachable in terms of first-user experience with compile times.

* Fix AppVeyor testing web-sys

* FIx a typo

* Udpate feature listings from rebase conflicts

* Add some crate docs and such
This commit is contained in:
Alex Crichton 2018-09-05 12:55:30 -07:00 committed by GitHub
parent c6d3011cff
commit 269c491380
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 1424 additions and 88 deletions

View File

@ -25,7 +25,7 @@ test_script:
- where chromedriver
- set CHROMEDRIVER=C:\Tools\WebDriver\chromedriver.exe
- cargo test -p js-sys --target wasm32-unknown-unknown
- cargo test -p web-sys --target wasm32-unknown-unknown
- cargo test --manifest-path crates/web-sys/Cargo.toml --target wasm32-unknown-unknown --all-features
- cargo test -p webidl-tests --target wasm32-unknown-unknown
branches:

View File

@ -97,8 +97,15 @@ matrix:
- *INSTALL_CHROMEDRIVER
script:
- export RUST_LOG=wasm_bindgen_test_runner
- CHROMEDRIVER=`pwd`/chromedriver cargo test -p web-sys --target wasm32-unknown-unknown
- GECKODRIVER=`pwd`/geckodriver cargo test -p web-sys --target wasm32-unknown-unknown
# Test out builds with just a few features
- cargo build --manifest-path crates/web-sys/Cargo.toml --target wasm32-unknown-unknown
- cargo build --manifest-path crates/web-sys/Cargo.toml --target wasm32-unknown-unknown --features Node
- cargo build --manifest-path crates/web-sys/Cargo.toml --target wasm32-unknown-unknown --features Element
- cargo build --manifest-path crates/web-sys/Cargo.toml --target wasm32-unknown-unknown --features Window
# Now run all the tests with all the features
- CHROMEDRIVER=`pwd`/chromedriver cargo test --manifest-path crates/web-sys/Cargo.toml --target wasm32-unknown-unknown --all-features
- GECKODRIVER=`pwd`/geckodriver cargo test --manifest-path crates/web-sys/Cargo.toml --target wasm32-unknown-unknown --all-features
addons:
firefox: latest
chrome: stable
@ -176,7 +183,10 @@ matrix:
- cargo install-update -a
script:
- (cd guide && mdbook build)
- cargo doc --no-deps -p wasm-bindgen -p web-sys -p js-sys -p wasm-bindgen-futures
- cargo doc --no-deps
- cargo doc --no-deps --manifest-path crates/js-sys/Cargo.toml
- cargo doc --no-deps --manifest-path crates/futures/Cargo.toml
- cargo doc --no-deps --manifest-path crates/web-sys/Cargo.toml --all-features
- mv target/doc guide/book/api
deploy:
provider: pages

View File

@ -77,6 +77,15 @@ where
}
}
impl<'a, T: ImportedTypes> ImportedTypes for &'a T {
fn imported_types<F>(&self, f: &mut F)
where
F: FnMut(&Ident, ImportedTypeKind),
{
(*self).imported_types(f)
}
}
impl ImportedTypes for ast::Program {
fn imported_types<F>(&self, f: &mut F)
where

View File

@ -15,9 +15,12 @@ macro_rules! bail_span {
)
}
#[derive(Debug)]
pub struct Diagnostic {
inner: Repr,
}
#[derive(Debug)]
enum Repr {
Single {
text: String,

View File

@ -103,6 +103,7 @@ pub fn wrap_import_function(function: ast::ImportFunction) -> ast::Import {
///
/// Hashes the public field here along with a few cargo-set env vars to
/// distinguish between runs of the procedural macro.
#[derive(Debug)]
pub struct ShortHash<T>(pub T);
impl<T: Hash> fmt::Display for ShortHash<T> {

File diff suppressed because it is too large Load Diff

View File

@ -1,9 +1,20 @@
# `web-sys`
[Documentation](https://rustwasm.github.io/wasm-bindgen/api/web_sys/)
Raw bindings to Web APIs for projects using `wasm-bindgen`.
The book: https://rustwasm.github.io/wasm-bindgen/web-sys.html
## Crate features
This crate by default contains very little when compiled as almost all of its
exposed APIs are gated by Cargo features. The exhaustive list of features can be
found in `crates/web-sys/Cargo.toml`, but the rule of thumb for `web-sys` is
that each type has its own cargo feature (named after the type). Using an API
requires enabling the features for all types used in the API, and APIs should
mention in the documentation what features they require.
## Tested WebIDL bindings
Below is a list of all the WebIDL files we want to generate bindings for, with a `x` where the

View File

@ -6,10 +6,11 @@ extern crate sourcefile;
use failure::{Fail, ResultExt};
use sourcefile::SourceFile;
use std::collections::HashSet;
use std::env;
use std::ffi::OsStr;
use std::fs;
use std::path;
use std::path::{self, PathBuf};
use std::process::{self, Command};
fn main() {
@ -41,7 +42,40 @@ fn try_main() -> Result<(), failure::Error> {
.with_context(|_| format!("reading contents of file \"{}\"", path.display()))?;
}
let bindings = match wasm_bindgen_webidl::compile(&source.contents) {
// Read our manifest, learn all `[feature]` directives with "toml parsing".
// Use all these names to match against environment variables set by Cargo
// to figure out which features are activated to we can pass that down to
// the webidl compiler.
let manifest_dir = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap());
let manifest = fs::read_to_string(manifest_dir.join("Cargo.toml"))?;
let features = manifest.lines().skip_while(|f| !f.starts_with("[features]"));
let enabled_features = env::vars()
.map(|p| p.0)
.filter(|p| p.starts_with("CARGO_FEATURE_"))
.map(|mut p| {
p.drain(0.."CARGO_FEATURE_".len());
p
})
.collect::<HashSet<_>>();
let mut allowed = Vec::new();
for feature in features.filter(|f| !f.starts_with("#") && !f.starts_with("[")) {
let mut parts = feature.split('=');
let name = parts.next().unwrap().trim();
if enabled_features.contains(&name.to_uppercase()) {
allowed.push(name);
}
}
// If we're printing all features don't filter anything
let allowed = if env::var("__WASM_BINDGEN_DUMP_FEATURES").is_ok() {
None
} else {
Some(&allowed[..])
};
let bindings = match wasm_bindgen_webidl::compile(&source.contents, allowed) {
Ok(bindings) => bindings,
Err(e) => match e.kind() {
wasm_bindgen_webidl::ErrorKind::ParsingWebIDLSourcePos(pos) => {

View File

@ -1,3 +1,16 @@
//! Raw API bindings for Web APIs
//!
//! This is a procedurally generated crate from browser WebIDL which provides a
//! binding to all APIs that browser provide on the web.
//!
//! This crate by default contains very little when compiled as almost all of
//! its exposed APIs are gated by Cargo features. The exhaustive list of
//! features can be found in `crates/web-sys/Cargo.toml`, but the rule of thumb
//! for `web-sys` is that each type has its own cargo feature (named after the
//! type). Using an API requires enabling the features for all types used in the
//! API, and APIs should mention in the documentation what features they
//! require.
#![doc(html_root_url = "https://docs.rs/web-sys/0.2")]
extern crate wasm_bindgen;

View File

@ -17,7 +17,7 @@ fn main() {
let out_dir = PathBuf::from(env::var_os("OUT_DIR").unwrap());
for (i, (idl, path)) in idls.enumerate() {
println!("processing {:?}", path);
let mut generated_rust = wasm_bindgen_webidl::compile(&idl).unwrap();
let mut generated_rust = wasm_bindgen_webidl::compile(&idl, None).unwrap();
let out_file = out_dir.join(path.file_name().unwrap())
.with_extension("rs");

View File

@ -10,6 +10,7 @@
use std::cmp::Ordering;
use std::collections::{BTreeMap, BTreeSet};
use proc_macro2::Ident;
use weedle::{DictionaryDefinition, PartialDictionaryDefinition};
use weedle::argument::Argument;
use weedle::attribute::*;
@ -24,6 +25,7 @@ use util;
/// Collection of constructs that may use partial.
#[derive(Default)]
pub(crate) struct FirstPassRecord<'src> {
pub(crate) builtin_idents: BTreeSet<Ident>,
pub(crate) interfaces: BTreeMap<&'src str, InterfaceData<'src>>,
pub(crate) enums: BTreeMap<&'src str, &'src weedle::EnumDefinition<'src>>,
/// The mixins, mapping their name to the webidl ast node for the mixin.
@ -701,8 +703,10 @@ impl<'a> FirstPassRecord<'a> {
Some(class) => class,
None => return,
};
if set.insert(camel_case_ident(superclass)) {
self.fill_superclasses(superclass, set);
if self.interfaces.contains_key(superclass) {
if set.insert(camel_case_ident(superclass)) {
self.fill_superclasses(superclass, set);
}
}
}

View File

@ -29,18 +29,16 @@ mod idl_type;
mod util;
mod error;
use std::collections::BTreeSet;
use std::collections::{BTreeSet, HashSet, BTreeMap};
use std::env;
use std::fs;
use std::io::{self, Read};
use std::iter::FromIterator;
use std::path::Path;
use backend::ast;
use backend::TryToTokens;
use backend::defined::{ImportedTypeDefinitions, RemoveUndefinedImports};
use backend::defined::ImportedTypeReferences;
use backend::util::{ident_ty, rust_ident, raw_ident, wrap_import_function};
use failure::ResultExt;
use proc_macro2::{Ident, Span};
use weedle::attribute::{ExtendedAttributeList};
use weedle::dictionary::DictionaryMember;
@ -52,17 +50,10 @@ use idl_type::ToIdlType;
pub use error::{Error, ErrorKind, Result};
/// Parse the WebIDL at the given path into a wasm-bindgen AST.
fn parse_file(webidl_path: &Path) -> Result<backend::ast::Program> {
let file = fs::File::open(webidl_path).context(ErrorKind::OpeningWebIDLFile)?;
let mut file = io::BufReader::new(file);
let mut source = String::new();
file.read_to_string(&mut source).context(ErrorKind::ReadingWebIDLFile)?;
parse(&source)
}
/// Parse a string of WebIDL source text into a wasm-bindgen AST.
fn parse(webidl_source: &str) -> Result<backend::ast::Program> {
fn parse(webidl_source: &str, allowed_types: Option<&[&str]>)
-> Result<backend::ast::Program>
{
let definitions = match weedle::parse(webidl_source) {
Ok(def) => def,
Err(e) => {
@ -84,10 +75,23 @@ fn parse(webidl_source: &str) -> Result<backend::ast::Program> {
}
};
let mut first_pass_record = Default::default();
let mut first_pass_record: FirstPassRecord = Default::default();
first_pass_record.builtin_idents = builtin_idents();
definitions.first_pass(&mut first_pass_record, ())?;
let mut program = Default::default();
// Prune out everything in the `first_pass_record` which isn't allowed, or
// is otherwise gated from not actually being generated.
if let Some(allowed_types) = allowed_types {
let allowed = allowed_types.iter().cloned().collect::<HashSet<_>>();
let filter = |name: &&str| {
allowed.contains(&camel_case_ident(name)[..])
};
retain(&mut first_pass_record.enums, &filter);
retain(&mut first_pass_record.dictionaries, &filter);
retain(&mut first_pass_record.interfaces, &filter);
}
for e in first_pass_record.enums.values() {
first_pass_record.append_enum(&mut program, e);
}
@ -104,43 +108,70 @@ fn parse(webidl_source: &str) -> Result<backend::ast::Program> {
Ok(program)
}
/// Compile the given WebIDL file into Rust source text containing
/// `wasm-bindgen` bindings to the things described in the WebIDL.
pub fn compile_file(webidl_path: &Path) -> Result<String> {
let ast = parse_file(webidl_path)?;
Ok(compile_ast(ast))
fn retain<K: Copy + Ord, V>(
map: &mut BTreeMap<K, V>,
mut filter: impl FnMut(&K) -> bool,
) {
let mut to_remove = Vec::new();
for k in map.keys() {
if !filter(k) {
to_remove.push(*k);
}
}
for k in to_remove {
map.remove(&k);
}
}
/// Compile the given WebIDL source text into Rust source text containing
/// `wasm-bindgen` bindings to the things described in the WebIDL.
pub fn compile(webidl_source: &str) -> Result<String> {
let ast = parse(webidl_source)?;
pub fn compile(
webidl_source: &str,
allowed_types: Option<&[&str]>,
) -> Result<String> {
let ast = parse(webidl_source, allowed_types)?;
Ok(compile_ast(ast))
}
fn builtin_idents() -> BTreeSet<Ident> {
BTreeSet::from_iter(
vec![
"str", "char", "bool", "JsValue", "u8", "i8", "u16", "i16", "u32", "i32", "u64", "i64",
"usize", "isize", "f32", "f64", "Result", "String", "Vec", "Option",
"ArrayBuffer", "Object", "Promise",
].into_iter()
.map(|id| proc_macro2::Ident::new(id, proc_macro2::Span::call_site())),
)
}
/// Run codegen on the AST to generate rust code.
fn compile_ast(mut ast: backend::ast::Program) -> String {
// Iteratively prune all entries from the AST which reference undefined
// fields. Each pass may remove definitions of types and so we need to
// reexecute this pass to see if we need to keep removing types until we
// reach a steady state.
let builtin = BTreeSet::from_iter(
vec![
"str", "char", "bool", "JsValue", "u8", "i8", "u16", "i16", "u32", "i32", "u64", "i64",
"usize", "isize", "f32", "f64", "Result", "String", "Vec", "Option",
"ArrayBuffer", "Object", "Promise",
].into_iter()
.map(|id| proc_macro2::Ident::new(id, proc_macro2::Span::call_site())),
);
let builtin = builtin_idents();
let mut all_definitions = BTreeSet::new();
let track = env::var_os("__WASM_BINDGEN_DUMP_FEATURES");
loop {
let mut defined = builtin.clone();
ast.imported_type_definitions(&mut |id| {
defined.insert(id.clone());
if track.is_some() {
all_definitions.insert(id.clone());
}
});
if !ast.remove_undefined_imports(&|id| defined.contains(id)) {
break
}
}
if let Some(path) = track {
let contents = all_definitions.into_iter()
.map(|s| format!("{} = []", s))
.collect::<Vec<_>>()
.join("\n");
fs::write(path, contents).unwrap();
}
let mut tokens = proc_macro2::TokenStream::new();
if let Err(e) = ast.try_to_tokens(&mut tokens) {
@ -380,26 +411,29 @@ impl<'src> FirstPassRecord<'src> {
name: &'src str,
data: &InterfaceData<'src>,
) {
let doc_comment = Some(format!(
let mut doc_comment = Some(format!(
"The `{}` object\n\n{}",
name,
mdn_doc(name, None),
));
let mut import_type = backend::ast::ImportType {
vis: public(),
rust_name: rust_ident(camel_case_ident(name).as_str()),
js_name: name.to_string(),
attrs: Vec::new(),
doc_comment: None,
instanceof_shim: format!("__widl_instanceof_{}", name),
extends: self.all_superclasses(name)
.map(|name| Ident::new(&name, Span::call_site()))
.collect(),
};
self.append_required_features_doc(&import_type, &mut doc_comment);
import_type.doc_comment = doc_comment;
program.imports.push(backend::ast::Import {
module: None,
js_namespace: None,
kind: backend::ast::ImportKind::Type(backend::ast::ImportType {
vis: public(),
rust_name: rust_ident(camel_case_ident(name).as_str()),
js_name: name.to_string(),
attrs: Vec::new(),
doc_comment,
instanceof_shim: format!("__widl_instanceof_{}", name),
extends: self.all_superclasses(name)
.map(|name| Ident::new(&name, Span::call_site()))
.collect(),
}),
kind: backend::ast::ImportKind::Type(import_type),
});
for (id, op_data) in data.operations.iter() {
@ -473,7 +507,7 @@ impl<'src> FirstPassRecord<'src> {
.map(|interface_data| interface_data.global)
.unwrap_or(false);
for import_function in self.create_getter(
for mut import_function in self.create_getter(
identifier,
&type_.type_,
self_name,
@ -482,11 +516,14 @@ impl<'src> FirstPassRecord<'src> {
attrs,
container_attrs,
) {
let mut doc = import_function.doc_comment.take();
self.append_required_features_doc(&import_function, &mut doc);
import_function.doc_comment = doc;
program.imports.push(wrap_import_function(import_function));
}
if !readonly {
for import_function in self.create_setter(
for mut import_function in self.create_setter(
identifier,
&type_.type_,
self_name,
@ -495,6 +532,9 @@ impl<'src> FirstPassRecord<'src> {
attrs,
container_attrs,
) {
let mut doc = import_function.doc_comment.take();
self.append_required_features_doc(&import_function, &mut doc);
import_function.doc_comment = doc;
program.imports.push(wrap_import_function(import_function));
}
}
@ -535,7 +575,7 @@ impl<'src> FirstPassRecord<'src> {
};
let doc = match id {
OperationId::Constructor(_) |
OperationId::Operation(None) => None,
OperationId::Operation(None) => Some(String::new()),
OperationId::Operation(Some(name)) => {
Some(format!(
"The `{}()` method\n\n{}",
@ -555,8 +595,39 @@ impl<'src> FirstPassRecord<'src> {
};
let attrs = data.definition_attributes;
for mut method in self.create_imports(attrs, kind, id, op_data) {
method.doc_comment = doc.clone();
let mut doc = doc.clone();
self.append_required_features_doc(&method, &mut doc);
method.doc_comment = doc;
program.imports.push(wrap_import_function(method));
}
}
fn append_required_features_doc(
&self,
item: impl ImportedTypeReferences,
doc: &mut Option<String>,
) {
let doc = match doc {
Some(doc) => doc,
None => return,
};
let mut required = BTreeSet::new();
item.imported_type_references(&mut |f| {
if !self.builtin_idents.contains(f) {
required.insert(f.clone());
}
});
if required.len() == 0 {
return
}
let list = required.iter()
.map(|ident| format!("`{}`", ident))
.collect::<Vec<_>>()
.join(", ");
doc.push_str(&format!(
"\n\n*This function requires the following crate features \
to be activated: {}*",
list,
));
}
}

View File

@ -53,7 +53,7 @@ pub fn mdn_doc(class: &str, method: Option<&str>) -> String {
if let Some(method) = method {
link.push_str(&format!("/{}", method));
}
format!("[Documentation]({})", link).into()
format!("[MDN Documentation]({})", link).into()
}
// Array type is borrowed for arguments (`&[T]`) and owned for return value (`Vec<T>`).
@ -401,13 +401,23 @@ impl<'src> FirstPassRecord<'src> {
{
// First up, prune all signatures that reference unsupported arguments.
// We won't consider these until said arguments are implemented.
//
// Note that we handle optional arguments as well. Optional arguments
// should only appear at the end of argument lists and when we see one
// we can simply push our signature so far onto the list for the
// signature where that and all remaining optional arguments are
// undefined.
let mut signatures = Vec::new();
'outer:
for signature in data.signatures.iter() {
let mut idl_args = Vec::with_capacity(signature.args.len());
for arg in signature.args.iter() {
for (i, arg) in signature.args.iter().enumerate() {
if arg.optional {
assert!(signature.args[i..].iter().all(|a| a.optional));
signatures.push((signature, idl_args.clone()));
}
match arg.ty.to_idl_type(self) {
Some(t) => idl_args.push((t, arg)),
Some(t) => idl_args.push(t),
None => continue 'outer,
}
}
@ -434,23 +444,7 @@ impl<'src> FirstPassRecord<'src> {
args: Vec::with_capacity(signature.args.len()),
});
for (i, (idl_type, arg)) in idl_args.iter().enumerate() {
// If this is an optional argument, then all remaining arguments
// should also be optional (if any). This means that all the
// signatures we've built up so far are valid signatures because
// we're just going to omit all the future arguments. As a
// result we duplicate all the previous signatures we've made in
// the list. The duplicates will be modified in-place below.
if arg.optional {
assert!(signature.args[i..].iter().all(|a| a.optional));
let end = actual_signatures.len();
for j in start..end {
let sig = actual_signatures[j].clone();
actual_signatures.push(sig);
}
start = end;
}
for (i, idl_type) in idl_args.iter().enumerate() {
// small sanity check
assert!(start < actual_signatures.len());
for sig in actual_signatures[start..].iter() {

View File

@ -9,4 +9,13 @@ crate-type = ["cdylib"]
[dependencies]
js-sys = { path = "../../crates/js-sys" }
wasm-bindgen = { path = "../.." }
web-sys = { path = "../../crates/web-sys" }
[dependencies.web-sys]
path = "../../crates/web-sys"
features = [
'CanvasRenderingContext2d',
'Document',
'Element',
'HtmlCanvasElement',
'Window',
]

View File

@ -10,7 +10,17 @@ crate-type = ["cdylib"]
futures = "0.1.20"
wasm-bindgen = { path = "../..", features = ["serde-serialize"] }
js-sys = { path = "../../crates/js-sys" }
web-sys = { path = "../../crates/web-sys" }
wasm-bindgen-futures = { path = "../../crates/futures" }
serde = "^1.0.59"
serde_derive = "^1.0.59"
serde_derive = "^1.0.59"
[dependencies.web-sys]
path = "../../crates/web-sys"
features = [
'Headers',
'Request',
'RequestInit',
'RequestMode',
'Response',
'Window',
]

View File

@ -8,4 +8,17 @@ crate-type = ["cdylib"]
[dependencies]
wasm-bindgen = { path = "../.." }
web-sys = { path = "../../crates/web-sys" }
[dependencies.web-sys]
path = "../../crates/web-sys"
features = [
'AudioContext',
'AudioDestinationNode',
'AudioNode',
'AudioParam',
'AudioScheduledSourceNode',
'BaseAudioContext',
'GainNode',
'OscillatorNode',
'OscillatorType',
]

View File

@ -10,17 +10,10 @@ The `web-sys` crate has this file and directory layout:
├── src
│ └── lib.rs
└── webidls
├── available
│ └── ...
└── enabled
└── ...
```
### `webidls/available/*.webidl`
These are all the different WebIDL definitions we intend to support, but don't
yet. At the time of writing, these are the majority of `.webidl`s.
### `webidls/enabled/*.webidl`
These are the WebIDL interfaces that we will actually generate bindings for (or
@ -40,3 +33,12 @@ time in `build.rs`. Here is the whole `src/lib.rs` file:
```rust
{{#include ../../../crates/web-sys/src/lib.rs}}
```
### Cargo features
When compiled the crate is almost empty by default, which probably isn't what
you want! Due to the very large number of APIs, this crate uses features to
enable portions of its API to reduce compile times. The list of features in
`Cargo.toml` all correspond to types in the generated functions. Enabling a
feature enables that type. All methods should indicate what features need to be
activated to use the method.

View File

@ -5,8 +5,7 @@ You can test the `web-sys` crate by running `cargo test` within the
```sh
cd wasm-bindgen/crates/web-sys
cargo test
cargo test --target wasm32-unknown-unknown
cargo test --target wasm32-unknown-unknown --all-features
```
The Wasm tests all run within a headless browser. See [the `wasm-bindgen-test`