Plugin (#473)

swc_common:
 - apply patch from rust-lang/rust#59693

swc:
 - use &Options instead of Options
 - configures commons::CM
 - exposes `handler`

.gitmodules

@@ -1,16 +1,3 @@
 [submodule "ecmascript/test262-parser-tests"]
 	path = ecmascript/parser/tests/test262-parser
 	url = https://github.com/tc39/test262-parser-tests.git
-	shallow = true
-[submodule "tests/projects/rxjs"]
-	path = tests/projects/rxjs/repo
-	url = https://github.com/ReactiveX/rxjs.git
-	shallow = true
-[submodule "tests/projects/webpack"]
-	path = tests/projects/webpack/repo
-	url = https://github.com/webpack/webpack.git
-	shallow = true
-[submodule "tests/projects/angular/repo"]
-	path = tests/projects/angular/repo
-	url = https://github.com/angular/angular.git
-	shallow = true

common/src/syntax_pos/span_encoding.rs

@@ -1,36 +1,157 @@
-// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
-// file at the top-level directory of this distribution and at
-// http://rust-lang.org/COPYRIGHT.
-//
-// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
-// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
-// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
-// option. This file may not be copied, modified, or distributed
-// except according to those terms.
-
 // Spans are encoded using 1-bit tag and 2 different encoding formats (one for
 // each tag value). One format is used for keeping span data inline,
 // another contains index into an out-of-line span interner.
 // The encoding format for inline spans were obtained by optimizing over crates
 // in rustc/libstd. See https://internals.rust-lang.org/t/rfc-compiler-refactoring-spans/1357/28
-use super::hygiene::SyntaxContext;
-use crate::syntax_pos::{BytePos, SpanData, CM, GLOBALS};
+
+use crate::{hygiene::SyntaxContext, syntax_pos::CM, BytePos, SpanData, GLOBALS};
 use hashbrown::HashMap;
 use serde::{
     de::Deserializer,
     ser::{SerializeStruct, Serializer},
     Deserialize, Serialize,
 };
-use std::hash::{Hash, Hasher};
-
 /// A compressed span.
-/// Contains either fields of `SpanData` inline if they are small, or index into
-/// span interner. The primary goal of `Span` is to be as small as possible and
-/// fit into other structures (that's why it uses `packed` as well). Decoding
-/// speed is the second priority. See `SpanData` for the info on span fields in
-/// decoded representation.
-#[repr(packed)]
-pub struct Span(u32);
+///
+/// `SpanData` is 12 bytes, which is a bit too big to stick everywhere. `Span`
+/// is a form that only takes up 8 bytes, with less space for the length and
+/// context. The vast majority (99.9%+) of `SpanData` instances will fit within
+/// those 8 bytes; any `SpanData` whose fields don't fit into a `Span` are
+/// stored in a separate interner table, and the `Span` will index into that
+/// table. Interning is rare enough that the cost is low, but common enough
+/// that the code is exercised regularly.
+///
+/// An earlier version of this code used only 4 bytes for `Span`, but that was
+/// slower because only 80--90% of spans could be stored inline (even less in
+/// very large crates) and so the interner was used a lot more.
+///
+/// Inline (compressed) format:
+/// - `span.base_or_index == span_data.lo`
+/// - `span.len_or_tag == len == span_data.hi - span_data.lo` (must be `<=
+///   MAX_LEN`)
+/// - `span.ctxt == span_data.ctxt` (must be `<= MAX_CTXT`)
+///
+/// Interned format:
+/// - `span.base_or_index == index` (indexes into the interner table)
+/// - `span.len_or_tag == LEN_TAG` (high bit set, all other bits are zero)
+/// - `span.ctxt == 0`
+///
+/// The inline form uses 0 for the tag value (rather than 1) so that we don't
+/// need to mask out the tag bit when getting the length, and so that the
+/// dummy span can be all zeroes.
+///
+/// Notes about the choice of field sizes:
+/// - `base` is 32 bits in both `Span` and `SpanData`, which means that `base`
+///   values never cause interning. The number of bits needed for `base` depends
+///   on the crate size. 32 bits allows up to 4 GiB of code in a crate.
+///   `script-servo` is the largest crate in `rustc-perf`, requiring 26 bits for
+///   some spans.
+/// - `len` is 15 bits in `Span` (a u16, minus 1 bit for the tag) and 32 bits in
+///   `SpanData`, which means that large `len` values will cause interning. The
+///   number of bits needed for `len` does not depend on the crate size. The
+///   most common number of bits for `len` are 0--7, with a peak usually at 3 or
+///   4, and then it drops off quickly from 8 onwards. 15 bits is enough for
+///   99.99%+ of cases, but larger values (sometimes 20+ bits) might occur
+///   dozens of times in a typical crate.
+/// - `ctxt` is 16 bits in `Span` and 32 bits in `SpanData`, which means that
+///   large `ctxt` values will cause interning. The number of bits needed for
+///   `ctxt` values depend partly on the crate size and partly on the form of
+///   the code. No crates in `rustc-perf` need more than 15 bits for `ctxt`, but
+///   larger crates might need more than 16 bits.
+#[derive(Clone, Copy, Eq, PartialEq, Hash)]
+pub struct Span {
+    base_or_index: u32,
+    len_or_tag: u16,
+    ctxt_or_zero: u16,
+}
+
+const LEN_TAG: u16 = 0b1000_0000_0000_0000;
+const MAX_LEN: u32 = 0b0111_1111_1111_1111;
+const MAX_CTXT: u32 = 0b1111_1111_1111_1111;
+
+/// Dummy span, both position and length are zero, syntax context is zero as
+/// well.
+pub const DUMMY_SP: Span = Span {
+    base_or_index: 0,
+    len_or_tag: 0,
+    ctxt_or_zero: 0,
+};
+
+impl Span {
+    #[inline]
+    pub fn new(mut lo: BytePos, mut hi: BytePos, ctxt: SyntaxContext) -> Self {
+        if lo > hi {
+            std::mem::swap(&mut lo, &mut hi);
+        }
+
+        let (base, len, ctxt2) = (lo.0, hi.0 - lo.0, ctxt.as_u32());
+
+        if len <= MAX_LEN && ctxt2 <= MAX_CTXT {
+            // Inline format.
+            Span {
+                base_or_index: base,
+                len_or_tag: len as u16,
+                ctxt_or_zero: ctxt2 as u16,
+            }
+        } else {
+            // Interned format.
+            let index = with_span_interner(|interner| interner.intern(&SpanData { lo, hi, ctxt }));
+            Span {
+                base_or_index: index,
+                len_or_tag: LEN_TAG,
+                ctxt_or_zero: 0,
+            }
+        }
+    }
+
+    #[inline]
+    pub fn data(self) -> SpanData {
+        if self.len_or_tag != LEN_TAG {
+            // Inline format.
+            debug_assert!(self.len_or_tag as u32 <= MAX_LEN);
+            SpanData {
+                lo: BytePos(self.base_or_index),
+                hi: BytePos(self.base_or_index + self.len_or_tag as u32),
+                ctxt: SyntaxContext::from_u32(self.ctxt_or_zero as u32),
+            }
+        } else {
+            // Interned format.
+            debug_assert!(self.ctxt_or_zero == 0);
+            let index = self.base_or_index;
+            with_span_interner(|interner| *interner.get(index))
+        }
+    }
+}
+
+#[derive(Default)]
+pub struct SpanInterner {
+    spans: HashMap<SpanData, u32>,
+    span_data: Vec<SpanData>,
+}
+
+impl SpanInterner {
+    fn intern(&mut self, span_data: &SpanData) -> u32 {
+        if let Some(index) = self.spans.get(span_data) {
+            return *index;
+        }
+
+        let index = self.spans.len() as u32;
+        self.span_data.push(*span_data);
+        self.spans.insert(*span_data, index);
+        index
+    }
+
+    #[inline]
+    fn get(&self, index: u32) -> &SpanData {
+        &self.span_data[index as usize]
+    }
+}
+
+// If an interner exists, return it. Otherwise, prepare a fresh one.
+#[inline]
+fn with_span_interner<T, F: FnOnce(&mut SpanInterner) -> T>(f: F) -> T {
+    GLOBALS.with(|globals| f(&mut *globals.span_interner.lock()))
+}
 
 #[derive(Serialize)]
 struct Loc {
@@ -90,153 +211,3 @@ impl<'de> Deserialize<'de> for Span {
         Ok(Span::new(data.lo, data.hi, data.ctxt))
     }
 }
-
-impl Copy for Span {}
-impl Clone for Span {
-    #[inline]
-    fn clone(&self) -> Span {
-        *self
-    }
-}
-impl PartialEq for Span {
-    #[inline]
-    fn eq(&self, other: &Span) -> bool {
-        let a = self.0;
-        let b = other.0;
-        a == b
-    }
-}
-impl Eq for Span {}
-impl Hash for Span {
-    #[inline]
-    fn hash<H: Hasher>(&self, state: &mut H) {
-        let a = self.0;
-        a.hash(state)
-    }
-}
-
-/// Dummy span, both position and length are zero, syntax context is zero as
-/// well. This span is kept inline and encoded with format 0.
-pub const DUMMY_SP: Span = Span(0);
-
-impl Span {
-    #[inline]
-    pub fn new(lo: BytePos, hi: BytePos, ctxt: SyntaxContext) -> Self {
-        encode(&if lo <= hi {
-            SpanData { lo, hi, ctxt }
-        } else {
-            SpanData {
-                lo: hi,
-                hi: lo,
-                ctxt,
-            }
-        })
-    }
-
-    #[inline]
-    pub fn data(self) -> SpanData {
-        decode(self)
-    }
-}
-
-// Tags
-const TAG_INLINE: u32 = 0;
-const TAG_INTERNED: u32 = 1;
-const TAG_MASK: u32 = 1;
-
-// Fields indexes
-const BASE_INDEX: usize = 0;
-const LEN_INDEX: usize = 1;
-const CTXT_INDEX: usize = 2;
-
-// Tag = 0, inline format.
-// -------------------------------------------------------------
-// | base 31:8  | len 7:1  | ctxt (currently 0 bits) | tag 0:0 |
-// -------------------------------------------------------------
-// Since there are zero bits for ctxt, only SpanData with a 0 SyntaxContext
-// can be inline.
-const INLINE_SIZES: [u32; 3] = [24, 7, 0];
-const INLINE_OFFSETS: [u32; 3] = [8, 1, 1];
-
-// Tag = 1, interned format.
-// ------------------------
-// | index 31:1 | tag 0:0 |
-// ------------------------
-const INTERNED_INDEX_SIZE: u32 = 31;
-const INTERNED_INDEX_OFFSET: u32 = 1;
-
-#[inline]
-fn encode(sd: &SpanData) -> Span {
-    let (base, len, ctxt) = (sd.lo.0, sd.hi.0 - sd.lo.0, sd.ctxt.as_u32());
-
-    let val = if (base >> INLINE_SIZES[BASE_INDEX]) == 0
-        && (len >> INLINE_SIZES[LEN_INDEX]) == 0
-        && (ctxt >> INLINE_SIZES[CTXT_INDEX]) == 0
-    {
-        (base << INLINE_OFFSETS[BASE_INDEX])
-            | (len << INLINE_OFFSETS[LEN_INDEX])
-            | (ctxt << INLINE_OFFSETS[CTXT_INDEX])
-            | TAG_INLINE
-    } else {
-        let index = with_span_interner(|interner| interner.intern(sd));
-        (index << INTERNED_INDEX_OFFSET) | TAG_INTERNED
-    };
-    Span(val)
-}
-
-#[inline]
-fn decode(span: Span) -> SpanData {
-    let val = span.0;
-
-    // Extract a field at position `pos` having size `size`.
-    let extract = |pos: u32, size: u32| {
-        let mask = ((!0u32) as u64 >> (32 - size)) as u32; // Can't shift u32 by 32
-        (val >> pos) & mask
-    };
-
-    let (base, len, ctxt) = if val & TAG_MASK == TAG_INLINE {
-        (
-            extract(INLINE_OFFSETS[BASE_INDEX], INLINE_SIZES[BASE_INDEX]),
-            extract(INLINE_OFFSETS[LEN_INDEX], INLINE_SIZES[LEN_INDEX]),
-            extract(INLINE_OFFSETS[CTXT_INDEX], INLINE_SIZES[CTXT_INDEX]),
-        )
-    } else {
-        let index = extract(INTERNED_INDEX_OFFSET, INTERNED_INDEX_SIZE);
-        return with_span_interner(|interner| *interner.get(index));
-    };
-    SpanData {
-        lo: BytePos(base),
-        hi: BytePos(base + len),
-        ctxt: SyntaxContext::from_u32(ctxt),
-    }
-}
-
-#[derive(Default)]
-pub struct SpanInterner {
-    spans: HashMap<SpanData, u32>,
-    span_data: Vec<SpanData>,
-}
-
-impl SpanInterner {
-    fn intern(&mut self, span_data: &SpanData) -> u32 {
-        if let Some(index) = self.spans.get(span_data) {
-            return *index;
-        }
-
-        let index = self.spans.len() as u32;
-        self.span_data.push(*span_data);
-        self.spans.insert(*span_data, index);
-        index
-    }
-
-    #[inline]
-    fn get(&self, index: u32) -> &SpanData {
-        &self.span_data[index as usize]
-    }
-}
-
-// If an interner exists, return it. Otherwise, prepare a fresh one.
-#[inline]
-fn with_span_interner<T, F: FnOnce(&mut SpanInterner) -> T>(f: F) -> T {
-    GLOBALS.with(|globals| f(&mut *globals.span_interner.lock()))
-}

ecmascript/ast/src/prop.rs

@@ -91,5 +91,6 @@ pub enum PropName {
 pub struct ComputedPropName {
     /// Span including `[` and `]`.
     pub span: Span,
+    #[serde(rename = "expression")]
     pub expr: Box<Expr>,
 }

ecmascript/codegen/tests/references/569a2c1bad3beeb2.js

@@ -0,0 +1,2 @@
+({ a , ...b })=>0
+;

ecmascript/parser/benches/lexer.rs

@@ -4,7 +4,7 @@ extern crate test;
 
 use std::hint::black_box;
 use swc_common::FileName;
-use swc_ecma_parser::{lexer::Lexer, Parser, Session, SourceFileInput, Syntax};
+use swc_ecma_parser::{lexer::Lexer, Session, SourceFileInput, Syntax};
 use test::Bencher;
 
 #[bench]
@@ -82,7 +82,7 @@ fn bench_module(b: &mut Bencher, syntax: Syntax, src: &'static str) {
         b.iter(|| {
             let lexer = Lexer::new(
                 session,
-                Default::default(),
+                syntax,
                 Default::default(),
                 SourceFileInput::from(&*fm),
                 None,

ecmascript/parser/src/parser/macros.rs

@@ -12,7 +12,7 @@ macro_rules! is {
     ($p:expr, BindingIdent) => {{
         let ctx = $p.ctx();
         match cur!($p, false) {
-            Ok(&Word(ref w)) => !ctx.is_reserved_word(&w.clone().into()),
+            Ok(&Word(ref w)) => !ctx.is_reserved_word(&w.cow()),
             _ => false,
         }
     }};
@@ -20,7 +20,7 @@ macro_rules! is {
     ($p:expr, IdentRef) => {{
         let ctx = $p.ctx();
         match cur!($p, false) {
-            Ok(&Word(ref w)) => !ctx.is_reserved_word(&w.clone().into()),
+            Ok(&Word(ref w)) => !ctx.is_reserved_word(&w.cow()),
             _ => false,
         }
     }};
@@ -48,7 +48,7 @@ macro_rules! peeked_is {
     ($p:expr, BindingIdent) => {{
         let ctx = $p.ctx();
         match peek!($p) {
-            Ok(&Word(ref w)) => !ctx.is_reserved_word(&w.clone().into()),
+            Ok(&Word(ref w)) => !ctx.is_reserved_word(&w.cow()),
             _ => false,
         }
     }};
@@ -56,7 +56,7 @@ macro_rules! peeked_is {
     ($p:expr, IdentRef) => {{
         let ctx = $p.ctx();
         match peek!($p) {
-            Ok(&Word(ref w)) => !ctx.is_reserved_word(&w.clone().into()),
+            Ok(&Word(ref w)) => !ctx.is_reserved_word(&w.cow()),
             _ => false,
         }
     }};

ecmascript/parser/src/token/mod.rs

@@ -6,7 +6,10 @@ use crate::error::Error;
 pub(crate) use ast::AssignOp as AssignOpToken;
 use ast::{BinaryOp, Str};
 use enum_kind::Kind;
-use std::fmt::{self, Debug, Display, Formatter};
+use std::{
+    borrow::Cow,
+    fmt::{self, Debug, Display, Formatter},
+};
 use swc_atoms::JsWord;
 #[cfg(feature = "fold")]
 use swc_common::Fold;
@@ -550,3 +553,15 @@ impl Token {
         }
     }
 }
+
+impl Word {
+    pub(crate) fn cow(&self) -> Cow<JsWord> {
+        match *self {
+            Word::Keyword(k) => Cow::Owned(k.into_js_word()),
+            Word::Ident(ref w) => Cow::Borrowed(&w),
+            Word::False => Cow::Owned(js_word!("false")),
+            Word::True => Cow::Owned(js_word!("true")),
+            Word::Null => Cow::Owned(js_word!("null")),
+        }
+    }
+}

examples/usage.rs

@@ -20,7 +20,7 @@ fn main() {
 
     c.process_js_file(
         fm,
-        Options {
+        &Options {
             ..Default::default()
         },
     )

macros/ast_node/Cargo.toml

@@ -30,4 +30,5 @@ features = ["derive", "fold", "parsing", "printing"]
 
 [dev-dependencies]
 swc_common = { version = "0.4", path = "../../common", features = ["fold"] }
-serde = { version = "1", features = ["derive"] }
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"

macros/ast_node/benches/serde.rs

@@ -0,0 +1,103 @@
+#![feature(box_syntax)]
+#![feature(test)]
+
+extern crate test;
+
+use ast_node::ast_node;
+use serde::{Deserialize, Serialize};
+use std::hint::black_box;
+use swc_common::{Span, DUMMY_SP};
+use test::Bencher;
+
+#[derive(Serialize, Deserialize)]
+pub struct SerdeStr {
+    span: Span,
+    value: String,
+}
+
+#[ast_node("String")]
+pub struct Str {
+    span: Span,
+    value: String,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct SerdeNum {
+    span: Span,
+    value: u64,
+}
+
+#[ast_node("Number")]
+pub struct Num {
+    span: Span,
+    value: u64,
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(tag = "type")]
+pub enum Serde {
+    Number(SerdeNum),
+    String(SerdeStr),
+}
+
+#[ast_node]
+pub enum AstNode {
+    #[tag("Number")]
+    Number(Num),
+    #[tag("String")]
+    String(Str),
+}
+
+#[bench]
+fn ser_serde(b: &mut Bencher) {
+    let src = Serde::String(SerdeStr {
+        span: DUMMY_SP,
+        value: String::from("perf-diff"),
+    });
+
+    b.iter(|| black_box(serde_json::to_string(&src).unwrap()));
+}
+
+#[bench]
+fn de_serde(b: &mut Bencher) {
+    let src = serde_json::to_string(&Serde::String(SerdeStr {
+        span: DUMMY_SP,
+        value: String::from("perf-diff"),
+    }))
+    .unwrap();
+    println!("{}", src);
+    b.bytes = src.len() as _;
+
+    b.iter(|| {
+        let t: Serde = serde_json::from_str(&src).unwrap();
+
+        black_box(t);
+    });
+}
+
+#[bench]
+fn ser_ast_node(b: &mut Bencher) {
+    let src = AstNode::String(Str {
+        span: DUMMY_SP,
+        value: String::from("perf-diff"),
+    });
+
+    b.iter(|| black_box(serde_json::to_string(&src).unwrap()));
+}
+
+#[bench]
+fn de_ast_node(b: &mut Bencher) {
+    let src = serde_json::to_string(&AstNode::String(Str {
+        span: DUMMY_SP,
+        value: String::from("perf-diff"),
+    }))
+    .unwrap();
+    println!("{}", src);
+    b.bytes = src.len() as _;
+
+    b.iter(|| {
+        let t: AstNode = serde_json::from_str(&src).unwrap();
+
+        black_box(t);
+    });
+}

src/config/mod.rs

@@ -35,7 +35,7 @@ pub struct ParseOptions {
     pub syntax: Syntax,
 }
 
-#[derive(Default, Clone, Serialize, Deserialize)]
+#[derive(Default, Serialize, Deserialize)]
 #[serde(deny_unknown_fields, rename_all = "camelCase")]
 pub struct Options {
     #[serde(flatten, default)]

src/lib.rs

@@ -42,7 +42,7 @@ pub struct Compiler {
     globals: Globals,
     /// CodeMap
     pub cm: Arc<SourceMap>,
-    handler: Handler,
+    pub handler: Handler,
 }
 
 #[derive(Serialize)]
@@ -61,7 +61,7 @@ impl Compiler {
     where
         F: FnOnce() -> R,
     {
-        GLOBALS.set(&self.globals, op)
+        GLOBALS.set(&self.globals, || common::CM.set(&self.cm, || op()))
     }
 
     /// This method parses a javascript / typescript file
@@ -243,9 +243,9 @@ impl Compiler {
     pub fn process_js_file(
         &self,
         fm: Arc<SourceFile>,
-        opts: Options,
+        opts: &Options,
     ) -> Result<TransformOutput, Error> {
-        let config = self.run(|| self.config_for_file(&opts, &*fm))?;
+        let config = self.run(|| self.config_for_file(opts, &*fm))?;
 
         self.process_js(fm, config)
     }

tests/projects.rs

@@ -10,7 +10,7 @@ fn file(f: &str) -> Result<NormalizedOutput, StdErr> {
         let fm = cm.load_file(Path::new(f)).expect("failed to load file");
         let s = c.process_js_file(
             fm,
-            Options {
+            &Options {
                 swcrc: true,
                 ..Default::default()
             },
@@ -44,7 +44,7 @@ fn project(dir: &str) {
                 let fm = cm.load_file(entry.path()).expect("failed to load file");
                 let _ = c.process_js_file(
                     fm,
-                    Options {
+                    &Options {
                         swcrc: true,
                         ..Default::default()
                     },
@@ -57,20 +57,20 @@ fn project(dir: &str) {
         .expect("");
 }
 
-#[test]
-fn angular_core() {
-    project("tests/projects/angular/repo/packages/core/src");
-}
-
-#[test]
-fn rxjs() {
-    project("tests/projects/rxjs/repo/src");
-}
-
-#[test]
-fn webpack() {
-    project("tests/projects/webpack/repo/lib");
-}
+//#[test]
+//fn angular_core() {
+//    project("tests/projects/angular/repo/packages/core/src");
+//}
+//
+//#[test]
+//fn rxjs() {
+//    project("tests/projects/rxjs/repo/src");
+//}
+//
+//#[test]
+//fn webpack() {
+//    project("tests/projects/webpack/repo/lib");
+//}
 
 #[test]
 fn issue_467() {

tests/projects/angular/.swcrc

@@ -1,9 +0,0 @@
-{
-  "jsc": {
-    "parser": {
-      "syntax": "typescript",
-      "tsx": false,
-      "decorators": false
-    }
-  }
-}

tests/projects/angular/repo

@@ -1 +0,0 @@
-Subproject commit 5de7960f019701e4e26dc6a7809c244ef94b5e30

tests/projects/rxjs/repo

@@ -1 +0,0 @@
-Subproject commit d2f68db3a49079adffff52c5fbb950aa6fcb19ff

tests/projects/webpack/repo

@@ -1 +0,0 @@
-Subproject commit 9b9c9ab7f10b0bd774d026289c6811b3d104dcaa