feat(webpack): Add ast reducer (#2875)

swc_estree_compat:
 - Fix stack overflow related to object patterns.

swc_webpack_}ast:
 - Implement AST reducer.

.github/workflows/cargo.yml

@@ -167,6 +167,7 @@ jobs:
           - swc_timer
           - swc_visit
           - swc_visit_macros
+          - swc_webpack_ast
           - testing
           - testing_macros
           - wasm

Cargo.lock

@@ -3154,7 +3154,7 @@ dependencies = [
 
 [[package]]
 name = "swc_estree_compat"
-version = "0.3.0"
+version = "0.3.1"
 dependencies = [
  "ahash",
  "anyhow",
@@ -3359,6 +3359,29 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "swc_webpack_ast"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "rayon",
+ "serde_json",
+ "swc_atoms 0.2.9",
+ "swc_common",
+ "swc_ecma_ast",
+ "swc_ecma_parser",
+ "swc_ecma_transforms_base",
+ "swc_ecma_transforms_testing",
+ "swc_ecma_utils",
+ "swc_ecma_visit",
+ "swc_estree_ast",
+ "swc_estree_compat",
+ "swc_node_base",
+ "swc_timer",
+ "testing",
+ "tracing",
+]
+
 [[package]]
 name = "syn"
 version = "1.0.65"

Cargo.toml

@@ -11,6 +11,7 @@ members = [
   "crates/swc_plugin_testing",
   "crates/swc_stylis",
   "crates/swc_timer",
+  "crates/swc_webpack_ast",
   "crates/wasm",
 ]
 

crates/swc_ecma_transforms_testing/src/lib.rs

@@ -685,7 +685,7 @@ fn test_fixture_inner<P>(
             as_folder(::swc_ecma_utils::DropSpan {
                 preserve_ctxt: true,
             }),
-            "output.js",
+            "expected.js",
             syntax,
             &expected,
         )?;

crates/swc_estree_compat/Cargo.toml

@@ -6,7 +6,7 @@ edition = "2018"
 license = "Apache-2.0/MIT"
 name = "swc_estree_compat"
 repository = "https://github.com/swc-project/swc.git"
-version = "0.3.0"
+version = "0.3.1"
 
 [package.metadata.docs.rs]
 all-features = true

crates/swc_estree_compat/src/babelify/module.rs

@@ -141,7 +141,7 @@ impl Babelify for ModuleItem {
     type Output = ModuleItemOutput;
 
     fn parallel(cnt: usize) -> bool {
-        cnt >= 4
+        cnt >= 32
     }
 
     fn babelify(self, ctx: &Context) -> Self::Output {

crates/swc_estree_compat/src/babelify/pat.rs

@@ -55,7 +55,11 @@ impl From<PatOutput> for ObjectPropVal {
     fn from(pat: PatOutput) -> Self {
         match pat {
             PatOutput::Expr(e) => ObjectPropVal::Expr(e),
-            other => ObjectPropVal::Pattern(other.into()),
+            PatOutput::Id(p) => ObjectPropVal::Pattern(PatternLike::Id(p)),
+            PatOutput::Array(p) => ObjectPropVal::Pattern(PatternLike::ArrayPat(p)),
+            PatOutput::Rest(p) => ObjectPropVal::Pattern(PatternLike::RestEl(p)),
+            PatOutput::Object(p) => ObjectPropVal::Pattern(PatternLike::ObjectPat(p)),
+            PatOutput::Assign(p) => ObjectPropVal::Pattern(PatternLike::AssignmentPat(p)),
         }
     }
 }
@@ -119,7 +123,10 @@ impl From<PatOutput> for Param {
         match pat {
             PatOutput::Id(i) => Param::Id(i),
             PatOutput::Rest(r) => Param::Rest(r),
-            other => other.into(),
+            PatOutput::Array(p) => Param::Pat(Pattern::Array(p)),
+            PatOutput::Object(p) => Param::Pat(Pattern::Object(p)),
+            PatOutput::Assign(p) => Param::Pat(Pattern::Assignment(p)),
+            PatOutput::Expr(p) => panic!("Cannot convert {:?} to Param", p),
         }
     }
 }

crates/swc_webpack_ast/.gitignore

@@ -0,0 +1 @@
+tests/**/output.json

crates/swc_webpack_ast/Cargo.toml

@@ -0,0 +1,31 @@
+[package]
+authors = ["강동윤 <kdy1997.dev@gmail.com>"]
+description = "Webpack AST optimizer"
+edition = "2018"
+license = "Apache-2.0"
+name = "swc_webpack_ast"
+repository = "https://github.com/swc-project/swc.git"
+version = "0.1.0"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+anyhow = "1.0.48"
+rayon = "1.5.1"
+serde_json = "1.0.72"
+swc_atoms = {version = "0.2.9", path = "../swc_atoms"}
+swc_common = {version = "0.14.6", path = "../swc_common"}
+swc_ecma_ast = {version = "0.58.0", path = "../swc_ecma_ast"}
+swc_ecma_transforms_base = {version = "0.44.3", path = "../swc_ecma_transforms_base"}
+swc_ecma_utils = {version = "0.52.3", path = "../swc_ecma_utils"}
+swc_ecma_visit = {version = "0.44.0", path = "../swc_ecma_visit"}
+swc_estree_ast = {version = "0.2.0", path = "../swc_estree_ast"}
+swc_estree_compat = {version = "0.3.0", path = "../swc_estree_compat"}
+swc_timer = {version = "0.1.0", path = "../swc_timer"}
+tracing = "0.1.29"
+
+[dev-dependencies]
+swc_ecma_parser = {version = "0.78.9", path = "../swc_ecma_parser"}
+swc_ecma_transforms_testing = {version = "0.45.1", path = "../swc_ecma_transforms_testing"}
+swc_node_base = {version = "0.5.1", path = "../swc_node_base"}
+testing = {version = "0.15.2", path = "../testing"}

crates/swc_webpack_ast/benches/webpack_ast.rs

@@ -0,0 +1,48 @@
+#![feature(test)]
+
+use std::path::Path;
+use swc_ecma_parser::{EsConfig, Parser, StringInput, Syntax};
+use test::Bencher;
+
+extern crate swc_node_base;
+extern crate test;
+
+/// this benchmark requires real input, which cannot be committed into git
+/// repository
+#[bench]
+#[cfg(not(all))]
+fn total(b: &mut Bencher) {
+    let input = Path::new("tests/fixture/real/input.js");
+
+    b.iter(|| {
+        testing::run_test(false, |cm, handler| {
+            let fm = cm.load_file(&input).unwrap();
+
+            let module = {
+                let mut p = Parser::new(
+                    Syntax::Es(EsConfig {
+                        jsx: true,
+                        ..Default::default()
+                    }),
+                    StringInput::from(&*fm),
+                    None,
+                );
+                let res = p
+                    .parse_module()
+                    .map_err(|e| e.into_diagnostic(&handler).emit());
+
+                for e in p.take_errors() {
+                    e.into_diagnostic(&handler).emit()
+                }
+
+                res?
+            };
+
+            let s = swc_webpack_ast::webpack_ast(cm.clone(), fm.clone(), module).unwrap();
+            println!("{} bytes", s.len());
+
+            Ok(())
+        })
+        .unwrap();
+    });
+}

crates/swc_webpack_ast/scripts/bench.js

@@ -0,0 +1,66 @@
+
+
+
+const Benchmark = require('benchmark');
+const acorn = require("acorn");
+const jsx = require("acorn-jsx");
+const parser = acorn.Parser.extend(jsx());
+
+
+const fs = require('fs');
+const path = require('path');
+
+
+const src = fs.readFileSync(path.join(process.argv[2], "input.js"), 'utf8');
+const jsonStr = fs.readFileSync(path.join(process.argv[2], "output.json"), 'utf8');
+
+
+{
+    parser.parse(src, {
+        ecmaVersion: 2020,
+        ranges: true,
+        allowHashBang: true,
+        sourceType: 'module'
+    })
+}
+const suite = new Benchmark.Suite;
+
+suite
+    .add('acorn', () => {
+        parser.parse(src, {
+            ecmaVersion: 2020,
+            ranges: true,
+            allowHashBang: true,
+            sourceType: 'module'
+        })
+    })
+    .add({
+        name: 'acorn-real',
+        fn: (deferred) => {
+            fs.promises.readFile(path.join(process.argv[2], "input.js"), 'utf8')
+                .then((src) => {
+                    parser.parse(src, {
+                        ecmaVersion: 2020,
+                        ranges: true,
+                        allowHashBang: true,
+                        sourceType: 'module'
+                    });
+                    deferred.resolve()
+                })
+        },
+        defer: true,
+        async: true,
+        queued: true
+    })
+    .add('json', () => {
+        JSON.parse(jsonStr)
+    })
+    .on('cycle', function (event) {
+        console.log(String(event.target));
+    })
+    .on('complete', function () {
+        console.log('Fastest is ' + this.filter('fastest').map('name'));
+    })
+    .run({
+        async: true
+    })

crates/swc_webpack_ast/src/lib.rs

@@ -0,0 +1,45 @@
+use crate::reducer::ast_reducer;
+use anyhow::{Context, Error};
+use swc_common::{sync::Lrc, Mark, SourceFile, SourceMap};
+use swc_ecma_ast::*;
+use swc_ecma_transforms_base::resolver::resolver_with_mark;
+use swc_ecma_visit::VisitMutWith;
+use swc_estree_ast::flavor::Flavor;
+use swc_estree_compat::babelify::Babelify;
+use swc_timer::timer;
+
+pub mod reducer;
+
+/// `n` is expected to be pure (`resolver` is not applied)
+pub fn webpack_ast(
+    cm: Lrc<SourceMap>,
+    fm: Lrc<SourceFile>,
+    mut n: Module,
+) -> Result<String, Error> {
+    let _timer = timer!("webpack_ast");
+    let top_level_mark = Mark::fresh(Mark::root());
+
+    Flavor::Acorn.with(|| {
+        {
+            let _timer = timer!("resolver");
+            n.visit_mut_with(&mut resolver_with_mark(top_level_mark));
+        }
+        {
+            n.visit_mut_with(&mut ast_reducer(top_level_mark));
+        }
+
+        let ctx = swc_estree_compat::babelify::Context {
+            fm,
+            cm: cm.clone(),
+            comments: Default::default(),
+        };
+
+        let babel_ast = {
+            let _timer = timer!("babelify");
+            n.babelify(&ctx)
+        };
+
+        let _timer = timer!("acorn ast to json");
+        serde_json::to_string(&babel_ast).context("failed to serialize babel ast")
+    })
+}

crates/swc_webpack_ast/tests/fixture.rs

@@ -0,0 +1,66 @@
+use std::{fs, path::PathBuf};
+use swc_common::{chain, Mark};
+use swc_ecma_parser::{EsConfig, Parser, StringInput, Syntax};
+use swc_ecma_transforms_base::resolver::resolver_with_mark;
+use swc_ecma_transforms_testing::test_fixture;
+use swc_ecma_visit::as_folder;
+use swc_webpack_ast::reducer::ast_reducer;
+
+#[testing::fixture("tests/fixture/**/input.js")]
+fn fixture(input: PathBuf) {
+    let output = input.parent().unwrap().join("output.js");
+
+    test_fixture(
+        Syntax::Es(EsConfig {
+            jsx: true,
+            ..Default::default()
+        }),
+        &|_| {
+            let top_level_mark = Mark::fresh(Mark::root());
+
+            chain!(
+                resolver_with_mark(top_level_mark),
+                as_folder(ast_reducer(top_level_mark))
+            )
+        },
+        &input,
+        &output,
+    );
+}
+
+#[testing::fixture("tests/fixture/**/input.js")]
+fn test_babelify(input: PathBuf) {
+    let output_path = input.parent().unwrap().join("output.json");
+
+    testing::run_test(false, |cm, handler| {
+        let fm = cm.load_file(&input).unwrap();
+
+        let module = {
+            let mut p = Parser::new(
+                Syntax::Es(EsConfig {
+                    jsx: true,
+                    ..Default::default()
+                }),
+                StringInput::from(&*fm),
+                None,
+            );
+            let res = p
+                .parse_module()
+                .map_err(|e| e.into_diagnostic(&handler).emit());
+
+            for e in p.take_errors() {
+                e.into_diagnostic(&handler).emit()
+            }
+
+            res?
+        };
+
+        let s = swc_webpack_ast::webpack_ast(cm.clone(), fm.clone(), module).unwrap();
+        println!("{} bytes", s.len());
+
+        fs::write(&output_path, s.as_bytes()).unwrap();
+
+        Ok(())
+    })
+    .unwrap();
+}

crates/swc_webpack_ast/tests/fixture/array-literal/input.js

@@ -0,0 +1,5 @@
+import { a } from "x";
+import b from "y";
+
+const arr = [a, 'foo', b(), 'bar'];
+console.log(arr);

crates/swc_webpack_ast/tests/fixture/array-literal/output.js

@@ -0,0 +1,3 @@
+import { a } from "x";
+import b from "y";
+a, b();

crates/swc_webpack_ast/tests/fixture/base/input.js

@@ -0,0 +1,10 @@
+import { a } from "x";
+import b from "y";
+
+function d() {
+    a.x.y(), console.log(b);
+    require("z");
+    module.hot.accept("x", x => {
+
+    })
+}

crates/swc_webpack_ast/tests/fixture/base/output.js

@@ -0,0 +1,6 @@
+import { a } from "x";
+import b from "y";
+a.x.y(), b;
+require("z");
+module.hot.accept("x", ()=>{
+});

crates/swc_webpack_ast/tests/fixture/bin-expr/1/input.js

@@ -0,0 +1,6 @@
+import y from 'y;'
+
+const obj = {
+    [1 + 1]: 2,
+    y,
+}

crates/swc_webpack_ast/tests/fixture/bin-expr/1/output.js

@@ -0,0 +1,2 @@
+import y from "y;";
+y;

crates/swc_webpack_ast/tests/fixture/bin-expr/2/input.js

@@ -0,0 +1,9 @@
+import x from 'x;/'
+import y from 'y;'
+
+const obj = {
+    [1 + y]: x,
+    y,
+}
+
+console.log(obj)

crates/swc_webpack_ast/tests/fixture/bin-expr/2/output.js

@@ -0,0 +1,3 @@
+import x from "x;/";
+import y from "y;";
+y, x, y;

crates/swc_webpack_ast/tests/fixture/bin-expr/3/input.js

@@ -0,0 +1,9 @@
+import x from 'x;/'
+import y from 'y;'
+
+const obj = {
+    [1 + y]: 1,
+    x,
+}
+
+console.log(obj)

crates/swc_webpack_ast/tests/fixture/bin-expr/3/output.js

@@ -0,0 +1,3 @@
+import x from "x;/";
+import y from "y;";
+y, x;

crates/swc_webpack_ast/tests/fixture/class-decl/1/input.js

@@ -0,0 +1,8 @@
+import bar from 'bar';
+
+class Foo {
+    [bar]() { }
+}
+
+
+console.log(new Foo())

crates/swc_webpack_ast/tests/fixture/class-decl/1/output.js

@@ -0,0 +1,5 @@
+import bar from "bar";
+class Foo {
+    [bar]() {
+    }
+}

crates/swc_webpack_ast/tests/fixture/new-expr/1/input.js

@@ -0,0 +1,6 @@
+import Bar from 'bar';
+
+class Foo extends Bar {
+}
+
+console.log(new Foo())

crates/swc_webpack_ast/tests/fixture/new-expr/1/output.js

@@ -0,0 +1,3 @@
+import Bar from "bar";
+class Foo extends Bar {
+}

crates/swc_webpack_ast/tests/fixture/object-literal/input.js

@@ -0,0 +1,12 @@
+import { a } from "x";
+import b from "y";
+
+const d = {};
+
+const obj = {
+    a,
+    [b]: 'foo',
+    c: 'bar',
+    d
+};
+console.log(obj);

crates/swc_webpack_ast/tests/fixture/object-literal/output.js

@@ -0,0 +1,3 @@
+import { a } from "x";
+import b from "y";
+a, b;

crates/swc_webpack_ast/tests/fixture/object-patterns/input.js

@@ -0,0 +1,10 @@
+import { a } from "x";
+import b from "y";
+
+function d() {
+    a.x.y(), console.log(b);
+    require("z");
+    module.hot.accept("x", ({ a = 1, b: { foo }, c: { ...rest } }) => {
+
+    })
+}

crates/swc_webpack_ast/tests/fixture/object-patterns/output.js

@@ -0,0 +1,6 @@
+import { a } from "x";
+import b from "y";
+a.x.y(), b;
+require("z");
+module.hot.accept("x", ()=>{
+});

crates/swc_webpack_ast/tests/fixture/opt-chain/1/input.js

@@ -0,0 +1 @@
+const a = obj?.foo;

crates/swc_webpack_ast/tests/fixture/opt-chain/in-jsx/input.js

@@ -0,0 +1,5 @@
+
+
+export const foo = <Ad
+    hash={a.b ? null : c?.d}
+/>

crates/swc_webpack_ast/tests/fixture/opt-chain/in-jsx/output.js

@@ -0,0 +1 @@
+export const foo = null;

crates/swc_webpack_ast/tests/fixture/try-catch/1/input.js

@@ -0,0 +1,19 @@
+export function foo (){
+    try { 
+        const meta = this.meta();
+        // const { event, team, user } = this.props;
+        const b = team ? event.user : user;
+    
+        if (meta && meta.p1) {
+          return meta.p1;
+        } else if (meta && meta.p2) {
+          return meta.p2;
+        } else if (meta && meta.p3) {
+          return meta.p3;
+        }
+    
+        return b.p4 === u.u ? 'You' : b.u;
+      } catch (e) {
+        return 'You';
+      }
+}

crates/swc_webpack_ast/tests/fixture/try-catch/1/output.js

@@ -0,0 +1,2 @@
+export function foo() {
+}

package.json

@@ -75,8 +75,10 @@
         "@types/jest": "^26.0.23",
         "@types/node": "^14.14.41",
         "acorn": "^8.6.0",
+        "acorn-jsx": "^5.3.2",
         "axios": "^0.21.1",
         "babel-plugin-transform-node-env-inline": "^0.4.3",
+        "benchmark": "^2.1.4",
         "class-validator": "^0.13.1",
         "core-js": "^2.6.11",
         "cross-env": "^7.0.3",

yarn.lock

@@ -1730,6 +1730,11 @@ acorn-globals@^6.0.0:
     acorn "^7.1.1"
     acorn-walk "^7.1.1"
 
+acorn-jsx@^5.3.2:
+  version "5.3.2"
+  resolved "https://registry.yarnpkg.com/acorn-jsx/-/acorn-jsx-5.3.2.tgz#7ed5bb55908b3b2f1bc55c6af1653bada7f07937"
+  integrity sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==
+
 acorn-walk@^7.1.1:
   version "7.2.0"
   resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-7.2.0.tgz#0de889a601203909b0fbe07b8938dc21d2e967bc"
@@ -1942,6 +1947,14 @@ before-after-hook@^2.1.0, before-after-hook@^2.2.0:
   resolved "https://registry.yarnpkg.com/before-after-hook/-/before-after-hook-2.2.2.tgz#a6e8ca41028d90ee2c24222f201c90956091613e"
   integrity sha512-3pZEU3NT5BFUo/AD5ERPWOgQOCZITni6iavr5AUw5AUwQjMlI0kzu5btnyD39AF0gUEsDPwJT+oY1ORBJijPjQ==
 
+benchmark@^2.1.4:
+  version "2.1.4"
+  resolved "https://registry.yarnpkg.com/benchmark/-/benchmark-2.1.4.tgz#09f3de31c916425d498cc2ee565a0ebf3c2a5629"
+  integrity sha1-CfPeMckWQl1JjMLuVloOvzwqVik=
+  dependencies:
+    lodash "^4.17.4"
+    platform "^1.3.3"
+
 bl@^4.1.0:
   version "4.1.0"
   resolved "https://registry.yarnpkg.com/bl/-/bl-4.1.0.tgz#451535264182bec2fbbc83a62ab98cf11d9f7b3a"
@@ -3625,7 +3638,7 @@ lodash.templatesettings@^4.0.0:
   dependencies:
     lodash._reinterpolate "^3.0.0"
 
-lodash@^4.17.21, lodash@^4.7.0:
+lodash@^4.17.21, lodash@^4.17.4, lodash@^4.7.0:
   version "4.17.21"
   resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c"
   integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==
@@ -3972,6 +3985,11 @@ pkg-dir@^4.2.0:
   dependencies:
     find-up "^4.0.0"
 
+platform@^1.3.3:
+  version "1.3.6"
+  resolved "https://registry.yarnpkg.com/platform/-/platform-1.3.6.tgz#48b4ce983164b209c2d45a107adb31f473a6e7a7"
+  integrity sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==
+
 prelude-ls@~1.1.2:
   version "1.1.2"
   resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.1.2.tgz#21932a549f5e52ffd9a827f570e04be62a97da54"