Fix spack, second step (#1031)

swc_bundler:
 - Support almost all kind of imports / exports (except computed key access to an es module)
 - Parallelize processing of exports

bundler/Cargo.toml

@@ -6,7 +6,7 @@ edition = "2018"
 license = "Apache-2.0/MIT"
 name = "swc_bundler"
 repository = "https://github.com/swc-project/swc.git"
-version = "0.6.2"
+version = "0.7.0"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 [features]
@@ -30,7 +30,7 @@ swc_common = {version = "0.10.0", path = "../common"}
 swc_ecma_ast = {version = "0.31.0", path = "../ecmascript/ast"}
 swc_ecma_codegen = {version = "0.35.0", path = "../ecmascript/codegen"}
 swc_ecma_parser = {version = "0.37.0", path = "../ecmascript/parser"}
-swc_ecma_transforms = {version = "0.23.0", path = "../ecmascript/transforms"}
+swc_ecma_transforms = {version = "0.23.1", path = "../ecmascript/transforms"}
 swc_ecma_utils = {version = "0.21.0", path = "../ecmascript/utils"}
 swc_ecma_visit = {version = "0.17.0", path = "../ecmascript/visit"}
 

bundler/examples/path.rs

@@ -20,6 +20,7 @@ fn main() {
         Config {
             require: true,
             external_modules,
+            ..Default::default()
         },
     );
     let mut entries = HashMap::default();

bundler/src/bundler/chunk/export.rs

@@ -1,9 +1,12 @@
 use super::plan::Plan;
 use crate::{
     bundler::load::{Specifier, TransformedModule},
-    util, Bundler, Load, Resolve,
+    util::IntoParallelIterator,
+    Bundler, Load, Resolve,
 };
 use anyhow::{Context, Error};
+#[cfg(feature = "concurrent")]
+use rayon::iter::ParallelIterator;
 use std::mem::{replace, take};
 use swc_common::{Spanned, SyntaxContext, DUMMY_SP};
 use swc_ecma_ast::*;
@@ -54,7 +57,9 @@ where
     ) -> Result<(), Error> {
         log::debug!("merge_reexports: {}", info.fm.name);
 
-        for (src, specifiers) in &info.exports.reexports {
+        let deps = (&*info.exports.reexports)
+            .into_par_iter()
+            .map(|(src, specifiers)| -> Result<_, Error> {
                 log::info!("Merging exports: {}  <- {}", info.fm.name, src.src.value);
 
                 let imported = self.scope.get_module(src.module_id).unwrap();
@@ -62,18 +67,6 @@ where
 
                 info.helpers.extend(&imported.helpers);
 
-            let (_, dep) = util::join(
-                || {
-                    self.run(|| {
-                        // entry.visit_mut_with(&mut ExportRenamer {
-                        //     from: SyntaxContext::empty().apply_mark(imported.
-                        // mark()),
-                        //     to: SyntaxContext::empty().apply_mark(info.
-                        // mark()), });
-                    })
-                },
-                || -> Result<_, Error> {
-                    self.run(|| {
                 let mut dep = self
                     .merge_modules(plan, src.module_id, false, false)
                     .with_context(|| {
@@ -83,22 +76,34 @@ where
                         )
                     })?;
 
+                // print_hygiene(&format!("dep: start"), &self.cm, &dep);
+
                 dep = self.remark_exports(dep, src.ctxt, None, false);
 
+                // print_hygiene(&format!("dep: remark exports"), &self.cm, &dep);
+
+                if !specifiers.is_empty() {
                     dep.visit_mut_with(&mut UnexportAsVar {
                         dep_ctxt: src.ctxt,
                         _entry_ctxt: info.ctxt(),
+                        _exports: &specifiers,
                     });
 
+                    // print_hygiene(&format!("dep: unexport as var"), &self.cm, &dep);
+
                     dep = dep.fold_with(&mut DepUnexporter {
                         exports: &specifiers,
                     });
 
-                        Ok(dep)
+                    // print_hygiene(&format!("dep: unexport"), &self.cm, &dep);
+                }
+
+                Ok((src, dep))
             })
-                },
+            .collect::<Vec<_>>();
-            );
+
-            let dep = dep?;
+        for dep in deps {
+            let (src, dep) = dep?;
 
             // Replace import statement / require with module body
             let mut injector = ExportInjector {
@@ -108,15 +113,11 @@ where
             entry.body.visit_mut_with(&mut injector);
 
             // print_hygiene(
-            //     &format!(
+            //     &format!("entry:injection {:?} <- {:?}", info.ctxt(), src.ctxt,),
-            //         "entry:injection {:?} <- {:?}",
-            //         SyntaxContext::empty().apply_mark(info.mark()),
-            //         SyntaxContext::empty().apply_mark(imported.mark()),
-            //     ),
             //     &self.cm,
             //     &entry,
             // );
-            // assert_eq!(injector.imported, vec![]);
+            assert_eq!(injector.imported, vec![]);
         }
 
         Ok(())
@@ -167,41 +168,6 @@ impl VisitMut for ExportInjector {
     }
 }
 
-struct ExportRenamer {
-    /// Syntax context for the top level.
-    from: SyntaxContext,
-    /// Syntax context for the top level nodes of dependency module.
-    to: SyntaxContext,
-}
-
-impl VisitMut for ExportRenamer {
-    noop_visit_mut_type!();
-
-    fn visit_mut_named_export(&mut self, export: &mut NamedExport) {
-        // if export.src.is_none() {
-        //     return;
-        // }
-
-        export.specifiers.visit_mut_children_with(self);
-    }
-
-    fn visit_mut_export_named_specifier(&mut self, s: &mut ExportNamedSpecifier) {
-        match &mut s.exported {
-            Some(v) => {
-                if v.span.ctxt == self.from {
-                    v.span = v.span.with_ctxt(self.to);
-                }
-            }
-            None => {}
-        }
-        if s.orig.span.ctxt == self.from {
-            s.orig.span = s.orig.span.with_ctxt(self.to);
-        }
-    }
-
-    fn visit_mut_stmt(&mut self, _: &mut Stmt) {}
-}
-
 /// Converts
 ///
 /// ```js
@@ -213,14 +179,17 @@ impl VisitMut for ExportRenamer {
 /// ```js
 /// const e3 = l1;
 /// ```
-struct UnexportAsVar {
+struct UnexportAsVar<'a> {
     /// Syntax context for the generated variables.
     dep_ctxt: SyntaxContext,
 
     _entry_ctxt: SyntaxContext,
+
+    /// Exports to preserve
+    _exports: &'a [Specifier],
 }
 
-impl VisitMut for UnexportAsVar {
+impl VisitMut for UnexportAsVar<'_> {
     noop_visit_mut_type!();
 
     fn visit_mut_module_item(&mut self, n: &mut ModuleItem) {

bundler/src/bundler/chunk/merge.rs

@@ -75,6 +75,16 @@ where
                 .filter(|(src, _)| {
                     log::trace!("Checking: {} <= {}", info.fm.name, src.src.value);
 
+                    // Import and export from same file. We use export to merge it.
+                    if info
+                        .exports
+                        .reexports
+                        .iter()
+                        .any(|(es, _)| es.module_id == src.module_id)
+                    {
+                        return false;
+                    }
+
                     // Skip if a dependency is going to be merged by other dependency
                     module_plan.chunks.contains(&src.module_id)
                 })
@@ -199,7 +209,7 @@ where
                             let mut dep = self.merge_modules(plan, id, false, true)?;
 
                             dep = self.remark_exports(dep, dep_info.ctxt(), None, true);
-                            // dep = dep.fold_with(&mut Unexporter);
+                            dep = dep.fold_with(&mut Unexporter);
 
                             // As transitive deps can have no direct relation with entry,
                             // remark_exports is not enough.

bundler/src/bundler/chunk/plan/mod.rs

@@ -385,11 +385,10 @@ where
             );
             builder.try_add_direct_dep(root_id, module_id, src.module_id);
 
-            builder
+            let rev = builder.reverse.entry(src.module_id).or_default();
-                .reverse
+            if !rev.contains(&module_id) {
-                .entry(src.module_id)
+                rev.push(module_id);
-                .or_default()
+            }
-                .push(module_id);
         }
     }
 }

bundler/src/bundler/chunk/remark.rs

@@ -253,6 +253,7 @@ impl Fold for ExportRenamer<'_> {
             }
 
             ModuleItem::ModuleDecl(ModuleDecl::ExportNamed(e)) if e.src.is_none() => {
+                if self.unexport {
                     let mut var_decls = Vec::with_capacity(e.specifiers.len());
 
                     e.specifiers.into_iter().for_each(|specifier| {
@@ -260,15 +261,19 @@ impl Fold for ExportRenamer<'_> {
                         let ident = match &specifier {
                             // TODO
                             ExportSpecifier::Namespace(s) => self.aliased_import(&s.name.sym),
-                        ExportSpecifier::Default(..) => self.aliased_import(&js_word!("default")),
+                            ExportSpecifier::Default(..) => {
+                                self.aliased_import(&js_word!("default"))
+                            }
                             ExportSpecifier::Named(s) => {
                                 if let Some(exported) = &s.exported {
                                     // We need remarking
 
                                     match self.aliased_import(&exported.sym) {
                                         Some(v) => {
-                                        let ctxt = self
+                                            let ctxt = self.mark_as_remarking_required(
-                                            .mark_as_remarking_required(v.clone(), s.orig.to_id());
+                                                v.clone(),
+                                                s.orig.to_id(),
+                                            );
                                             log::trace!(
                                                 "exported = {}{:?}",
                                                 exported.sym,
@@ -305,7 +310,9 @@ impl Fold for ExportRenamer<'_> {
                             let orig = match specifier {
                                 // TODO
                                 ExportSpecifier::Namespace(s) => s.name,
-                            ExportSpecifier::Default(..) => Ident::new(js_word!("default"), span),
+                                ExportSpecifier::Default(..) => {
+                                    Ident::new(js_word!("default"), span)
+                                }
                                 ExportSpecifier::Named(s) => s.orig,
                             };
 
@@ -333,6 +340,102 @@ impl Fold for ExportRenamer<'_> {
                     }
 
                     return Stmt::Empty(EmptyStmt { span }).into();
+                } else {
+                    let mut export_specifiers = Vec::with_capacity(e.specifiers.len());
+
+                    e.specifiers.into_iter().for_each(|specifier| {
+                        let span = specifier.span();
+                        let ident = match &specifier {
+                            // TODO
+                            ExportSpecifier::Namespace(s) => self.aliased_import(&s.name.sym),
+                            ExportSpecifier::Default(..) => {
+                                self.aliased_import(&js_word!("default"))
+                            }
+                            ExportSpecifier::Named(s) => {
+                                if let Some(exported) = &s.exported {
+                                    // We need remarking
+
+                                    match self.aliased_import(&exported.sym) {
+                                        Some(v) => {
+                                            let ctxt = self.mark_as_remarking_required(
+                                                v.clone(),
+                                                s.orig.to_id(),
+                                            );
+                                            log::trace!(
+                                                "exported = {}{:?}",
+                                                exported.sym,
+                                                exported.span.ctxt
+                                            );
+                                            log::trace!("id = {:?}", v);
+                                            log::trace!(
+                                                "orig = {}{:?}",
+                                                s.orig.sym,
+                                                s.orig.span.ctxt()
+                                            );
+
+                                            Some((v.0, ctxt))
+                                        }
+                                        None => None,
+                                    }
+                                } else {
+                                    match self.aliased_import(&s.orig.sym) {
+                                        Some(id) => {
+                                            let ctxt = self.mark_as_remarking_required(
+                                                (s.orig.sym.clone(), self.dep_ctxt),
+                                                s.orig.to_id(),
+                                            );
+
+                                            Some((id.0, ctxt))
+                                        }
+                                        None => None,
+                                    }
+                                }
+                            }
+                        };
+
+                        if let Some(i) = ident {
+                            let orig = match specifier {
+                                // TODO
+                                ExportSpecifier::Namespace(s) => s.name,
+                                ExportSpecifier::Default(..) => {
+                                    Ident::new(js_word!("default"), span)
+                                }
+                                ExportSpecifier::Named(s) => s.orig,
+                            };
+
+                            export_specifiers.push(ExportSpecifier::Named(ExportNamedSpecifier {
+                                span,
+                                orig,
+                                exported: Some(Ident::new(i.0, DUMMY_SP.with_ctxt(i.1))),
+                            }));
+
+                        // export_specifiers.push(VarDeclarator {
+                        //     span,
+                        //     name: Pat::Ident(Ident::new(i.0,
+                        // DUMMY_SP.with_ctxt(i.1))),
+                        //     init: Some(Box::new(Expr::Ident(orig))),
+                        //     definite: false,
+                        // })
+                        } else {
+                            log::debug!(
+                                "Removing export specifier {:?} as it's not imported (`unexport` \
+                                 is false, but it's not used)",
+                                specifier
+                            );
+                        }
+                    });
+
+                    if !export_specifiers.is_empty() {
+                        return ModuleItem::ModuleDecl(ModuleDecl::ExportNamed(NamedExport {
+                            type_only: false,
+                            span: e.span,
+                            specifiers: export_specifiers,
+                            src: None,
+                        }));
+                    }
+
+                    return Stmt::Empty(EmptyStmt { span }).into();
+                }
             }
 
             ModuleItem::ModuleDecl(ModuleDecl::ExportDecl(decl)) => {

bundler/src/bundler/finalize.rs

@@ -24,7 +24,7 @@ where
             let mut renamed = HashMap::default();
 
             for mut bundle in bundles {
-                bundle.module = self.drop_unused(bundle.module);
+                bundle.module = self.optimize(bundle.module);
                 match bundle.kind {
                     BundleKind::Named { .. } => {
                         // Inject helpers

bundler/src/bundler/mod.rs

@@ -13,15 +13,22 @@ mod finalize;
 mod helpers;
 mod import;
 mod load;
+mod optimize;
 mod scope;
 #[cfg(test)]
 mod tests;
-mod usage_analysis;
 
-#[derive(Debug)]
+#[derive(Debug, Default)]
 pub struct Config {
     /// If it's true, [Bundler] searches for require calls.
     pub require: bool,
+
+    /// If it's true, many temporary variables will be generated.
+    ///
+    /// This option exists mainly for testing. As inlining and dce removes all
+    /// temporary variables, it's really hard to see what's going on.
+    pub disable_inliner: bool,
+
     /// List of modules which should be preserved.
     pub external_modules: Vec<JsWord>,
 }

bundler/src/bundler/optimize.rs

@@ -0,0 +1,29 @@
+use crate::{Bundler, Load, Resolve};
+use swc_ecma_ast::*;
+use swc_ecma_transforms::optimization::simplify::{dce, inlining};
+use swc_ecma_visit::FoldWith;
+
+impl<L, R> Bundler<'_, L, R>
+where
+    L: Load,
+    R: Resolve,
+{
+    /// If used_exports is [None], all exports are treated as exported.
+    ///
+    /// Note: Context of used_exports is ignored, as the specifiers comes from
+    /// other module.
+    pub(super) fn optimize(&self, mut node: Module) -> Module {
+        self.run(|| {
+            if !self.config.disable_inliner {
+                node = node.fold_with(&mut inlining::inlining(inlining::Config {}))
+            }
+
+            node = node.fold_with(&mut dce::dce(dce::Config {
+                used: None,
+                used_mark: self.used_mark,
+            }));
+
+            node
+        })
+    }
+}

bundler/src/bundler/tests.rs

@@ -128,6 +128,7 @@ impl TestBuilder {
                     Default::default(),
                     Config {
                         require: true,
+                        disable_inliner: true,
                         external_modules: vec![],
                     },
                 );

bundler/src/bundler/usage_analysis.rs

@@ -1,43 +0,0 @@
-use crate::{Bundler, Load, Resolve};
-use swc_ecma_ast::*;
-use swc_ecma_transforms::optimization::simplify::dce;
-use swc_ecma_visit::FoldWith;
-
-impl<L, R> Bundler<'_, L, R>
-where
-    L: Load,
-    R: Resolve,
-{
-    /// If used_exports is [None], all exports are treated as exported.
-    ///
-    /// Note: Context of used_exports is ignored, as the specifiers comes from
-    /// other module.
-    pub(super) fn drop_unused(&self, node: Module) -> Module {
-        self.run(|| {
-            // let mut used = vec![];
-
-            // if let Some(used_exports) = used_exports {
-            //     for export in used_exports {
-            //         match export {
-            //             Specifier::Specific { alias, local, .. } => {
-            //                 used.push(alias.as_ref().unwrap_or(local).to_id());
-            //             }
-            //             Specifier::Namespace { local, .. } => {
-            //                 used.push(local.to_id());
-            //             }
-            //         }
-            //     }
-            // }
-
-            let used_mark = self.used_mark;
-
-            let mut v = dce::dce(dce::Config {
-                used: None,
-                used_mark,
-            });
-
-            let node = node.fold_with(&mut v);
-            node
-        })
-    }
-}

native/src/bundle.rs

@@ -85,6 +85,7 @@ impl Task for BundleTask {
                     .into_iter()
                     .map(From::from)
                     .collect(),
+                    ..Default::default()
                 },
             );
 

spack/tests/fixture.rs

@@ -136,6 +136,7 @@ fn reference_tests(tests: &mut Vec<TestDescAndFn>, errors: bool) -> Result<(), i
                         NodeResolver::new(),
                         Config {
                             require: true,
+                            disable_inliner: true,
                             external_modules: vec![
                                 "assert",
                                 "buffer",

spack/tests/pass/circular/mixed/output/entry.js

@@ -1,9 +1,9 @@
-console.log('c');
 class A {
     method() {
         return new B();
     }
 }
+console.log('c');
 class B extends A {
 }
 console.log(A, B);

spack/tests/pass/export/all-nested-1/input/a.js

@@ -0,0 +1 @@
+export * from './b';

spack/tests/pass/export/all-nested-1/input/b.js

@@ -0,0 +1 @@
+export * from './c'

spack/tests/pass/export/all-nested-1/input/c.js

@@ -0,0 +1,3 @@
+export const a = 1;
+export const b = 2;
+export const c = 3;

spack/tests/pass/export/all-nested-1/input/entry.js

@@ -0,0 +1 @@
+export * from './a'

spack/tests/pass/export/all-nested-1/output/entry.js

@@ -0,0 +1,3 @@
+export const a = 1;
+export const b = 2;
+export const c = 3;

spack/tests/pass/export/mixed-all-and-const/input/a.js

@@ -0,0 +1,2 @@
+export const a = 1;
+export * from './b'

spack/tests/pass/export/mixed-all-and-const/input/b.js

@@ -0,0 +1 @@
+export const b = 2;

spack/tests/pass/export/mixed-all-and-const/input/entry.js

@@ -0,0 +1 @@
+export * from './a'

spack/tests/pass/export/mixed-all-and-const/output/entry.js

@@ -0,0 +1,2 @@
+export const a = 1;
+export const b = 2;

spack/tests/pass/import-with-export/simple-1/input/a.js

@@ -0,0 +1,3 @@
+export const a = 1;
+export const b = 2;
+export const c = 3;

spack/tests/pass/import-with-export/simple-1/input/entry.js

@@ -0,0 +1,5 @@
+import { a } from './a';
+
+export * from './a'
+
+console.log(a);

spack/tests/pass/import-with-export/simple-1/output/entry.js

@@ -0,0 +1,4 @@
+export const a = 1;
+export const b = 2;
+export const c = 3;
+console.log(a);

spack/tests/pass/transitive/export-all-1/input/a.js

@@ -0,0 +1,2 @@
+export { c } from './c';
+export { d } from './d';

spack/tests/pass/transitive/export-all-1/input/b.js

@@ -0,0 +1,2 @@
+export const b = 1;
+export { e } from './e';

spack/tests/pass/transitive/export-all-1/input/c.js

@@ -0,0 +1 @@
+export const c = 3;

spack/tests/pass/transitive/export-all-1/input/d.js

@@ -0,0 +1 @@
+export const d = 4;

spack/tests/pass/transitive/export-all-1/input/e.js

@@ -0,0 +1 @@
+export const e = 5;

spack/tests/pass/transitive/export-all-1/input/entry.js

@@ -0,0 +1,2 @@
+export * from './a';
+export * from './b';

spack/tests/pass/transitive/export-all-1/output/entry.js

@@ -0,0 +1,7 @@
+const c1 = 3;
+export { c1 as c };
+const d1 = 4;
+export { d1 as d };
+export const b = 1;
+const e1 = 5;
+export { e1 as e };

spack/tests/pass/transitive/export-all-2/input/a.js

@@ -0,0 +1,2 @@
+export { c as d } from './c';
+export { d as c } from './d';

spack/tests/pass/transitive/export-all-2/input/b.js

@@ -0,0 +1,2 @@
+export const b = 1;
+export { e as a } from './e';

spack/tests/pass/transitive/export-all-2/input/c.js

@@ -0,0 +1 @@
+export const c = 3;

spack/tests/pass/transitive/export-all-2/input/d.js

@@ -0,0 +1 @@
+export const d = 4;

spack/tests/pass/transitive/export-all-2/input/e.js

@@ -0,0 +1 @@
+export const e = 5;

spack/tests/pass/transitive/export-all-2/input/entry.js

@@ -0,0 +1,2 @@
+export * from './a';
+export * from './b';

spack/tests/pass/transitive/export-all-2/output/entry.js

@@ -0,0 +1,7 @@
+const c1 = 3;
+export { c1 as d };
+const d = 4;
+export { d as c };
+export const b = 1;
+const e = 5;
+export { e as a };

spack/tests/pass/transitive/import/simple-1/input/a.js

@@ -0,0 +1,3 @@
+import { common } from './common';
+
+console.log(common, 'a.js')

spack/tests/pass/transitive/import/simple-1/input/b.js

@@ -0,0 +1,3 @@
+import { common } from './common';
+
+console.log(common, 'b.js')

spack/tests/pass/transitive/import/simple-1/input/common.js

@@ -0,0 +1 @@
+export const common = 1;

spack/tests/pass/transitive/import/simple-1/input/entry.js

@@ -0,0 +1,2 @@
+import './a';
+import './b';

spack/tests/pass/transitive/import/simple-1/output/entry.js

@@ -0,0 +1,3 @@
+const common = 1;
+console.log(common, 'a.js');
+console.log(common, 'b.js');

spack/tests/pass/transitive/import/simple-2/input/a.js

@@ -0,0 +1,4 @@
+import common1 from './common1';
+import common2 from './common2';
+
+console.log('a', common1, common2)

spack/tests/pass/transitive/import/simple-2/input/b.js

@@ -0,0 +1,4 @@
+import common3 from './common3';
+import common1 from './common1';
+
+console.log('b', common3, common1)

spack/tests/pass/transitive/import/simple-2/input/c.js

@@ -0,0 +1,5 @@
+import common4 from './common4';
+import common2 from './common2';
+import common3 from './common3';
+
+console.log('c', common4, common2, common3)

spack/tests/pass/transitive/import/simple-2/input/common1.js

@@ -0,0 +1 @@
+export const common1 = 1;

spack/tests/pass/transitive/import/simple-2/input/common2.js

@@ -0,0 +1,3 @@
+const common = 2;
+
+export { common as common2 }

spack/tests/pass/transitive/import/simple-2/input/common3.js

@@ -0,0 +1,4 @@
+export const common1 = 'Test failed :(';
+const common3 = 3;
+
+export { common3 }

spack/tests/pass/transitive/import/simple-2/input/common4.js

@@ -0,0 +1,8 @@
+var common4;
+
+try {
+    common4 = 4;
+} catch (e) {
+}
+
+export { common4 }

spack/tests/pass/transitive/import/simple-2/input/entry.js

@@ -0,0 +1,4 @@
+import './a'
+import './b'
+import './c'
+import './common3'

spack/tests/pass/transitive/import/simple-2/output/entry.js

@@ -0,0 +1,13 @@
+const common1 = 1;
+const common = 2;
+const common2 = common;
+console.log('a', common1, common2);
+const common3 = 3;
+const common31 = common3;
+console.log('b', common31, common1);
+var common4;
+try {
+    common4 = 4;
+} catch (e) {
+}
+console.log('c', common4, common2, common31);