Fix spack, second step (#1031)

swc_bundler:
 - Support almost all kind of imports / exports (except computed key access to an es module)
 - Parallelize processing of exports
This commit is contained in:
강동윤 2020-09-04 22:40:03 +09:00 committed by GitHub
parent 6524802ae5
commit 468abb9832
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
56 changed files with 398 additions and 203 deletions

View File

@ -6,7 +6,7 @@ edition = "2018"
license = "Apache-2.0/MIT" license = "Apache-2.0/MIT"
name = "swc_bundler" name = "swc_bundler"
repository = "https://github.com/swc-project/swc.git" repository = "https://github.com/swc-project/swc.git"
version = "0.6.2" version = "0.7.0"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[features] [features]
@ -30,7 +30,7 @@ swc_common = {version = "0.10.0", path = "../common"}
swc_ecma_ast = {version = "0.31.0", path = "../ecmascript/ast"} swc_ecma_ast = {version = "0.31.0", path = "../ecmascript/ast"}
swc_ecma_codegen = {version = "0.35.0", path = "../ecmascript/codegen"} swc_ecma_codegen = {version = "0.35.0", path = "../ecmascript/codegen"}
swc_ecma_parser = {version = "0.37.0", path = "../ecmascript/parser"} swc_ecma_parser = {version = "0.37.0", path = "../ecmascript/parser"}
swc_ecma_transforms = {version = "0.23.0", path = "../ecmascript/transforms"} swc_ecma_transforms = {version = "0.23.1", path = "../ecmascript/transforms"}
swc_ecma_utils = {version = "0.21.0", path = "../ecmascript/utils"} swc_ecma_utils = {version = "0.21.0", path = "../ecmascript/utils"}
swc_ecma_visit = {version = "0.17.0", path = "../ecmascript/visit"} swc_ecma_visit = {version = "0.17.0", path = "../ecmascript/visit"}

View File

@ -20,6 +20,7 @@ fn main() {
Config { Config {
require: true, require: true,
external_modules, external_modules,
..Default::default()
}, },
); );
let mut entries = HashMap::default(); let mut entries = HashMap::default();

View File

@ -1,9 +1,12 @@
use super::plan::Plan; use super::plan::Plan;
use crate::{ use crate::{
bundler::load::{Specifier, TransformedModule}, bundler::load::{Specifier, TransformedModule},
util, Bundler, Load, Resolve, util::IntoParallelIterator,
Bundler, Load, Resolve,
}; };
use anyhow::{Context, Error}; use anyhow::{Context, Error};
#[cfg(feature = "concurrent")]
use rayon::iter::ParallelIterator;
use std::mem::{replace, take}; use std::mem::{replace, take};
use swc_common::{Spanned, SyntaxContext, DUMMY_SP}; use swc_common::{Spanned, SyntaxContext, DUMMY_SP};
use swc_ecma_ast::*; use swc_ecma_ast::*;
@ -54,51 +57,53 @@ where
) -> Result<(), Error> { ) -> Result<(), Error> {
log::debug!("merge_reexports: {}", info.fm.name); log::debug!("merge_reexports: {}", info.fm.name);
for (src, specifiers) in &info.exports.reexports { let deps = (&*info.exports.reexports)
log::info!("Merging exports: {} <- {}", info.fm.name, src.src.value); .into_par_iter()
.map(|(src, specifiers)| -> Result<_, Error> {
log::info!("Merging exports: {} <- {}", info.fm.name, src.src.value);
let imported = self.scope.get_module(src.module_id).unwrap(); let imported = self.scope.get_module(src.module_id).unwrap();
assert!(imported.is_es6, "Reexports are es6 only"); assert!(imported.is_es6, "Reexports are es6 only");
info.helpers.extend(&imported.helpers); info.helpers.extend(&imported.helpers);
let (_, dep) = util::join( let mut dep = self
|| { .merge_modules(plan, src.module_id, false, false)
self.run(|| { .with_context(|| {
// entry.visit_mut_with(&mut ExportRenamer { format!(
// from: SyntaxContext::empty().apply_mark(imported. "failed to merge for reexport: ({}):{} <= ({}):{}",
// mark()), info.id, info.fm.name, src.module_id, src.src.value
// to: SyntaxContext::empty().apply_mark(info. )
// mark()), }); })?;
})
},
|| -> Result<_, Error> {
self.run(|| {
let mut dep = self
.merge_modules(plan, src.module_id, false, false)
.with_context(|| {
format!(
"failed to merge for reexport: ({}):{} <= ({}):{}",
info.id, info.fm.name, src.module_id, src.src.value
)
})?;
dep = self.remark_exports(dep, src.ctxt, None, false); // print_hygiene(&format!("dep: start"), &self.cm, &dep);
dep.visit_mut_with(&mut UnexportAsVar { dep = self.remark_exports(dep, src.ctxt, None, false);
dep_ctxt: src.ctxt,
_entry_ctxt: info.ctxt(),
});
dep = dep.fold_with(&mut DepUnexporter { // print_hygiene(&format!("dep: remark exports"), &self.cm, &dep);
exports: &specifiers,
});
Ok(dep) if !specifiers.is_empty() {
}) dep.visit_mut_with(&mut UnexportAsVar {
}, dep_ctxt: src.ctxt,
); _entry_ctxt: info.ctxt(),
let dep = dep?; _exports: &specifiers,
});
// print_hygiene(&format!("dep: unexport as var"), &self.cm, &dep);
dep = dep.fold_with(&mut DepUnexporter {
exports: &specifiers,
});
// print_hygiene(&format!("dep: unexport"), &self.cm, &dep);
}
Ok((src, dep))
})
.collect::<Vec<_>>();
for dep in deps {
let (src, dep) = dep?;
// Replace import statement / require with module body // Replace import statement / require with module body
let mut injector = ExportInjector { let mut injector = ExportInjector {
@ -108,15 +113,11 @@ where
entry.body.visit_mut_with(&mut injector); entry.body.visit_mut_with(&mut injector);
// print_hygiene( // print_hygiene(
// &format!( // &format!("entry:injection {:?} <- {:?}", info.ctxt(), src.ctxt,),
// "entry:injection {:?} <- {:?}",
// SyntaxContext::empty().apply_mark(info.mark()),
// SyntaxContext::empty().apply_mark(imported.mark()),
// ),
// &self.cm, // &self.cm,
// &entry, // &entry,
// ); // );
// assert_eq!(injector.imported, vec![]); assert_eq!(injector.imported, vec![]);
} }
Ok(()) Ok(())
@ -167,41 +168,6 @@ impl VisitMut for ExportInjector {
} }
} }
struct ExportRenamer {
/// Syntax context for the top level.
from: SyntaxContext,
/// Syntax context for the top level nodes of dependency module.
to: SyntaxContext,
}
impl VisitMut for ExportRenamer {
noop_visit_mut_type!();
fn visit_mut_named_export(&mut self, export: &mut NamedExport) {
// if export.src.is_none() {
// return;
// }
export.specifiers.visit_mut_children_with(self);
}
fn visit_mut_export_named_specifier(&mut self, s: &mut ExportNamedSpecifier) {
match &mut s.exported {
Some(v) => {
if v.span.ctxt == self.from {
v.span = v.span.with_ctxt(self.to);
}
}
None => {}
}
if s.orig.span.ctxt == self.from {
s.orig.span = s.orig.span.with_ctxt(self.to);
}
}
fn visit_mut_stmt(&mut self, _: &mut Stmt) {}
}
/// Converts /// Converts
/// ///
/// ```js /// ```js
@ -213,14 +179,17 @@ impl VisitMut for ExportRenamer {
/// ```js /// ```js
/// const e3 = l1; /// const e3 = l1;
/// ``` /// ```
struct UnexportAsVar { struct UnexportAsVar<'a> {
/// Syntax context for the generated variables. /// Syntax context for the generated variables.
dep_ctxt: SyntaxContext, dep_ctxt: SyntaxContext,
_entry_ctxt: SyntaxContext, _entry_ctxt: SyntaxContext,
/// Exports to preserve
_exports: &'a [Specifier],
} }
impl VisitMut for UnexportAsVar { impl VisitMut for UnexportAsVar<'_> {
noop_visit_mut_type!(); noop_visit_mut_type!();
fn visit_mut_module_item(&mut self, n: &mut ModuleItem) { fn visit_mut_module_item(&mut self, n: &mut ModuleItem) {

View File

@ -75,6 +75,16 @@ where
.filter(|(src, _)| { .filter(|(src, _)| {
log::trace!("Checking: {} <= {}", info.fm.name, src.src.value); log::trace!("Checking: {} <= {}", info.fm.name, src.src.value);
// Import and export from same file. We use export to merge it.
if info
.exports
.reexports
.iter()
.any(|(es, _)| es.module_id == src.module_id)
{
return false;
}
// Skip if a dependency is going to be merged by other dependency // Skip if a dependency is going to be merged by other dependency
module_plan.chunks.contains(&src.module_id) module_plan.chunks.contains(&src.module_id)
}) })
@ -199,7 +209,7 @@ where
let mut dep = self.merge_modules(plan, id, false, true)?; let mut dep = self.merge_modules(plan, id, false, true)?;
dep = self.remark_exports(dep, dep_info.ctxt(), None, true); dep = self.remark_exports(dep, dep_info.ctxt(), None, true);
// dep = dep.fold_with(&mut Unexporter); dep = dep.fold_with(&mut Unexporter);
// As transitive deps can have no direct relation with entry, // As transitive deps can have no direct relation with entry,
// remark_exports is not enough. // remark_exports is not enough.

View File

@ -385,11 +385,10 @@ where
); );
builder.try_add_direct_dep(root_id, module_id, src.module_id); builder.try_add_direct_dep(root_id, module_id, src.module_id);
builder let rev = builder.reverse.entry(src.module_id).or_default();
.reverse if !rev.contains(&module_id) {
.entry(src.module_id) rev.push(module_id);
.or_default() }
.push(module_id);
} }
} }
} }

View File

@ -253,86 +253,189 @@ impl Fold for ExportRenamer<'_> {
} }
ModuleItem::ModuleDecl(ModuleDecl::ExportNamed(e)) if e.src.is_none() => { ModuleItem::ModuleDecl(ModuleDecl::ExportNamed(e)) if e.src.is_none() => {
let mut var_decls = Vec::with_capacity(e.specifiers.len()); if self.unexport {
let mut var_decls = Vec::with_capacity(e.specifiers.len());
e.specifiers.into_iter().for_each(|specifier| { e.specifiers.into_iter().for_each(|specifier| {
let span = specifier.span(); let span = specifier.span();
let ident = match &specifier { let ident = match &specifier {
// TODO // TODO
ExportSpecifier::Namespace(s) => self.aliased_import(&s.name.sym), ExportSpecifier::Namespace(s) => self.aliased_import(&s.name.sym),
ExportSpecifier::Default(..) => self.aliased_import(&js_word!("default")), ExportSpecifier::Default(..) => {
ExportSpecifier::Named(s) => { self.aliased_import(&js_word!("default"))
if let Some(exported) = &s.exported { }
// We need remarking ExportSpecifier::Named(s) => {
if let Some(exported) = &s.exported {
// We need remarking
match self.aliased_import(&exported.sym) { match self.aliased_import(&exported.sym) {
Some(v) => { Some(v) => {
let ctxt = self let ctxt = self.mark_as_remarking_required(
.mark_as_remarking_required(v.clone(), s.orig.to_id()); v.clone(),
log::trace!( s.orig.to_id(),
"exported = {}{:?}", );
exported.sym, log::trace!(
exported.span.ctxt "exported = {}{:?}",
); exported.sym,
log::trace!("id = {:?}", v); exported.span.ctxt
log::trace!( );
"orig = {}{:?}", log::trace!("id = {:?}", v);
s.orig.sym, log::trace!(
s.orig.span.ctxt() "orig = {}{:?}",
); s.orig.sym,
s.orig.span.ctxt()
);
Some((v.0, ctxt)) Some((v.0, ctxt))
}
None => None,
} }
None => None, } else {
} match self.aliased_import(&s.orig.sym) {
} else { Some(id) => {
match self.aliased_import(&s.orig.sym) { let ctxt = self.mark_as_remarking_required(
Some(id) => { (s.orig.sym.clone(), self.dep_ctxt),
let ctxt = self.mark_as_remarking_required( s.orig.to_id(),
(s.orig.sym.clone(), self.dep_ctxt), );
s.orig.to_id(),
);
Some((id.0, ctxt)) Some((id.0, ctxt))
}
None => None,
} }
None => None,
} }
} }
}
};
if let Some(i) = ident {
let orig = match specifier {
// TODO
ExportSpecifier::Namespace(s) => s.name,
ExportSpecifier::Default(..) => Ident::new(js_word!("default"), span),
ExportSpecifier::Named(s) => s.orig,
}; };
var_decls.push(VarDeclarator { if let Some(i) = ident {
let orig = match specifier {
// TODO
ExportSpecifier::Namespace(s) => s.name,
ExportSpecifier::Default(..) => {
Ident::new(js_word!("default"), span)
}
ExportSpecifier::Named(s) => s.orig,
};
var_decls.push(VarDeclarator {
span,
name: Pat::Ident(Ident::new(i.0, DUMMY_SP.with_ctxt(i.1))),
init: Some(Box::new(Expr::Ident(orig))),
definite: false,
})
} else {
log::debug!(
"Removing export specifier {:?} as it's not imported",
specifier
);
}
});
if !var_decls.is_empty() {
self.extras.push(Stmt::Decl(Decl::Var(VarDecl {
span, span,
name: Pat::Ident(Ident::new(i.0, DUMMY_SP.with_ctxt(i.1))), kind: VarDeclKind::Const,
init: Some(Box::new(Expr::Ident(orig))), declare: false,
definite: false, decls: var_decls,
}) })))
} else {
log::debug!(
"Removing export specifier {:?} as it's not imported",
specifier
);
} }
});
if !var_decls.is_empty() { return Stmt::Empty(EmptyStmt { span }).into();
self.extras.push(Stmt::Decl(Decl::Var(VarDecl { } else {
span, let mut export_specifiers = Vec::with_capacity(e.specifiers.len());
kind: VarDeclKind::Const,
declare: false, e.specifiers.into_iter().for_each(|specifier| {
decls: var_decls, let span = specifier.span();
}))) let ident = match &specifier {
// TODO
ExportSpecifier::Namespace(s) => self.aliased_import(&s.name.sym),
ExportSpecifier::Default(..) => {
self.aliased_import(&js_word!("default"))
}
ExportSpecifier::Named(s) => {
if let Some(exported) = &s.exported {
// We need remarking
match self.aliased_import(&exported.sym) {
Some(v) => {
let ctxt = self.mark_as_remarking_required(
v.clone(),
s.orig.to_id(),
);
log::trace!(
"exported = {}{:?}",
exported.sym,
exported.span.ctxt
);
log::trace!("id = {:?}", v);
log::trace!(
"orig = {}{:?}",
s.orig.sym,
s.orig.span.ctxt()
);
Some((v.0, ctxt))
}
None => None,
}
} else {
match self.aliased_import(&s.orig.sym) {
Some(id) => {
let ctxt = self.mark_as_remarking_required(
(s.orig.sym.clone(), self.dep_ctxt),
s.orig.to_id(),
);
Some((id.0, ctxt))
}
None => None,
}
}
}
};
if let Some(i) = ident {
let orig = match specifier {
// TODO
ExportSpecifier::Namespace(s) => s.name,
ExportSpecifier::Default(..) => {
Ident::new(js_word!("default"), span)
}
ExportSpecifier::Named(s) => s.orig,
};
export_specifiers.push(ExportSpecifier::Named(ExportNamedSpecifier {
span,
orig,
exported: Some(Ident::new(i.0, DUMMY_SP.with_ctxt(i.1))),
}));
// export_specifiers.push(VarDeclarator {
// span,
// name: Pat::Ident(Ident::new(i.0,
// DUMMY_SP.with_ctxt(i.1))),
// init: Some(Box::new(Expr::Ident(orig))),
// definite: false,
// })
} else {
log::debug!(
"Removing export specifier {:?} as it's not imported (`unexport` \
is false, but it's not used)",
specifier
);
}
});
if !export_specifiers.is_empty() {
return ModuleItem::ModuleDecl(ModuleDecl::ExportNamed(NamedExport {
type_only: false,
span: e.span,
specifiers: export_specifiers,
src: None,
}));
}
return Stmt::Empty(EmptyStmt { span }).into();
} }
return Stmt::Empty(EmptyStmt { span }).into();
} }
ModuleItem::ModuleDecl(ModuleDecl::ExportDecl(decl)) => { ModuleItem::ModuleDecl(ModuleDecl::ExportDecl(decl)) => {

View File

@ -24,7 +24,7 @@ where
let mut renamed = HashMap::default(); let mut renamed = HashMap::default();
for mut bundle in bundles { for mut bundle in bundles {
bundle.module = self.drop_unused(bundle.module); bundle.module = self.optimize(bundle.module);
match bundle.kind { match bundle.kind {
BundleKind::Named { .. } => { BundleKind::Named { .. } => {
// Inject helpers // Inject helpers

View File

@ -13,15 +13,22 @@ mod finalize;
mod helpers; mod helpers;
mod import; mod import;
mod load; mod load;
mod optimize;
mod scope; mod scope;
#[cfg(test)] #[cfg(test)]
mod tests; mod tests;
mod usage_analysis;
#[derive(Debug)] #[derive(Debug, Default)]
pub struct Config { pub struct Config {
/// If it's true, [Bundler] searches for require calls. /// If it's true, [Bundler] searches for require calls.
pub require: bool, pub require: bool,
/// If it's true, many temporary variables will be generated.
///
/// This option exists mainly for testing. As inlining and dce removes all
/// temporary variables, it's really hard to see what's going on.
pub disable_inliner: bool,
/// List of modules which should be preserved. /// List of modules which should be preserved.
pub external_modules: Vec<JsWord>, pub external_modules: Vec<JsWord>,
} }

View File

@ -0,0 +1,29 @@
use crate::{Bundler, Load, Resolve};
use swc_ecma_ast::*;
use swc_ecma_transforms::optimization::simplify::{dce, inlining};
use swc_ecma_visit::FoldWith;
impl<L, R> Bundler<'_, L, R>
where
L: Load,
R: Resolve,
{
/// If used_exports is [None], all exports are treated as exported.
///
/// Note: Context of used_exports is ignored, as the specifiers comes from
/// other module.
pub(super) fn optimize(&self, mut node: Module) -> Module {
self.run(|| {
if !self.config.disable_inliner {
node = node.fold_with(&mut inlining::inlining(inlining::Config {}))
}
node = node.fold_with(&mut dce::dce(dce::Config {
used: None,
used_mark: self.used_mark,
}));
node
})
}
}

View File

@ -128,6 +128,7 @@ impl TestBuilder {
Default::default(), Default::default(),
Config { Config {
require: true, require: true,
disable_inliner: true,
external_modules: vec![], external_modules: vec![],
}, },
); );

View File

@ -1,43 +0,0 @@
use crate::{Bundler, Load, Resolve};
use swc_ecma_ast::*;
use swc_ecma_transforms::optimization::simplify::dce;
use swc_ecma_visit::FoldWith;
impl<L, R> Bundler<'_, L, R>
where
L: Load,
R: Resolve,
{
/// If used_exports is [None], all exports are treated as exported.
///
/// Note: Context of used_exports is ignored, as the specifiers comes from
/// other module.
pub(super) fn drop_unused(&self, node: Module) -> Module {
self.run(|| {
// let mut used = vec![];
// if let Some(used_exports) = used_exports {
// for export in used_exports {
// match export {
// Specifier::Specific { alias, local, .. } => {
// used.push(alias.as_ref().unwrap_or(local).to_id());
// }
// Specifier::Namespace { local, .. } => {
// used.push(local.to_id());
// }
// }
// }
// }
let used_mark = self.used_mark;
let mut v = dce::dce(dce::Config {
used: None,
used_mark,
});
let node = node.fold_with(&mut v);
node
})
}
}

View File

@ -85,6 +85,7 @@ impl Task for BundleTask {
.into_iter() .into_iter()
.map(From::from) .map(From::from)
.collect(), .collect(),
..Default::default()
}, },
); );

View File

@ -136,6 +136,7 @@ fn reference_tests(tests: &mut Vec<TestDescAndFn>, errors: bool) -> Result<(), i
NodeResolver::new(), NodeResolver::new(),
Config { Config {
require: true, require: true,
disable_inliner: true,
external_modules: vec![ external_modules: vec![
"assert", "assert",
"buffer", "buffer",

View File

@ -1,9 +1,9 @@
console.log('c');
class A { class A {
method() { method() {
return new B(); return new B();
} }
} }
console.log('c');
class B extends A { class B extends A {
} }
console.log(A, B); console.log(A, B);

View File

@ -0,0 +1 @@
export * from './b';

View File

@ -0,0 +1 @@
export * from './c'

View File

@ -0,0 +1,3 @@
export const a = 1;
export const b = 2;
export const c = 3;

View File

@ -0,0 +1 @@
export * from './a'

View File

@ -0,0 +1,3 @@
export const a = 1;
export const b = 2;
export const c = 3;

View File

@ -0,0 +1,2 @@
export const a = 1;
export * from './b'

View File

@ -0,0 +1 @@
export const b = 2;

View File

@ -0,0 +1 @@
export * from './a'

View File

@ -0,0 +1,2 @@
export const a = 1;
export const b = 2;

View File

@ -0,0 +1,3 @@
export const a = 1;
export const b = 2;
export const c = 3;

View File

@ -0,0 +1,5 @@
import { a } from './a';
export * from './a'
console.log(a);

View File

@ -0,0 +1,4 @@
export const a = 1;
export const b = 2;
export const c = 3;
console.log(a);

View File

@ -0,0 +1,2 @@
export { c } from './c';
export { d } from './d';

View File

@ -0,0 +1,2 @@
export const b = 1;
export { e } from './e';

View File

@ -0,0 +1 @@
export const c = 3;

View File

@ -0,0 +1 @@
export const d = 4;

View File

@ -0,0 +1 @@
export const e = 5;

View File

@ -0,0 +1,2 @@
export * from './a';
export * from './b';

View File

@ -0,0 +1,7 @@
const c1 = 3;
export { c1 as c };
const d1 = 4;
export { d1 as d };
export const b = 1;
const e1 = 5;
export { e1 as e };

View File

@ -0,0 +1,2 @@
export { c as d } from './c';
export { d as c } from './d';

View File

@ -0,0 +1,2 @@
export const b = 1;
export { e as a } from './e';

View File

@ -0,0 +1 @@
export const c = 3;

View File

@ -0,0 +1 @@
export const d = 4;

View File

@ -0,0 +1 @@
export const e = 5;

View File

@ -0,0 +1,2 @@
export * from './a';
export * from './b';

View File

@ -0,0 +1,7 @@
const c1 = 3;
export { c1 as d };
const d = 4;
export { d as c };
export const b = 1;
const e = 5;
export { e as a };

View File

@ -0,0 +1,3 @@
import { common } from './common';
console.log(common, 'a.js')

View File

@ -0,0 +1,3 @@
import { common } from './common';
console.log(common, 'b.js')

View File

@ -0,0 +1 @@
export const common = 1;

View File

@ -0,0 +1,2 @@
import './a';
import './b';

View File

@ -0,0 +1,3 @@
const common = 1;
console.log(common, 'a.js');
console.log(common, 'b.js');

View File

@ -0,0 +1,4 @@
import common1 from './common1';
import common2 from './common2';
console.log('a', common1, common2)

View File

@ -0,0 +1,4 @@
import common3 from './common3';
import common1 from './common1';
console.log('b', common3, common1)

View File

@ -0,0 +1,5 @@
import common4 from './common4';
import common2 from './common2';
import common3 from './common3';
console.log('c', common4, common2, common3)

View File

@ -0,0 +1 @@
export const common1 = 1;

View File

@ -0,0 +1,3 @@
const common = 2;
export { common as common2 }

View File

@ -0,0 +1,4 @@
export const common1 = 'Test failed :(';
const common3 = 3;
export { common3 }

View File

@ -0,0 +1,8 @@
var common4;
try {
common4 = 4;
} catch (e) {
}
export { common4 }

View File

@ -0,0 +1,4 @@
import './a'
import './b'
import './c'
import './common3'

View File

@ -0,0 +1,13 @@
const common1 = 1;
const common = 2;
const common2 = common;
console.log('a', common1, common2);
const common3 = 3;
const common31 = common3;
console.log('b', common31, common1);
var common4;
try {
common4 = 4;
} catch (e) {
}
console.log('c', common4, common2, common31);