feat(rplugin): Implement general AST processing plugin system (#2671)

swc_common:
 - Implement `StableAbi` for `BytePos`.
 - Implement `StableAbi` for `Span`.

rplugin:
 - Add `StableAst`.

swc_plugin_api:
 - Drop unused dependencies.

swc_ecma_plugin_ast:
 - Initialize crate.
This commit is contained in:
Donny/강동윤 2021-11-09 18:46:24 +09:00 committed by GitHub
parent 797f4fff14
commit bf0007bec0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 2186 additions and 8 deletions

44
Cargo.lock generated
View File

@ -2024,6 +2024,27 @@ version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "448296241d034b96c11173591deaa1302f2c17b56092106c1f92c1bc0183a8c9"
[[package]]
name = "rplugin"
version = "0.1.0"
dependencies = [
"abi_stable",
"rplugin_macros",
"string_cache",
"swc_common",
]
[[package]]
name = "rplugin_macros"
version = "0.1.0"
dependencies = [
"pmutil",
"proc-macro2",
"quote",
"swc_macros_common",
"syn",
]
[[package]]
name = "rustc-demangle"
version = "0.1.21"
@ -2499,7 +2520,7 @@ dependencies = [
[[package]]
name = "swc_common"
version = "0.14.5"
version = "0.14.6"
dependencies = [
"abi_stable",
"ahash",
@ -2763,6 +2784,23 @@ dependencies = [
"walkdir",
]
[[package]]
name = "swc_ecma_plugin_ast"
version = "0.1.0"
dependencies = [
"abi_stable",
"num-bigint",
"rplugin",
"serde",
"serde_json",
"swc_atoms 0.2.9",
"swc_common",
"swc_ecma_ast",
"swc_ecma_parser",
"swc_node_base",
"testing",
]
[[package]]
name = "swc_ecma_preset_env"
version = "0.64.0"
@ -3150,7 +3188,7 @@ dependencies = [
[[package]]
name = "swc_plugin_api"
version = "0.3.0"
version = "0.3.1"
dependencies = [
"abi_stable",
"anyhow",
@ -3158,8 +3196,6 @@ dependencies = [
"serde_json",
"swc_atoms 0.2.9",
"swc_common",
"swc_ecma_ast",
"swc_ecma_visit",
]
[[package]]

View File

@ -3,6 +3,7 @@ members = [
"css",
"css/stylis",
"ecmascript",
"ecmascript/plugin_ast",
"ecmascript/babel/compat",
"ecmascript/jsdoc",
"node/binding",

View File

@ -6,7 +6,7 @@ edition = "2018"
license = "Apache-2.0/MIT"
name = "swc_common"
repository = "https://github.com/swc-project/swc.git"
version = "0.14.5"
version = "0.14.6"
[lib]
crate-type = ["lib", "dylib"]

View File

@ -26,6 +26,8 @@ pub mod hygiene;
/// assume that the length of the `span = hi - lo`; there may be space in the
/// `BytePos` range between files.
#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, Ord, PartialOrd, Serialize, Deserialize)]
#[cfg_attr(feature = "abi_stable", repr(C))]
#[cfg_attr(feature = "abi_stable", derive(abi_stable::StableAbi))]
pub struct Span {
#[serde(rename = "start")]
pub lo: BytePos,
@ -800,6 +802,8 @@ pub trait Pos {
/// a lot of them.
#[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Debug, Serialize, Deserialize)]
#[serde(transparent)]
#[cfg_attr(feature = "abi_stable", repr(transparent))]
#[cfg_attr(feature = "abi_stable", derive(abi_stable::StableAbi))]
#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))]
pub struct BytePos(pub u32);

View File

@ -100,6 +100,7 @@
"regs",
"repr",
"rfind",
"rplugin",
"rpos",
"rposition",
"rsplit",

View File

@ -0,0 +1,25 @@
[package]
authors = ["강동윤 <kdy1997.dev@gmail.com>"]
description = "Special AST for swc, which can be passed to dynamic libraries directly"
edition = "2018"
license = "Apache-2.0/MIT"
name = "swc_ecma_plugin_ast"
repository = "https://github.com/swc-project/swc.git"
version = "0.1.0"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
abi_stable = "0.10.3"
num-bigint = "0.2"
rplugin = {version = "0.1", path = "../../rplugin/"}
swc_atoms = {version = "0.2", path = "../../atoms/"}
swc_common = {version = "0.14.6", path = "../../common/", features = ["plugin-base"]}
swc_ecma_ast = {version = "0.57.0", path = "../ast/"}
[dev-dependencies]
serde = "1"
serde_json = "1"
swc_ecma_parser = {version = "0.77", path = "../parser/"}
swc_node_base = {version = "0.5.1", path = "../../node/base"}
testing = {version = "0.15.1", path = "../../testing"}

View File

@ -0,0 +1,86 @@
#![feature(bench_black_box)]
#![feature(test)]
extern crate swc_node_base;
extern crate test;
use rplugin::StableAst;
use std::{hint::black_box, path::Path};
use swc_common::input::SourceFileInput;
use swc_ecma_ast::{EsVersion, Program};
use swc_ecma_parser::{lexer::Lexer, Parser};
use test::Bencher;
fn input() -> Program {
testing::run_test(false, |cm, _handler| {
let fm = cm.load_file(Path::new("benches/input.js")).unwrap();
let lexer = Lexer::new(
Default::default(),
EsVersion::latest(),
SourceFileInput::from(&*fm),
None,
);
let mut parser = Parser::new_from(lexer);
let program = parser.parse_program().unwrap();
Ok(program)
})
.unwrap()
}
#[bench]
fn json_serialize(b: &mut Bencher) {
let program = input();
b.iter(|| {
let v = serde_json::to_string(&program).unwrap();
black_box(v);
})
}
#[bench]
fn json_deserialize(b: &mut Bencher) {
let program = input();
let json_str = serde_json::to_string(&program).unwrap();
b.iter(|| {
let v: Program = serde_json::from_str(&json_str).unwrap();
black_box(v);
})
}
#[bench]
fn ast_clone(b: &mut Bencher) {
let program = input();
b.iter(|| {
let program = program.clone();
black_box(program);
})
}
#[bench]
fn ast_clone_to_stable(b: &mut Bencher) {
let program = input();
b.iter(|| {
let program = program.clone();
let v = swc_ecma_plugin_ast::Program::from_unstable(program);
black_box(v);
})
}
#[bench]
fn ast_clone_to_stable_then_to_unstable(b: &mut Bencher) {
let program = input();
b.iter(|| {
let program = program.clone();
let stable = swc_ecma_plugin_ast::Program::from_unstable(program);
let v = stable.into_unstable();
black_box(v);
})
}

View File

@ -0,0 +1,108 @@
import { SafeSubscriber, Subscriber } from './Subscriber';
import { isSubscription } from './Subscription';
import { observable as Symbol_observable } from './symbol/observable';
import { pipeFromArray } from './util/pipe';
import { config } from './config';
import { isFunction } from './util/isFunction';
var tmp = Symbol_observable;
export class Observable {
lift(operator) {
const observable = new Observable();
observable.source = this;
observable.operator = operator;
return observable;
}
subscribe(observerOrNext, error, complete) {
const subscriber = isSubscriber(observerOrNext) ? observerOrNext : new SafeSubscriber(observerOrNext, error, complete);
if (config.useDeprecatedSynchronousErrorHandling) {
this._deprecatedSyncErrorSubscribe(subscriber);
} else {
const { operator, source } = this;
subscriber.add(operator ? operator.call(subscriber, source) : source ? this._subscribe(subscriber) : this._trySubscribe(subscriber));
}
return subscriber;
}
_deprecatedSyncErrorSubscribe(subscriber) {
const localSubscriber = subscriber;
localSubscriber._syncErrorHack_isSubscribing = true;
const { operator } = this;
if (operator) {
subscriber.add(operator.call(subscriber, this.source));
} else {
try {
subscriber.add(this._subscribe(subscriber));
} catch (err) {
localSubscriber.__syncError = err;
}
}
let dest = localSubscriber;
while (dest) {
if ('__syncError' in dest) {
try {
throw dest.__syncError;
} finally {
subscriber.unsubscribe();
}
}
dest = dest.destination;
}
localSubscriber._syncErrorHack_isSubscribing = false;
}
_trySubscribe(sink) {
try {
return this._subscribe(sink);
} catch (err) {
sink.error(err);
}
}
forEach(next, promiseCtor2) {
promiseCtor2 = getPromiseCtor(promiseCtor2);
return new promiseCtor2((resolve, reject) => {
let subscription;
subscription = this.subscribe((value) => {
try {
next(value);
} catch (err) {
reject(err);
subscription?.unsubscribe();
}
}, reject, resolve);
});
}
_subscribe(subscriber1) {
return this.source?.subscribe(subscriber1);
}
[tmp]() {
return this;
}
pipe(...operations) {
return operations.length ? pipeFromArray(operations)(this) : this;
}
toPromise(promiseCtor1) {
promiseCtor1 = getPromiseCtor(promiseCtor1);
return new promiseCtor1((resolve, reject) => {
let value;
this.subscribe((x) => value = x
, (err) => reject(err)
, () => resolve(value)
);
});
}
constructor(subscribe1) {
if (subscribe1) {
this._subscribe = subscribe1;
}
}
}
Observable.create = (subscribe) => {
return new Observable(subscribe);
};
function getPromiseCtor(promiseCtor) {
return (promiseCtor ?? config.Promise) ?? Promise;
}
function isObserver(value) {
return value && isFunction(value.next) && isFunction(value.error) && isFunction(value.complete);
}
function isSubscriber(value) {
return value && value instanceof Subscriber || isObserver(value) && isSubscription(value);
}

File diff suppressed because it is too large Load Diff

View File

@ -6,7 +6,7 @@ edition = "2018"
license = "Apache-2.0/MIT"
name = "swc_plugin_api"
repository = "https://github.com/swc-project/swc.git"
version = "0.3.0"
version = "0.3.1"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@ -17,5 +17,3 @@ serde = "1.0.126"
serde_json = "1"
swc_atoms = {version = "0.2.7", path = "../../atoms"}
swc_common = {version = "0.14.5", path = "../../common", features = ["plugin-base"]}
swc_ecma_ast = {version = "0.57.0", path = "../../ecmascript/ast"}
swc_ecma_visit = {version = "0.43.0", path = "../../ecmascript/visit"}

17
rplugin/Cargo.toml Normal file
View File

@ -0,0 +1,17 @@
[package]
authors = ["강동윤 <kdy1997.dev@gmail.com>"]
description = "Rust plugin system for AST processing"
edition = "2018"
include = ["Cargo.toml", "src/**/*.rs"]
license = "Apache-2.0/MIT"
name = "rplugin"
repository = "https://github.com/swc-project/swc.git"
version = "0.1.0"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
abi_stable = "0.10.3"
rplugin_macros = {version = "0.1", path = "./macros/"}
string_cache = "0.8.2"
swc_common = {version = "0.14.6", path = "../common/"}

6
rplugin/README.md Normal file
View File

@ -0,0 +1,6 @@
# rplugin
General utilities for building plugin system for processing AST.
Refer to [rustdoc][] for more details.
[rustdoc]: https://docs.rs/rplugin

19
rplugin/macros/Cargo.toml Normal file
View File

@ -0,0 +1,19 @@
[package]
authors = ["강동윤 <kdy1997.dev@gmail.com>"]
description = "Macros for rpluign, rust plugin system for AST processing"
edition = "2018"
include = ["Cargo.toml", "src/**/*.rs"]
license = "Apache-2.0/MIT"
name = "rplugin_macros"
repository = "https://github.com/swc-project/swc.git"
version = "0.1.0"
[lib]
proc-macro = true
[dependencies]
pmutil = "0.5.3"
proc-macro2 = "1"
quote = "1"
swc_macros_common = {version = "0.3.2", path = "../../macros/common"}
syn = {version = "1", features = ["full"]}

344
rplugin/macros/src/lib.rs Normal file
View File

@ -0,0 +1,344 @@
extern crate proc_macro;
use pmutil::{q, ToTokensExt};
use swc_macros_common::{
call_site,
prelude::{BindedField, VariantBinder},
};
use syn::{
parse, punctuated::Punctuated, Arm, Attribute, Expr, ExprMatch, ExprStruct, Field, FieldValue,
Fields, Index, Item, ItemEnum, ItemImpl, ItemMod, ItemStruct, Member, Path, Token, Type,
Visibility,
};
#[proc_macro_attribute]
pub fn ast_for_plugin(
input: proc_macro::TokenStream,
module: proc_macro::TokenStream,
) -> proc_macro::TokenStream {
let normal_crate_path: Path = parse(input).expect("failed to parse argument as path");
let module: ItemMod = parse(module).expect("failed to parse input as a module");
let content = module.content.expect("module should have content").1;
let mut mod_items = vec![];
for mut item in content {
match &mut item {
Item::Struct(s) => {
add_stable_abi(&mut s.attrs);
patch_fields(&mut s.fields);
mod_items.push(Item::Impl(make_unstable_ast_impl_for_struct(
&normal_crate_path,
&s,
)));
}
Item::Enum(s) => {
add_stable_abi(&mut s.attrs);
for v in s.variants.iter_mut() {
patch_fields(&mut v.fields);
}
mod_items.push(Item::Impl(make_unstable_ast_impl_for_enum(
&normal_crate_path,
&s,
)));
}
_ => {}
}
mod_items.push(item);
}
ItemMod {
content: Some((call_site(), mod_items)),
..module
}
.dump()
.into()
}
fn patch_fields(f: &mut Fields) {
match f {
Fields::Named(f) => {
f.named.iter_mut().for_each(|f| {
patch_field(f);
});
}
Fields::Unnamed(f) => {
f.unnamed.iter_mut().for_each(|f| {
patch_field(f);
});
}
Fields::Unit => {}
}
}
fn patch_field(f: &mut Field) {
f.vis = Visibility::Inherited;
f.ty = patch_field_type(&f.ty);
}
fn patch_field_type(ty: &Type) -> Type {
if let Some(ty) = get_generic(&ty, "Box") {
return q!(
Vars {
ty: patch_field_type(ty),
},
(abi_stable::std_types::RBox<ty>)
)
.parse();
}
if let Some(ty) = get_generic(ty, "Vec") {
return q!(
Vars {
ty: patch_field_type(ty),
},
(abi_stable::std_types::RVec<ty>)
)
.parse();
}
if let Some(ty) = get_generic(ty, "Option") {
return q!(
Vars {
ty: patch_field_type(ty),
},
(abi_stable::std_types::ROption<ty>)
)
.parse();
}
match ty {
Type::Path(p) => {
if p.path.is_ident("String") {
return q!((abi_stable::std_types::RString)).parse();
}
}
_ => {}
}
ty.clone()
}
fn get_generic<'a>(ty: &'a Type, wrapper_name: &str) -> Option<&'a Type> {
match ty {
Type::Path(ty) => {
let ident = ty.path.segments.first().unwrap();
if ident.ident != wrapper_name {
return None;
}
match &ident.arguments {
syn::PathArguments::None => None,
syn::PathArguments::AngleBracketed(generic) => generic
.args
.iter()
.filter_map(|arg| match arg {
syn::GenericArgument::Type(ty) => Some(ty),
_ => None,
})
.next(),
syn::PathArguments::Parenthesized(_) => todo!(),
}
}
_ => None,
}
}
fn add_stable_abi(attrs: &mut Vec<Attribute>) {
let s = q!({
#[repr(C)]
#[derive(abi_stable::StableAbi)]
struct Dummy;
})
.parse::<ItemStruct>();
attrs.extend(s.attrs)
}
fn make_unstable_ast_impl_for_struct(normal_crate_path: &Path, src: &ItemStruct) -> ItemImpl {
let binder = [VariantBinder::new(
None,
&src.ident,
&src.fields,
&src.attrs,
)];
q!(
Vars {
normal_crate_path,
Type: &src.ident,
from_unstable_body: make_from_unstable_impl_body(normal_crate_path, &binder),
into_unstable_body: make_into_unstable_impl_body(normal_crate_path, &binder),
},
{
impl rplugin::StableAst for Type {
type Unstable = normal_crate_path::Type;
fn from_unstable(unstable_node: Self::Unstable) -> Self {
from_unstable_body
}
fn into_unstable(self) -> Self::Unstable {
into_unstable_body
}
}
}
)
.parse()
}
fn make_unstable_ast_impl_for_enum(normal_crate_path: &Path, src: &ItemEnum) -> ItemImpl {
let binder = src
.variants
.iter()
.map(|v| VariantBinder::new(Some(&src.ident), &v.ident, &v.fields, &v.attrs))
.collect::<Vec<_>>();
q!(
Vars {
normal_crate_path,
Type: &src.ident,
from_unstable_body: make_from_unstable_impl_body(normal_crate_path, &binder),
into_unstable_body: make_into_unstable_impl_body(normal_crate_path, &binder),
},
{
impl rplugin::StableAst for Type {
type Unstable = normal_crate_path::Type;
fn from_unstable(unstable_node: Self::Unstable) -> Self {
from_unstable_body
}
fn into_unstable(self) -> Self::Unstable {
into_unstable_body
}
}
}
)
.parse()
}
fn make_field_values<F>(fields: Vec<BindedField>, mut op: F) -> Punctuated<FieldValue, Token![,]>
where
F: FnMut(&BindedField) -> Expr,
{
fields
.into_iter()
.map(|f| {
// Call from_unstable for each field
FieldValue {
attrs: Default::default(),
member: f
.field()
.ident
.clone()
.map(Member::Named)
.unwrap_or_else(|| Member::Unnamed(Index::from(f.idx()))),
colon_token: Some(call_site()),
expr: op(&f),
}
})
.collect()
}
fn make_from_unstable_impl_body(normal_crate_path: &Path, variants: &[VariantBinder]) -> Expr {
let mut arms = vec![];
for v in variants {
let (pat, fields) = v.bind("_", None, None);
let fields = make_field_values(fields, |f| {
q!(Vars { name: &f.name() }, {
rplugin::StableAst::from_unstable(name)
})
.parse()
});
let pat = q!(
Vars {
pat,
normal_crate_path
},
{ normal_crate_path::pat }
)
.parse();
arms.push(Arm {
attrs: Default::default(),
pat,
guard: Default::default(),
fat_arrow_token: call_site(),
body: Box::new(Expr::Struct(ExprStruct {
attrs: Default::default(),
path: v.qual_path(),
fields,
dot2_token: None,
brace_token: call_site(),
rest: None,
})),
comma: Some(call_site()),
});
}
Expr::Match(ExprMatch {
attrs: Default::default(),
match_token: call_site(),
expr: q!((unstable_node)).parse(),
brace_token: call_site(),
arms,
})
}
fn make_into_unstable_impl_body(normal_crate_path: &Path, variants: &[VariantBinder]) -> Expr {
let mut arms = vec![];
for v in variants {
let (pat, fields) = v.bind("_", None, None);
let fields = make_field_values(fields, |f| {
q!(Vars { name: &f.name() }, {
rplugin::StableAst::into_unstable(name)
})
.parse()
});
arms.push(Arm {
attrs: Default::default(),
pat,
guard: Default::default(),
fat_arrow_token: call_site(),
body: Box::new(Expr::Struct(ExprStruct {
attrs: Default::default(),
path: q!(
Vars {
normal_crate_path,
Type: v.qual_path()
},
{ normal_crate_path::Type }
)
.parse(),
fields,
dot2_token: None,
brace_token: call_site(),
rest: None,
})),
comma: Some(call_site()),
});
}
Expr::Match(ExprMatch {
attrs: Default::default(),
match_token: call_site(),
expr: q!((self)).parse(),
brace_token: call_site(),
arms,
})
}

152
rplugin/src/lib.rs Normal file
View File

@ -0,0 +1,152 @@
//! AST processing plugin system.
//!
//! Designed as a general crate, to support using it from other libraries too.
//! This depends on the [abi_stable] crate.
//!
//! # Why new crate?
//!
//! I (kdy1) wants to pass the AST data to plugin without having to change the
//! type of AST nodes.
//!
//! e.g. Using [RBox](abi_stable::std_types::RBox) instead of
//! [Box](std::boxed::Box) is not an acceptable option, as many users already
//! use swc as a crate.
//!
//! Instead, we just use proc-macro to convert ast into something that can be
//! passed directly to the plugin, using [StableAbi][abi_stable::StableAbi]. In
//! plugin, we convert it back to normal AST before processing, so the plugin
//! authors can use exactly same code as core transforms. Of course, this is
//! slow, but it's only slow if it's compared to the changing type. It's much
//! faster than serializing/deserializing.
//!
//! In short, we do
//!
//! `Normal AST -> Plugin AST -> Normal AST -> plugin -> Normal AST -> Plugin
//! AST -> Normal AST`
//!
//! # Note for potential users
//!
//! Currently this crate depends on `swc_common`, but if there's a request, I
//! can change it to not depend on `swc_common`.
use abi_stable::{
std_types::{RBox, ROption, RString, RVec},
StableAbi,
};
pub use rplugin_macros::ast_for_plugin;
/// Implemented by AST nodes that has stable ABI, and passable to plugin.
pub trait StableAst: StableAbi {
type Unstable: Sized;
fn from_unstable(n: Self::Unstable) -> Self;
fn into_unstable(self) -> Self::Unstable;
}
impl<T> StableAst for RBox<T>
where
T: StableAst,
{
type Unstable = Box<T::Unstable>;
fn from_unstable(n: Self::Unstable) -> Self {
RBox::new(T::from_unstable(*n))
}
fn into_unstable(self) -> Self::Unstable {
Box::new(T::into_unstable(RBox::into_inner(self)))
}
}
impl<T> StableAst for ROption<T>
where
T: StableAst,
{
type Unstable = Option<T::Unstable>;
fn from_unstable(n: Self::Unstable) -> Self {
match n {
Some(v) => ROption::RSome(T::from_unstable(v)),
None => ROption::RNone,
}
}
fn into_unstable(self) -> Self::Unstable {
match self {
ROption::RSome(v) => Some(v.into_unstable()),
ROption::RNone => None,
}
}
}
impl<T> StableAst for RVec<T>
where
T: StableAst,
{
type Unstable = Vec<T::Unstable>;
fn from_unstable(n: Self::Unstable) -> Self {
n.into_iter().map(T::from_unstable).collect()
}
fn into_unstable(self) -> Self::Unstable {
self.into_iter().map(T::into_unstable).collect()
}
}
macro_rules! as_is {
($T:ty) => {
/// This is as-is
impl StableAst for $T {
type Unstable = $T;
#[inline(always)]
fn from_unstable(n: Self::Unstable) -> Self {
n
}
#[inline(always)]
fn into_unstable(self) -> Self::Unstable {
self
}
}
};
(
$T:ty,
$($tt:tt),*
) => {
as_is!($T);
as_is!($($tt),*);
};
}
as_is!(bool);
as_is!(usize, u8, u16, u32, u64);
as_is!(isize, i8, i16, i32, i64);
as_is!(f32, f64);
macro_rules! convert {
(
$U:ty as $T:ty
) => {
impl StableAst for $T {
type Unstable = $U;
#[inline(always)]
fn from_unstable(n: Self::Unstable) -> Self {
n.into()
}
#[inline(always)]
fn into_unstable(self) -> Self::Unstable {
self.into()
}
}
};
}
convert!(String as RString);
as_is!(swc_common::Span);

33
rplugin/tests/macro.rs Normal file
View File

@ -0,0 +1,33 @@
pub struct BinExpr {
pub left: Box<Expr>,
pub op: String,
pub right: Box<Expr>,
}
pub enum Expr {
Lit(Lit),
Bin(BinExpr),
}
pub enum Lit {
Number(usize),
}
#[rplugin::ast_for_plugin(crate)]
mod plugin {
pub struct BinExpr {
pub left: Box<Expr>,
pub op: String,
pub right: Box<Expr>,
}
pub enum Expr {
Lit(Lit),
Bin(BinExpr),
}
pub enum Lit {
Number(usize),
}
}