swc-project/swc
1
1
Fork 0
mirror of https://github.com/swc-project/swc.git synced 2024-10-04 04:07:18 +03:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

refactor(plugin): Refactor transform executor (#5147)

Browse Source
This commit is contained in:
OJ Kwon 2022-07-07 23:02:42 -07:00 committed by GitHub
parent 07ffb76793
commit e8214babf5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 180 additions and 156 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

125
crates/swc/src/plugin.rs
View File

@ -95,8 +95,10 @@ impl RustPlugins {
inner: self.comments.clone(), inner: self.comments.clone(),
}, },
|| { || {
let span = tracing::span!(tracing::Level::INFO, "serialize_program").entered();
let program = VersionedSerializable::new(n); let program = VersionedSerializable::new(n);
let mut serialized = PluginSerializedBytes::try_serialize(&program)?; let mut serialized_program = PluginSerializedBytes::try_serialize(&program)?;
drop(span);
// Run plugin transformation against current program. // Run plugin transformation against current program.
// We do not serialize / deserialize between each plugin execution but // We do not serialize / deserialize between each plugin execution but
@ -106,29 +108,6 @@ impl RustPlugins {
// transform. // transform.
if let Some(plugins) = &self.plugins { if let Some(plugins) = &self.plugins {
for p in plugins { for p in plugins {
let span = tracing::span!(
tracing::Level::INFO,
"serialize_context",
plugin_module = p.0.as_str()
);
let context_span_guard = span.enter();
let config_json = serde_json::to_string(&p.1)
.context("Failed to serialize plugin config as json")
.and_then(|value| {
PluginSerializedBytes::try_serialize(&VersionedSerializable::new(
value,
))
})?;
let context_json = serde_json::to_string(&self.plugin_context)
.context("Failed to serialize plugin context as json")
.and_then(|value| {
PluginSerializedBytes::try_serialize(&VersionedSerializable::new(
value,
))
})?;
let resolved_path = self let resolved_path = self
.resolver .resolver
.as_ref() .as_ref()
@ -140,31 +119,66 @@ impl RustPlugins {
} else { } else {
anyhow::bail!("Failed to resolve plugin path: {:?}", resolved_path); anyhow::bail!("Failed to resolve plugin path: {:?}", resolved_path);
}; };
let mut transform_plugin_executor =
swc_plugin_runner::create_plugin_transform_executor(
&path,
&swc_plugin_runner::cache::PLUGIN_MODULE_CACHE,
&self.source_map,
)?;
let span = tracing::span!(
tracing::Level::INFO,
"serialize_context",
plugin_module = p.0.as_str()
);
let context_span_guard = span.enter();
let serialized_config_json = serde_json::to_string(&p.1)
.context("Failed to serialize plugin config as json")
.and_then(|value| {
PluginSerializedBytes::try_serialize(&VersionedSerializable::new(
value,
))
})?;
let serialized_context_json = serde_json::to_string(&self.plugin_context)
.context("Failed to serialize plugin context as json")
.and_then(|value| {
PluginSerializedBytes::try_serialize(&VersionedSerializable::new(
value,
))
})?;
drop(context_span_guard); drop(context_span_guard);
let span = tracing::span!( let span = tracing::span!(
tracing::Level::INFO, tracing::Level::INFO,
"execute_plugin_runner", "execute_plugin_runner",
plugin_module = p.0.as_str() plugin_module = p.0.as_str()
); )
let transform_span_guard = span.enter(); .entered();
serialized = swc_plugin_runner::apply_transform_plugin(
&p.0, serialized_program = transform_plugin_executor
&path, .transform(
&swc_plugin_runner::cache::PLUGIN_MODULE_CACHE, &serialized_program,
serialized, &serialized_config_json,
config_json, &serialized_context_json,
context_json, should_enable_comments_proxy,
should_enable_comments_proxy, )
&self.source_map, .with_context(|| {
)?; format!(
drop(transform_span_guard); "failed to invoke `{}` as js transform plugin at {}",
&p.0,
path.display()
)
})?;
drop(span);
} }
} }
// Plugin transformation is done. Deserialize transformed bytes back // Plugin transformation is done. Deserialize transformed bytes back
// into Program // into Program
serialized.deserialize().map(|v| v.into_inner()) serialized_program.deserialize().map(|v| v.into_inner())
}, },
) )
} }
@ -188,11 +202,11 @@ impl RustPlugins {
}, },
|| { || {
let program = VersionedSerializable::new(n); let program = VersionedSerializable::new(n);
let mut serialized = PluginSerializedBytes::try_serialize(&program)?; let mut serialized_program = PluginSerializedBytes::try_serialize(&program)?;
if let Some(plugins) = &self.plugins { if let Some(plugins) = &self.plugins {
for p in plugins { for p in plugins {
let config_json = serde_json::to_string(&p.1) let serialized_config_json = serde_json::to_string(&p.1)
.context("Failed to serialize plugin config as json") .context("Failed to serialize plugin config as json")
.and_then(|value| { .and_then(|value| {
PluginSerializedBytes::try_serialize(&VersionedSerializable::new( PluginSerializedBytes::try_serialize(&VersionedSerializable::new(
@ -200,7 +214,7 @@ impl RustPlugins {
)) ))
})?; })?;
let context_json = serde_json::to_string(&self.plugin_context) let serialized_context_json = serde_json::to_string(&self.plugin_context)
.context("Failed to serialize plugin context as json") .context("Failed to serialize plugin context as json")
.and_then(|value| { .and_then(|value| {
PluginSerializedBytes::try_serialize(&VersionedSerializable::new( PluginSerializedBytes::try_serialize(&VersionedSerializable::new(
@ -208,20 +222,27 @@ impl RustPlugins {
)) ))
})?; })?;
serialized = swc_plugin_runner::apply_transform_plugin( let mut transform_plugin_executor =
&p.0, swc_plugin_runner::create_plugin_transform_executor(
&PathBuf::from(&p.0), &PathBuf::from(&p.0),
&swc_plugin_runner::cache::PLUGIN_MODULE_CACHE, &swc_plugin_runner::cache::PLUGIN_MODULE_CACHE,
serialized, &self.source_map,
config_json, )?;
context_json,
should_enable_comments_proxy, serialized_program = transform_plugin_executor
&self.source_map, .transform(
)?; &serialized_program,
&serialized_config_json,
&serialized_context_json,
should_enable_comments_proxy,
)
.with_context(|| {
format!("failed to invoke `{}` as js transform plugin", &p.0)
})?;
} }
} }
serialized.deserialize().map(|v| v.into_inner()) serialized_program.deserialize().map(|v| v.into_inner())
}, },
) )
} }

2
crates/swc_cli/src/commands/plugin.rs
View File

@ -228,7 +228,7 @@ impl VisitMut for TransformVisitor {
/// as returning ptr back to host. /// as returning ptr back to host.
/// ///
/// It is possible to opt out from macro by writing transform fn manually via /// It is possible to opt out from macro by writing transform fn manually via
/// `__plugin_process_impl( /// `__transform_plugin_process_impl(
/// ast_ptr: *const u8, /// ast_ptr: *const u8,
/// ast_ptr_len: i32, /// ast_ptr_len: i32,
/// config_str_ptr: *const u8, /// config_str_ptr: *const u8,

5
crates/swc_plugin_macro/src/lib.rs
View File

@ -19,7 +19,8 @@ pub fn plugin_transform(
fn handle_func(func: ItemFn) -> TokenStream { fn handle_func(func: ItemFn) -> TokenStream {
let ident = func.sig.ident.clone(); let ident = func.sig.ident.clone();
let process_impl_ident = Ident::new("__plugin_process_impl", Span::call_site()); let transform_process_impl_ident =
Ident::new("__transform_plugin_process_impl", Span::call_site());
let ret = quote! { let ret = quote! {
#func #func
@ -59,7 +60,7 @@ fn handle_func(func: ItemFn) -> TokenStream {
// There are some cases error won't be wrapped up however - for example, we expect // There are some cases error won't be wrapped up however - for example, we expect
// serialization of PluginError itself should succeed. // serialization of PluginError itself should succeed.
#[no_mangle] #[no_mangle]
pub fn #process_impl_ident(ast_ptr: *const u8, ast_ptr_len: i32, config_str_ptr: *const u8, config_str_ptr_len: i32, context_str_ptr: *const u8, context_str_ptr_len: i32, should_enable_comments_proxy: i32) -> i32 { pub fn #transform_process_impl_ident(ast_ptr: *const u8, ast_ptr_len: i32, config_str_ptr: *const u8, config_str_ptr_len: i32, context_str_ptr: *const u8, context_str_ptr_len: i32, should_enable_comments_proxy: i32) -> i32 {
// Reconstruct `Program` & config string from serialized program // Reconstruct `Program` & config string from serialized program
// Host (SWC) should allocate memory, copy bytes and pass ptr to plugin. // Host (SWC) should allocate memory, copy bytes and pass ptr to plugin.
let program = unsafe { swc_plugin::deserialize_from_ptr(ast_ptr, ast_ptr_len).map(|v| v.into_inner()) }; let program = unsafe { swc_plugin::deserialize_from_ptr(ast_ptr, ast_ptr_len).map(|v| v.into_inner()) };

4
crates/swc_plugin_proxy/src/comments/host_comments_storage.rs
View File

@ -5,8 +5,8 @@
/// specifically `SingleThreadedComments`. It doesn't support thread-safety /// specifically `SingleThreadedComments`. It doesn't support thread-safety
/// which does not allow to be passed into wasmerEnv (HostEnvironment). A scoped /// which does not allow to be passed into wasmerEnv (HostEnvironment). A scoped
/// tls holds inner comments as global, per each transform execution. Refer /// tls holds inner comments as global, per each transform execution. Refer
/// `swc::RustPlugins::apply_transform_plugin` for the responsibility to manage /// `swc_plugin::transform_execytor::TransformExecutor::transform` for the
/// actual data. /// responsibility to manage actual data.
/// ///
/// Should never attempt to use this other than plugin_runner. /// Should never attempt to use this other than plugin_runner.
// TODO: This storage does not support mutable yet // TODO: This storage does not support mutable yet

26
crates/swc_plugin_runner/benches/invoke.rs
View File

@ -62,24 +62,32 @@ fn bench_transform(b: &mut Bencher, plugin_dir: &Path) {
let program = VersionedSerializable::new(program); let program = VersionedSerializable::new(program);
let program_ser = PluginSerializedBytes::try_serialize(&program).unwrap(); let program_ser = PluginSerializedBytes::try_serialize(&program).unwrap();
let res = swc_plugin_runner::apply_transform_plugin( let mut transform_plugin_executor = swc_plugin_runner::create_plugin_transform_executor(
"test",
&plugin_dir &plugin_dir
.join("target") .join("target")
.join("wasm32-unknown-unknown") .join("wasm32-unknown-unknown")
.join("release") .join("release")
.join("swc_internal_plugin.wasm"), .join("swc_internal_plugin.wasm"),
&cache, &swc_plugin_runner::cache::PLUGIN_MODULE_CACHE,
program_ser,
PluginSerializedBytes::try_serialize(&VersionedSerializable::new(String::from("{}")))
.unwrap(),
PluginSerializedBytes::try_serialize(&VersionedSerializable::new(String::from("{}")))
.unwrap(),
true,
&cm, &cm,
) )
.unwrap(); .unwrap();
let res = transform_plugin_executor
.transform(
&program_ser,
&PluginSerializedBytes::try_serialize(&VersionedSerializable::new(String::from(
"{}",
)))
.unwrap(),
&PluginSerializedBytes::try_serialize(&VersionedSerializable::new(String::from(
"{}",
)))
.unwrap(),
true,
)
.unwrap();
let _ = black_box(res); let _ = black_box(res);
}) })
} }

41
crates/swc_plugin_runner/src/lib.rs
View File

@ -1,8 +1,8 @@
use std::{path::Path, sync::Arc}; use std::{path::Path, sync::Arc};
use anyhow::{Context, Error}; use anyhow::Error;
use once_cell::sync::Lazy; use once_cell::sync::Lazy;
use swc_common::{plugin::PluginSerializedBytes, SourceMap}; use swc_common::SourceMap;
use transform_executor::TransformExecutor; use transform_executor::TransformExecutor;
pub mod cache; pub mod cache;
@ -12,33 +12,18 @@ mod load_plugin;
mod memory_interop; mod memory_interop;
mod transform_executor; mod transform_executor;
// entrypoint fn swc calls to perform its transform via plugin. /**
#[allow(clippy::too_many_arguments)] * Attempt to create a executor to run plugin binaries.
pub fn apply_transform_plugin( * Internally this will try to load binary from given cache which can fail,
plugin_name: &str, * returns error in that case.
*
* Note you CANNOT reuse executor once trasform has been executed: executor
* is stateful.
*/
pub fn create_plugin_transform_executor(
path: &Path, path: &Path,
cache: &Lazy<cache::PluginModuleCache>, cache: &Lazy<cache::PluginModuleCache>,
program: PluginSerializedBytes,
config_json: PluginSerializedBytes,
context_json: PluginSerializedBytes,
should_enable_comments_proxy: bool,
source_map: &Arc<SourceMap>, source_map: &Arc<SourceMap>,
) -> Result<PluginSerializedBytes, Error> { ) -> Result<TransformExecutor, Error> {
(|| -> Result<_, Error> { TransformExecutor::new(path, cache, source_map)
let mut transform_tracker = TransformExecutor::new(path, cache, source_map)?;
let should_enable_comments_proxy = if should_enable_comments_proxy { 1 } else { 0 };
transform_tracker.transform(
&program,
&config_json,
&context_json,
should_enable_comments_proxy,
)
})()
.with_context(|| {
format!(
"failed to invoke `{}` as js transform plugin at {}",
plugin_name,
path.display()
)
})
} }

16
crates/swc_plugin_runner/src/transform_executor.rs
View File

@ -40,7 +40,7 @@ impl TransformExecutor {
exported_plugin_transform: instance exported_plugin_transform: instance
.exports .exports
.get_native_function::<(i32, i32, i32, i32, i32, i32, i32), i32>( .get_native_function::<(i32, i32, i32, i32, i32, i32, i32), i32>(
"__plugin_process_impl", "__transform_plugin_process_impl",
)?, )?,
exported_plugin_free: instance exported_plugin_free: instance
.exports .exports
@ -102,14 +102,26 @@ impl TransformExecutor {
} }
} }
/**
* Check compile-time versions of AST schema between the plugin and
* the host. Returns true if it's compatible, false otherwise.
*
* Host should appropriately handle if plugin is not compatible to the
* current runtime.
*/
pub fn is_transform_schema_compatible(&self) -> bool {
todo!("Not supported yet");
}
#[tracing::instrument(level = "info", skip_all)] #[tracing::instrument(level = "info", skip_all)]
pub fn transform( pub fn transform(
&mut self, &mut self,
program: &PluginSerializedBytes, program: &PluginSerializedBytes,
config: &PluginSerializedBytes, config: &PluginSerializedBytes,
context: &PluginSerializedBytes, context: &PluginSerializedBytes,
should_enable_comments_proxy: i32, should_enable_comments_proxy: bool,
) -> Result<PluginSerializedBytes, Error> { ) -> Result<PluginSerializedBytes, Error> {
let should_enable_comments_proxy = if should_enable_comments_proxy { 1 } else { 0 };
let guest_program_ptr = self.write_bytes_into_guest(program)?; let guest_program_ptr = self.write_bytes_into_guest(program)?;
let config_str_ptr = self.write_bytes_into_guest(config)?; let config_str_ptr = self.write_bytes_into_guest(config)?;
let context_str_ptr = self.write_bytes_into_guest(context)?; let context_str_ptr = self.write_bytes_into_guest(context)?;

97
crates/swc_plugin_runner/tests/integration.rs
View File

@ -96,18 +96,13 @@ fn internal() -> Result<(), Error> {
.expect("Should serializable"); .expect("Should serializable");
let cache: Lazy<PluginModuleCache> = Lazy::new(PluginModuleCache::new); let cache: Lazy<PluginModuleCache> = Lazy::new(PluginModuleCache::new);
let mut plugin_transform_executor =
swc_plugin_runner::create_plugin_transform_executor(&path, &cache, &cm)
.expect("Should load plugin");
let program_bytes = swc_plugin_runner::apply_transform_plugin( let program_bytes = plugin_transform_executor
"internal-test", .transform(&program, &config, &context, false)
&path, .expect("Plugin should apply transform");
&cache,
program,
config,
context,
false,
&cm,
)
.expect("Plugin should apply transform");
let program: Program = program_bytes let program: Program = program_bytes
.deserialize() .deserialize()
@ -153,17 +148,13 @@ fn internal() -> Result<(), Error> {
let cache: Lazy<PluginModuleCache> = Lazy::new(PluginModuleCache::new); let cache: Lazy<PluginModuleCache> = Lazy::new(PluginModuleCache::new);
let _res = HANDLER.set(&handler, || { let _res = HANDLER.set(&handler, || {
swc_plugin_runner::apply_transform_plugin( let mut plugin_transform_executor =
"internal-test", swc_plugin_runner::create_plugin_transform_executor(&path, &cache, &cm)
&path, .expect("Should load plugin");
&cache,
program, plugin_transform_executor
config, .transform(&program, &config, &context, false)
context, .expect("Plugin should apply transform")
false,
&cm,
)
.expect("Plugin should apply transform")
}); });
Ok(()) Ok(())
@ -190,38 +181,44 @@ fn internal() -> Result<(), Error> {
.expect("Should serializable"); .expect("Should serializable");
let cache: Lazy<PluginModuleCache> = Lazy::new(PluginModuleCache::new); let cache: Lazy<PluginModuleCache> = Lazy::new(PluginModuleCache::new);
serialized_program = swc_plugin_runner::apply_transform_plugin( let mut plugin_transform_executor =
"internal-test", swc_plugin_runner::create_plugin_transform_executor(&path, &cache, &cm)
&path, .expect("Should load plugin");
&cache,
serialized_program, serialized_program = plugin_transform_executor
PluginSerializedBytes::try_serialize(&VersionedSerializable::new("{}".to_string())) .transform(
&serialized_program,
&PluginSerializedBytes::try_serialize(&VersionedSerializable::new(
"{}".to_string(),
))
.expect("Should serializable"), .expect("Should serializable"),
PluginSerializedBytes::try_serialize(&VersionedSerializable::new( &PluginSerializedBytes::try_serialize(&VersionedSerializable::new(
"{sourceFileName: 'multiple_plugin_test'}".to_string(), "{sourceFileName: 'multiple_plugin_test'}".to_string(),
)) ))
.expect("Should serializable"), .expect("Should serializable"),
false, false,
&cm, )
) .expect("Plugin should apply transform");
.expect("Plugin should apply transform");
// TODO: we'll need to apply 2 different plugins // TODO: we'll need to apply 2 different plugins
serialized_program = swc_plugin_runner::apply_transform_plugin( let mut plugin_transform_executor =
"internal-test", swc_plugin_runner::create_plugin_transform_executor(&path, &cache, &cm)
&path, .expect("Should load plugin");
&cache,
serialized_program, serialized_program = plugin_transform_executor
PluginSerializedBytes::try_serialize(&VersionedSerializable::new("{}".to_string())) .transform(
&serialized_program,
&PluginSerializedBytes::try_serialize(&VersionedSerializable::new(
"{}".to_string(),
))
.expect("Should serializable"), .expect("Should serializable"),
PluginSerializedBytes::try_serialize(&VersionedSerializable::new( &PluginSerializedBytes::try_serialize(&VersionedSerializable::new(
"{sourceFileName: 'multiple_plugin_test2'}".to_string(), "{sourceFileName: 'multiple_plugin_test2'}".to_string(),
)) ))
.expect("Should serializable"), .expect("Should serializable"),
false, false,
&cm, )
) .expect("Plugin should apply transform");
.expect("Plugin should apply transform");
let program: Program = serialized_program let program: Program = serialized_program
.deserialize() .deserialize()

18
tests/rust-plugins/swc_internal_plugin/Cargo.lock generated
View File

@ -722,7 +722,7 @@ dependencies = [
[[package]] [[package]]
name = "swc_common" name = "swc_common"
version = "0.21.0" version = "0.22.0"
dependencies = [ dependencies = [
"ahash", "ahash",
"anyhow", "anyhow",
@ -750,7 +750,7 @@ dependencies = [
[[package]] [[package]]
name = "swc_ecma_ast" name = "swc_ecma_ast"
version = "0.82.0" version = "0.83.0"
dependencies = [ dependencies = [
"bitflags", "bitflags",
"bytecheck", "bytecheck",
@ -767,7 +767,7 @@ dependencies = [
[[package]] [[package]]
name = "swc_ecma_parser" name = "swc_ecma_parser"
version = "0.109.1" version = "0.110.0"
dependencies = [ dependencies = [
"either", "either",
"enum_kind", "enum_kind",
@ -784,7 +784,7 @@ dependencies = [
[[package]] [[package]]
name = "swc_ecma_utils" name = "swc_ecma_utils"
version = "0.91.0" version = "0.92.0"
dependencies = [ dependencies = [
"indexmap", "indexmap",
"once_cell", "once_cell",
@ -798,7 +798,7 @@ dependencies = [
[[package]] [[package]]
name = "swc_ecma_visit" name = "swc_ecma_visit"
version = "0.68.0" version = "0.69.0"
dependencies = [ dependencies = [
"num-bigint", "num-bigint",
"swc_atoms", "swc_atoms",
@ -810,7 +810,7 @@ dependencies = [
[[package]] [[package]]
name = "swc_ecmascript" name = "swc_ecmascript"
version = "0.176.0" version = "0.178.0"
dependencies = [ dependencies = [
"swc_ecma_ast", "swc_ecma_ast",
"swc_ecma_parser", "swc_ecma_parser",
@ -848,7 +848,7 @@ dependencies = [
[[package]] [[package]]
name = "swc_plugin" name = "swc_plugin"
version = "0.72.0" version = "0.74.0"
dependencies = [ dependencies = [
"swc_atoms", "swc_atoms",
"swc_common", "swc_common",
@ -868,7 +868,7 @@ dependencies = [
[[package]] [[package]]
name = "swc_plugin_proxy" name = "swc_plugin_proxy"
version = "0.7.0" version = "0.8.0"
dependencies = [ dependencies = [
"better_scoped_tls", "better_scoped_tls",
"bytecheck", "bytecheck",
@ -887,7 +887,7 @@ dependencies = [
[[package]] [[package]]
name = "swc_visit_macros" name = "swc_visit_macros"
version = "0.3.2" version = "0.3.5"
dependencies = [ dependencies = [
"Inflector", "Inflector",
"pmutil", "pmutil",

2
tests/rust-plugins/swc_internal_plugin/src/lib.rs
View File

@ -29,7 +29,7 @@ impl VisitMut for ConsoleOutputReplacer {
/// as returning ptr back to host. /// as returning ptr back to host.
/// ///
/// It is possible to opt out from macro by writing transform fn manually via /// It is possible to opt out from macro by writing transform fn manually via
/// `__plugin_process_impl( /// `__transform_plugin_process_impl(
/// ast_ptr: *const u8, /// ast_ptr: *const u8,
/// ast_ptr_len: i32, /// ast_ptr_len: i32,
/// config_str_ptr: *const u8, /// config_str_ptr: *const u8,