From f06c862a9ff2c6510c3b88b4d5bd4807a11b5bfb Mon Sep 17 00:00:00 2001 From: OJ Kwon Date: Mon, 18 Apr 2022 22:25:44 -0700 Subject: [PATCH] feat(plugin): Support `lookup_char_pos` of `SourceMap` (#4364) --- Cargo.lock | 8 +- crates/swc/src/config/mod.rs | 5 +- crates/swc/src/plugin.rs | 5 + crates/swc_common/Cargo.toml | 2 +- crates/swc_common/src/plugin.rs | 30 ++++ crates/swc_ecma_ast/Cargo.toml | 2 +- crates/swc_plugin/src/lib.rs | 7 +- crates/swc_plugin_macro/src/lib.rs | 1 + crates/swc_plugin_proxy/Cargo.toml | 2 +- crates/swc_plugin_proxy/src/comments/mod.rs | 1 - .../src/comments/plugin_comments_proxy.rs | 69 ++-------- crates/swc_plugin_proxy/src/lib.rs | 3 +- .../src/memory_interop/mod.rs | 6 + .../read_returned_result_from_host.rs | 129 ++++++++++++++++++ crates/swc_plugin_proxy/src/source_map/mod.rs | 1 + .../src/source_map/plugin_source_map_proxy.rs | 33 ++++- .../src/imported_fn/comments.rs | 42 +----- .../swc_plugin_runner/src/imported_fn/mod.rs | 20 ++- .../src/imported_fn/source_map.rs | 42 ++++-- crates/swc_plugin_runner/src/lib.rs | 8 +- crates/swc_plugin_runner/src/load_plugin.rs | 4 +- .../swc_plugin_runner/src/memory_interop.rs | 41 +++++- .../src/transform_executor.rs | 11 +- crates/swc_plugin_runner/tests/integration.rs | 4 + .../swc_internal_plugin/Cargo.lock | 70 +++++++--- 25 files changed, 397 insertions(+), 149 deletions(-) create mode 100644 crates/swc_plugin_proxy/src/memory_interop/mod.rs create mode 100644 crates/swc_plugin_proxy/src/memory_interop/read_returned_result_from_host.rs diff --git a/Cargo.lock b/Cargo.lock index 4fc0fe8cc9e..1c522f01071 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2367,9 +2367,9 @@ checksum = "11000e6ba5020e53e7cc26f73b91ae7d5496b4977851479edb66b694c0675c21" [[package]] name = "rkyv" -version = "0.7.36" +version = "0.7.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5230ae2981a885590b0dc84e0b24c0ed23ad24f7adc0eb824b26cafa961f7c36" +checksum = "1f08c8062c1fe1253064043b8fc07bfea1b9702b71b4a86c11ea3588183b12e1" dependencies = [ "bytecheck", "hashbrown 0.12.0", @@ -2381,9 +2381,9 @@ dependencies = [ [[package]] name = "rkyv_derive" -version = "0.7.36" +version = "0.7.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fc752d5925dbcb324522f3a4c93193d17f107b2e11810913aa3ad352fa01480" +checksum = "e289706df51226e84814bf6ba1a9e1013112ae29bc7a9878f73fce360520c403" dependencies = [ "proc-macro2", "quote", diff --git a/crates/swc/src/config/mod.rs b/crates/swc/src/config/mod.rs index 4187c97646e..a3a86d768d8 100644 --- a/crates/swc/src/config/mod.rs +++ b/crates/swc/src/config/mod.rs @@ -467,9 +467,11 @@ impl Options { } let comments = comments.cloned(); + let source_map = cm.clone(); crate::plugin::plugins( Some(plugin_resolver), comments, + source_map, experimental, plugin_context, ) @@ -494,7 +496,8 @@ impl Options { swc_plugin_runner::cache::init_plugin_module_cache_once(); let comments = comments.cloned(); - crate::plugin::plugins(None, comments, experimental, plugin_context) + let source_map = cm.clone(); + crate::plugin::plugins(None, comments, source_map, experimental, plugin_context) }; #[cfg(not(feature = "plugin"))] diff --git a/crates/swc/src/plugin.rs b/crates/swc/src/plugin.rs index 3069a5a9078..bb001bd4331 100644 --- a/crates/swc/src/plugin.rs +++ b/crates/swc/src/plugin.rs @@ -45,6 +45,7 @@ pub struct PluginContext { pub fn plugins( resolver: Option>, comments: Option, + source_map: std::sync::Arc, config: crate::config::JscExperimental, plugin_context: PluginContext, ) -> impl Fold { @@ -52,6 +53,7 @@ pub fn plugins( RustPlugins { resolver, comments, + source_map, plugins: config.plugins, plugin_context, } @@ -67,6 +69,7 @@ struct RustPlugins { resolver: Option>, comments: Option, plugins: Option>, + source_map: std::sync::Arc, plugin_context: PluginContext, } @@ -141,6 +144,7 @@ impl RustPlugins { config_json, context_json, should_enable_comments_proxy, + &self.source_map, )?; drop(transform_span_guard); } @@ -188,6 +192,7 @@ impl RustPlugins { config_json, context_json, should_enable_comments_proxy, + &self.source_map, )?; } } diff --git a/crates/swc_common/Cargo.toml b/crates/swc_common/Cargo.toml index 73457f0a0be..756f5ddc0d3 100644 --- a/crates/swc_common/Cargo.toml +++ b/crates/swc_common/Cargo.toml @@ -41,7 +41,7 @@ from_variant = {version = "0.1.3", path = "../from_variant"} num-bigint = "0.4" once_cell = "1.10.0" parking_lot = {version = "0.12.0", optional = true} -rkyv = {version = "0.7.28", optional = true} +rkyv = { version = "0.7.37", optional = true } rustc-hash = "1.1.0" serde = {version = "1.0.119", features = ["derive"]} siphasher = "0.3.9" diff --git a/crates/swc_common/src/plugin.rs b/crates/swc_common/src/plugin.rs index 1937a2aea27..2b2474748eb 100644 --- a/crates/swc_common/src/plugin.rs +++ b/crates/swc_common/src/plugin.rs @@ -126,4 +126,34 @@ impl Serialized { let serialized = Serialized::new_for_plugin(raw_ptr_bytes, raw_allocated_ptr_len); Serialized::deserialize(&serialized) } + + /// Deserialize `Fallible` struct from raw ptr. This is similar to + /// `deserialize_from_ptr` but for the struct requires bounds to the + /// SharedSerializeRegistry which cannot be Infallible. Internally this does + /// not call deserialize with Infallible deserializer, use + /// SharedDeserializeMap instead. + /// + /// # Safety + /// This is unsafe by construting bytes slice from raw ptr also deserialize + /// it without slice bound check. + pub unsafe fn deserialize_from_ptr_fallible( + raw_allocated_ptr: *const u8, + raw_allocated_ptr_len: i32, + ) -> Result + where + W: rkyv::Archive, + W::Archived: rkyv::Deserialize, + { + // Create serialized bytes slice from ptr + let raw_ptr_bytes = unsafe { + std::slice::from_raw_parts(raw_allocated_ptr, raw_allocated_ptr_len.try_into()?) + }; + + let serialized = Serialized::new_for_plugin(raw_ptr_bytes, raw_allocated_ptr_len); + + unsafe { + rkyv::from_bytes_unchecked(serialized.as_ref()) + .map_err(|err| Error::msg("Failed to deserialize given ptr")) + } + } } diff --git a/crates/swc_ecma_ast/Cargo.toml b/crates/swc_ecma_ast/Cargo.toml index 3f97b1a1909..bb3aa44fc69 100644 --- a/crates/swc_ecma_ast/Cargo.toml +++ b/crates/swc_ecma_ast/Cargo.toml @@ -24,7 +24,7 @@ rkyv-impl = ["rkyv", "swc_common/rkyv-impl"] arbitrary = {version = "1", optional = true, features = ["derive"]} is-macro = "0.2.0" num-bigint = {version = "0.4", features = ["serde"]} -rkyv = {version = "0.7.28", optional = true} +rkyv = { version = "0.7.37", optional = true } serde = {version = "1.0.133", features = ["derive"]} string_enum = {version = "0.3.1", path = "../string_enum"} swc_atoms = {version = "0.2", path = "../swc_atoms"} diff --git a/crates/swc_plugin/src/lib.rs b/crates/swc_plugin/src/lib.rs index 7881e32905e..3d4844e14d2 100644 --- a/crates/swc_plugin/src/lib.rs +++ b/crates/swc_plugin/src/lib.rs @@ -13,6 +13,7 @@ pub mod comments { pub mod source_map { pub use swc_common::source_map::{CharPos, Loc, MultiByteChar, NonNarrowChar, SourceFile}; + pub use swc_plugin_proxy::PluginSourceMapProxy; } pub mod utils { @@ -46,7 +47,7 @@ pub mod environment { // We don't set target cfg as it'll block macro expansions // in ide (i.e rust-analyzer) or non-wasm target `cargo check` pub use swc_plugin_macro::plugin_transform; -use swc_plugin_proxy::PluginCommentsProxy; +use swc_plugin_proxy::{PluginCommentsProxy, PluginSourceMapProxy}; #[cfg(target_arch = "wasm32")] mod allocation; #[cfg(target_arch = "wasm32")] @@ -64,6 +65,10 @@ pub struct TransformPluginProgramMetadata { /// This is a proxy to the actual data lives in the host. Only when plugin /// attempts to read these it'll ask to the host to get values. pub comments: Option, + /// Proxy to the sourceMap for the Program passed into plugin. + /// This is a proxy to the actual data lives in the host. Only when plugin + /// attempts to read these it'll ask to the host to get values. + pub source_map: PluginSourceMapProxy, /// Stringified JSON value for given plugin's configuration. /// This is readonly. Changing value in plugin doesn't affect host's /// behavior. diff --git a/crates/swc_plugin_macro/src/lib.rs b/crates/swc_plugin_macro/src/lib.rs index 1cd2e3470c9..5d02c83587e 100644 --- a/crates/swc_plugin_macro/src/lib.rs +++ b/crates/swc_plugin_macro/src/lib.rs @@ -103,6 +103,7 @@ fn handle_func(func: ItemFn) -> TokenStream { let plugin_comments_proxy = if should_enable_comments_proxy == 1 { Some(swc_plugin::comments::PluginCommentsProxy) } else { None }; let mut metadata = swc_plugin::TransformPluginProgramMetadata { comments: plugin_comments_proxy, + source_map: swc_plugin::source_map::PluginSourceMapProxy, plugin_config: config, transform_context: context }; diff --git a/crates/swc_plugin_proxy/Cargo.toml b/crates/swc_plugin_proxy/Cargo.toml index 59ffe19f9e2..918be41f355 100644 --- a/crates/swc_plugin_proxy/Cargo.toml +++ b/crates/swc_plugin_proxy/Cargo.toml @@ -18,7 +18,7 @@ plugin-mode = [] [dependencies] better_scoped_tls = { version = "0.1.0", path = "../better_scoped_tls" } -rkyv = "0.7.36" +rkyv = "0.7.37" swc_common = { version = "0.17.20", path = "../swc_common", features = [ "plugin-base", ] } diff --git a/crates/swc_plugin_proxy/src/comments/mod.rs b/crates/swc_plugin_proxy/src/comments/mod.rs index 31e456cfce1..db74873241c 100644 --- a/crates/swc_plugin_proxy/src/comments/mod.rs +++ b/crates/swc_plugin_proxy/src/comments/mod.rs @@ -3,6 +3,5 @@ mod plugin_comments_proxy; #[cfg(feature = "plugin-rt")] pub use host_comments_storage::{HostCommentsStorage, COMMENTS}; -pub use plugin_comments_proxy::CommentsVecPtr; #[cfg(feature = "plugin-mode")] pub use plugin_comments_proxy::PluginCommentsProxy; diff --git a/crates/swc_plugin_proxy/src/comments/plugin_comments_proxy.rs b/crates/swc_plugin_proxy/src/comments/plugin_comments_proxy.rs index dff3bdfefd2..4c7fe68fea9 100644 --- a/crates/swc_plugin_proxy/src/comments/plugin_comments_proxy.rs +++ b/crates/swc_plugin_proxy/src/comments/plugin_comments_proxy.rs @@ -1,11 +1,13 @@ #[cfg(feature = "plugin-mode")] -use swc_common::plugin::Serialized; -#[cfg(feature = "plugin-mode")] use swc_common::{ comments::{Comment, Comments}, BytePos, }; +#[cfg(feature = "plugin-mode")] +#[cfg_attr(not(target_arch = "wasm32"), allow(unused))] +use crate::memory_interop::read_returned_result_from_host; + #[cfg(target_arch = "wasm32")] extern "C" { fn __copy_comment_to_host_env(bytes_ptr: i32, bytes_ptr_len: i32); @@ -24,10 +26,6 @@ extern "C" { fn __add_pure_comment_proxy(byte_pos: u32); } -/// A struct to exchange allocated Vec between memory spaces. -#[derive(rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] -pub struct CommentsVecPtr(pub i32, pub i32); - /// A struct implements `swc_common::comments::Comments` for the plugin. /// This is a proxy to the host's comments reference while plugin transform /// runs, does not contain actual data but lazily requests to the host for each @@ -52,7 +50,8 @@ impl PluginCommentsProxy { { #[cfg(target_arch = "wasm32")] { - let serialized = Serialized::serialize(value).expect("Should able to serialize value"); + let serialized = swc_common::plugin::Serialized::serialize(value) + .expect("Should able to serialize value"); let serialized_comment_ptr_ref = serialized.as_ref(); unsafe { // We need to copy PluginCommentProxy's param for add_leading (Comment, or @@ -69,54 +68,6 @@ impl PluginCommentsProxy { } } } - - /// Utility wrapper to call host fn which returns a Comment or Vec. - #[cfg_attr(not(target_arch = "wasm32"), allow(unused))] - fn read_returned_comments_from_host(&self, f: F) -> Option - where - F: FnOnce(i32) -> i32, - R: rkyv::Archive, - R::Archived: rkyv::Deserialize, - { - // Allocate CommentsVecPtr to get return value from the host - let comments_vec_ptr = CommentsVecPtr(0, 0); - let serialized_comments_vec_ptr = Serialized::serialize(&comments_vec_ptr) - .expect("Should able to serialize CommentsVecPtr"); - let serialized_comments_vec_ptr_ref = serialized_comments_vec_ptr.as_ref(); - let serialized_comments_vec_ptr_raw_ptr = serialized_comments_vec_ptr_ref.as_ptr(); - let serialized_comments_vec_ptr_raw_len = serialized_comments_vec_ptr_ref.len(); - - #[cfg(target_arch = "wasm32")] - { - let ret = f(serialized_comments_vec_ptr_raw_ptr as _); - - // Host fn call completes: by contract in comments_proxy, if return value is 0 - // we know there's no value to read. Otherwise, we know host filled in - // CommentsVecPtr to the pointer for the actual value for the - // results. - if ret == 0 { - return None; - } - } - - // Now reconstruct CommentsVecPtr to reveal ptr to the allocated - // Vec - let comments_vec_ptr: CommentsVecPtr = unsafe { - Serialized::deserialize_from_ptr( - serialized_comments_vec_ptr_raw_ptr, - serialized_comments_vec_ptr_raw_len - .try_into() - .expect("Should able to convert ptr length"), - ) - .expect("Should able to deserialize CommentsVecPtr") - }; - - // Using CommentsVecPtr's value, reconstruct actual Comment, or Vec - Some(unsafe { - Serialized::deserialize_from_ptr(comments_vec_ptr.0 as _, comments_vec_ptr.1) - .expect("Returned comments should be serializable") - }) - } } #[cfg(feature = "plugin-mode")] @@ -160,7 +111,7 @@ impl Comments for PluginCommentsProxy { fn take_leading(&self, pos: BytePos) -> Option> { #[cfg(target_arch = "wasm32")] - return self.read_returned_comments_from_host(|serialized_ptr| unsafe { + return read_returned_result_from_host(|serialized_ptr| unsafe { __take_leading_comments_proxy(pos.0, serialized_ptr) }); @@ -170,7 +121,7 @@ impl Comments for PluginCommentsProxy { fn get_leading(&self, pos: BytePos) -> Option> { #[cfg(target_arch = "wasm32")] - return self.read_returned_comments_from_host(|serialized_ptr| unsafe { + return read_returned_result_from_host(|serialized_ptr| unsafe { __get_leading_comments_proxy(pos.0, serialized_ptr) }); @@ -216,7 +167,7 @@ impl Comments for PluginCommentsProxy { fn take_trailing(&self, pos: BytePos) -> Option> { #[cfg(target_arch = "wasm32")] - return self.read_returned_comments_from_host(|serialized_ptr| unsafe { + return read_returned_result_from_host(|serialized_ptr| unsafe { __take_trailing_comments_proxy(pos.0, serialized_ptr) }); @@ -226,7 +177,7 @@ impl Comments for PluginCommentsProxy { fn get_trailing(&self, pos: BytePos) -> Option> { #[cfg(target_arch = "wasm32")] - return self.read_returned_comments_from_host(|serialized_ptr| unsafe { + return read_returned_result_from_host(|serialized_ptr| unsafe { __get_trailing_comments_proxy(pos.0, serialized_ptr) }); diff --git a/crates/swc_plugin_proxy/src/lib.rs b/crates/swc_plugin_proxy/src/lib.rs index d36c7f3c7e5..ca101a430c5 100644 --- a/crates/swc_plugin_proxy/src/lib.rs +++ b/crates/swc_plugin_proxy/src/lib.rs @@ -1,9 +1,10 @@ mod comments; +mod memory_interop; mod source_map; -pub use comments::CommentsVecPtr; #[cfg(feature = "plugin-mode")] pub use comments::PluginCommentsProxy; #[cfg(feature = "plugin-rt")] pub use comments::{HostCommentsStorage, COMMENTS}; +pub use memory_interop::AllocatedBytesPtr; #[cfg(feature = "plugin-mode")] pub use source_map::PluginSourceMapProxy; diff --git a/crates/swc_plugin_proxy/src/memory_interop/mod.rs b/crates/swc_plugin_proxy/src/memory_interop/mod.rs new file mode 100644 index 00000000000..ad7e5a48b7e --- /dev/null +++ b/crates/swc_plugin_proxy/src/memory_interop/mod.rs @@ -0,0 +1,6 @@ +mod read_returned_result_from_host; +pub use read_returned_result_from_host::AllocatedBytesPtr; +#[cfg_attr(not(target_arch = "wasm32"), allow(unused))] +pub(crate) use read_returned_result_from_host::{ + read_returned_result_from_host, read_returned_result_from_host_fallible, +}; diff --git a/crates/swc_plugin_proxy/src/memory_interop/read_returned_result_from_host.rs b/crates/swc_plugin_proxy/src/memory_interop/read_returned_result_from_host.rs new file mode 100644 index 00000000000..c3535a48806 --- /dev/null +++ b/crates/swc_plugin_proxy/src/memory_interop/read_returned_result_from_host.rs @@ -0,0 +1,129 @@ +#[cfg_attr(not(target_arch = "wasm32"), allow(unused))] +use swc_common::plugin::Serialized; + +/// A struct to exchange allocated data between memory spaces. +#[derive(rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] +pub struct AllocatedBytesPtr(pub i32, pub i32); + +/// Performs an interop while calling host fn to get non-determined size return +/// values from the host. This is based on the contract between host's imported +/// fn, by imported fn allocated memory for the guest space then hand over its +/// ptr and length via a struct. Refer plugin_runner/imported_fn/mod.rs for the +/// detail. +/// +/// Returns a struct AllocatedBytesPtr to the ptr for actual return value if +/// host fn allocated return value, None otherwise. +#[cfg_attr(not(target_arch = "wasm32"), allow(unused))] +fn read_returned_result_from_host_inner(f: F) -> Option +where + F: FnOnce(i32) -> i32, +{ + // Allocate AllocatedBytesPtr to get return value from the host + let allocated_bytes_ptr = AllocatedBytesPtr(0, 0); + let serialized_allocated_bytes_ptr = Serialized::serialize(&allocated_bytes_ptr) + .expect("Should able to serialize AllocatedBytesPtr"); + let serialized_allocated_bytes_ptr_ref = serialized_allocated_bytes_ptr.as_ref(); + let serialized_allocated_bytes_raw_ptr = serialized_allocated_bytes_ptr_ref.as_ptr(); + + #[cfg(target_arch = "wasm32")] + { + let ret = f(serialized_allocated_bytes_raw_ptr as _); + + // Host fn call completes: by contract in host proxy, if return value is 0 + // we know there's no value to read. Otherwise, we know host filled in + // AllocatedBytesPtr to the pointer for the actual value for the + // results. + if ret == 0 { + return None; + } + } + + // Return reconstructted AllocatedBytesPtr to reveal ptr to the allocated bytes + Some(unsafe { + Serialized::deserialize_from_ptr( + serialized_allocated_bytes_raw_ptr, + serialized_allocated_bytes_ptr_ref + .len() + .try_into() + .expect("Should able to convert ptr length"), + ) + .expect("Should able to deserialize AllocatedBytesPtr") + }) +} + +/// Performs deserialization to the actual return value type from returned ptr. +/// +/// This fn is for the Infallible types works for most of the cases. +#[cfg_attr(not(target_arch = "wasm32"), allow(unused))] +pub fn read_returned_result_from_host(f: F) -> Option +where + F: FnOnce(i32) -> i32, + R: rkyv::Archive, + R::Archived: rkyv::Deserialize, +{ + let allocated_returned_value_ptr = read_returned_result_from_host_inner(f); + + // Using AllocatedBytesPtr's value, reconstruct actual return value + allocated_returned_value_ptr.map(|allocated_returned_value_ptr| unsafe { + Serialized::deserialize_from_ptr( + allocated_returned_value_ptr.0 as _, + allocated_returned_value_ptr.1, + ) + .expect("Returned value should be serializable") + }) +} + +/// Performs deserialization to the actual return value type from returned ptr. +/// +/// This behaves same as read_returned_result_from_host, the only difference is +/// this is for the `Fallible` struct to deserialize. If a struct contains +/// shared pointers like Arc, Rc rkyv requires trait bounds to the +/// SharedSerializeRegistry which cannot be infallible. +#[cfg_attr(not(target_arch = "wasm32"), allow(unused))] +pub fn read_returned_result_from_host_fallible(f: F) -> Option +where + F: FnOnce(i32) -> i32, + R: rkyv::Archive, + R::Archived: rkyv::Deserialize, +{ + // Allocate AllocatedBytesPtr to get return value from the host + let allocated_bytes_ptr = AllocatedBytesPtr(0, 0); + let serialized_allocated_bytes_ptr = Serialized::serialize(&allocated_bytes_ptr) + .expect("Should able to serialize AllocatedBytesPtr"); + let serialized_allocated_bytes_ptr_ref = serialized_allocated_bytes_ptr.as_ref(); + let serialized_allocated_bytes_raw_ptr = serialized_allocated_bytes_ptr_ref.as_ptr(); + + #[cfg(target_arch = "wasm32")] + { + let ret = f(serialized_allocated_bytes_raw_ptr as _); + + // Host fn call completes: by contract in host proxy, if return value is 0 + // we know there's no value to read. Otherwise, we know host filled in + // AllocatedBytesPtr to the pointer for the actual value for the + // results. + if ret == 0 { + return None; + } + } + + // Now reconstruct AllocatedBytesPtr to reveal ptr to the allocated bytes + let allocated_returned_value_ptr: AllocatedBytesPtr = unsafe { + Serialized::deserialize_from_ptr( + serialized_allocated_bytes_raw_ptr, + serialized_allocated_bytes_ptr_ref + .len() + .try_into() + .expect("Should able to convert ptr length"), + ) + .expect("Should able to deserialize AllocatedBytesPtr") + }; + + // Using AllocatedBytesPtr's value, reconstruct actual return value + Some(unsafe { + Serialized::deserialize_from_ptr_fallible( + allocated_returned_value_ptr.0 as _, + allocated_returned_value_ptr.1, + ) + .expect("Returned value should be serializable") + }) +} diff --git a/crates/swc_plugin_proxy/src/source_map/mod.rs b/crates/swc_plugin_proxy/src/source_map/mod.rs index e0ff25c9906..82de2339674 100644 --- a/crates/swc_plugin_proxy/src/source_map/mod.rs +++ b/crates/swc_plugin_proxy/src/source_map/mod.rs @@ -1,2 +1,3 @@ mod plugin_source_map_proxy; +#[cfg(feature = "plugin-mode")] pub use plugin_source_map_proxy::PluginSourceMapProxy; diff --git a/crates/swc_plugin_proxy/src/source_map/plugin_source_map_proxy.rs b/crates/swc_plugin_proxy/src/source_map/plugin_source_map_proxy.rs index 7f34fd010f5..f560ed14eb1 100644 --- a/crates/swc_plugin_proxy/src/source_map/plugin_source_map_proxy.rs +++ b/crates/swc_plugin_proxy/src/source_map/plugin_source_map_proxy.rs @@ -1 +1,32 @@ -pub struct PluginSourceMapProxy {} +#[cfg(feature = "plugin-mode")] +use swc_common::{BytePos, Loc}; + +#[cfg(feature = "plugin-mode")] +#[cfg_attr(not(target_arch = "wasm32"), allow(unused))] +use crate::memory_interop::read_returned_result_from_host_fallible; + +#[cfg(target_arch = "wasm32")] +extern "C" { + fn __lookup_char_pos_source_map_proxy(byte_pos: u32, allocated_ret_ptr: i32) -> i32; +} + +#[cfg(feature = "plugin-mode")] +#[derive(Debug, Copy, Clone)] +pub struct PluginSourceMapProxy; + +/// Subset of SourceMap interface supported in plugin. +/// Unlike `Comments`, this does not fully implement `SourceMap`. +#[cfg(feature = "plugin-mode")] +impl PluginSourceMapProxy { + #[cfg_attr(not(target_arch = "wasm32"), allow(unused))] + pub fn lookup_char_pos(&self, pos: BytePos) -> Loc { + #[cfg(target_arch = "wasm32")] + return read_returned_result_from_host_fallible(|serialized_ptr| unsafe { + __lookup_char_pos_source_map_proxy(pos.0, serialized_ptr) + }) + .expect("Host should return Loc"); + + #[cfg(not(target_arch = "wasm32"))] + unimplemented!("Sourcemap proxy cannot be called in this context") + } +} diff --git a/crates/swc_plugin_runner/src/imported_fn/comments.rs b/crates/swc_plugin_runner/src/imported_fn/comments.rs index bc136998f29..5de3b9222cd 100644 --- a/crates/swc_plugin_runner/src/imported_fn/comments.rs +++ b/crates/swc_plugin_runner/src/imported_fn/comments.rs @@ -6,10 +6,10 @@ use swc_common::{ plugin::Serialized, BytePos, }; -use swc_plugin_proxy::{CommentsVecPtr, COMMENTS}; +use swc_plugin_proxy::COMMENTS; use wasmer::{LazyInit, Memory, NativeFunc}; -use crate::memory_interop::{copy_bytes_into_host, write_into_memory_view}; +use crate::memory_interop::{allocate_return_values_into_guest, copy_bytes_into_host}; /// External environment state for imported (declared in host, injected into /// guest) fn for comments proxy. @@ -99,44 +99,6 @@ where default } -/// Set `return` value to pass into guest from functions returning values with -/// non-deterministic size like `Vec`. Guest pre-allocates a struct to -/// contain ptr to the value, host in here allocates guest memory for the actual -/// value then returns its ptr with length to the preallocated struct. -fn allocate_return_values_into_guest( - memory: &Memory, - alloc_guest_memory: &NativeFunc, - allocated_ret_ptr: i32, - serialized_bytes: &Serialized, -) { - let serialized_bytes_len = serialized_bytes.as_ref().len(); - - let (allocated_ptr, allocated_ptr_len) = - write_into_memory_view(memory, serialized_bytes, |_| { - // In most cases our host-plugin trampoline works in a way that - // plugin pre-allocates - // memory before calling host imported fn. But in case of - // comments return value is Vec which - // guest cannot predetermine size to allocate, instead - // let host allocate by calling guest's alloc via attached - // hostenvironment. - alloc_guest_memory - .call( - serialized_bytes_len - .try_into() - .expect("Should be able to convert size"), - ) - .expect("Should able to allocate memory in the plugin") - }); - - // Retuning (allocated_ptr, len) into caller (plugin) - let comment_ptr_serialized = - Serialized::serialize(&CommentsVecPtr(allocated_ptr, allocated_ptr_len)) - .expect("Should be serializable"); - - write_into_memory_view(memory, &comment_ptr_serialized, |_| allocated_ret_ptr); -} - /// Common logics for add_*_comment/comments. fn add_comments_inner(env: &CommentHostEnvironment, byte_pos: u32, f: F) where diff --git a/crates/swc_plugin_runner/src/imported_fn/mod.rs b/crates/swc_plugin_runner/src/imported_fn/mod.rs index 0089840aa94..7f85e0e87a2 100644 --- a/crates/swc_plugin_runner/src/imported_fn/mod.rs +++ b/crates/swc_plugin_runner/src/imported_fn/mod.rs @@ -14,7 +14,7 @@ *│ │ │get_leading_comments_proxy()│◀┼───┼────┼──┼─┤get_leading() │ │ │ *│ │ │ │ │ │ │ │ │ │ │ │ *│ │ │ │ │ │ │ │ │ ┌──────────────────────────┐ │ │ │ -*│ │ │ │─┼───┼──┬─┼──┼─┼─▶CommentsVecPtr(ptr, len) │ │ │ │ +*│ │ │ │─┼───┼──┬─┼──┼─┼─▶AllocatedBytesPtr(p,len) │ │ │ │ *│ │ └────────────────────────────┘ │ │ │ │ │ │ │ │ │ │ │ *│ │ │ │ │ │ │ │ └─────────────┬────────────┘ │ │ │ *│ │ │ │ │ │ │ │ │ │ │ │ @@ -37,13 +37,14 @@ * read, deserialize memory host wrote. * - In case of `get_leading`, returned value is non-deterministic vec * (`Vec`) guest cannot preallocate with specific length. Instead, -* guest passes a fixed size struct (CommentsVecPtr), once host allocates +* guest passes a fixed size struct (AllocatedBytesPtr), once host allocates * actual vec into guest it'll write pointer to the vec into the struct. */ use std::sync::Arc; use parking_lot::Mutex; +use swc_common::SourceMap; use wasmer::{imports, Function, ImportObject, Module}; use crate::{ @@ -71,11 +72,14 @@ mod span; use handler::*; use hygiene::*; +use self::source_map::{lookup_char_pos_proxy, SourceMapHostEnvironment}; + /// Create an ImportObject includes functions to be imported from host to the /// plugins. pub(crate) fn build_import_object( module: &Module, transform_result: &Arc>>, + source_map: Arc, ) -> ImportObject { let wasmer_store = module.store(); @@ -199,6 +203,16 @@ pub(crate) fn build_import_object( let add_pure_comment_fn_decl = Function::new_native(wasmer_store, add_pure_comment_proxy); + // source_map + let source_map_buffer = Arc::new(Mutex::new(vec![])); + let source_map = Arc::new(Mutex::new(source_map)); + + let lookup_char_pos_source_map_fn_decl = Function::new_native_with_env( + wasmer_store, + SourceMapHostEnvironment::new(&source_map, &source_map_buffer), + lookup_char_pos_proxy, + ); + imports! { "env" => { // transform @@ -232,6 +246,8 @@ pub(crate) fn build_import_object( "__take_trailing_comments_proxy" => take_trailing_comments_fn_decl, "__get_trailing_comments_proxy" => get_trailing_comments_fn_decl, "__add_pure_comment_proxy" => add_pure_comment_fn_decl, + // source_map + "__lookup_char_pos_source_map_proxy" =>lookup_char_pos_source_map_fn_decl, } } } diff --git a/crates/swc_plugin_runner/src/imported_fn/source_map.rs b/crates/swc_plugin_runner/src/imported_fn/source_map.rs index 59cc647bfdc..03a2e0ce977 100644 --- a/crates/swc_plugin_runner/src/imported_fn/source_map.rs +++ b/crates/swc_plugin_runner/src/imported_fn/source_map.rs @@ -2,9 +2,9 @@ use std::sync::Arc; use parking_lot::Mutex; use swc_common::{plugin::Serialized, BytePos, SourceMap}; -use wasmer::{LazyInit, Memory}; +use wasmer::{LazyInit, Memory, NativeFunc}; -use crate::memory_interop::write_into_memory_view; +use crate::memory_interop::allocate_return_values_into_guest; /// External environment state for imported (declared in host, injected into /// guest) fn for source map proxy. @@ -12,24 +12,50 @@ use crate::memory_interop::write_into_memory_view; pub struct SourceMapHostEnvironment { #[wasmer(export)] pub memory: wasmer::LazyInit, + /// Attached imported fn `__alloc` to the hostenvironment to allow any other + /// imported fn can allocate guest's memory space from host runtime. + #[wasmer(export(name = "__alloc"))] + pub alloc_guest_memory: LazyInit>, pub source_map: Arc>>, + /// A buffer to non-determined size of return value from the host. + pub mutable_source_map_buffer: Arc>>, } impl SourceMapHostEnvironment { - pub fn new(source_map: &Arc>>) -> SourceMapHostEnvironment { + pub fn new( + source_map: &Arc>>, + mutable_source_map_buffer: &Arc>>, + ) -> SourceMapHostEnvironment { SourceMapHostEnvironment { memory: LazyInit::default(), + alloc_guest_memory: LazyInit::default(), source_map: source_map.clone(), + mutable_source_map_buffer: mutable_source_map_buffer.clone(), } } } -pub fn lookup_char_pos_proxy(env: &SourceMapHostEnvironment, byte_pos: u32, allocated_ptr: i32) { - let ret = (env.source_map.lock()).lookup_char_pos(BytePos(byte_pos)); - +pub fn lookup_char_pos_proxy( + env: &SourceMapHostEnvironment, + byte_pos: u32, + allocated_ret_ptr: i32, +) -> i32 { if let Some(memory) = env.memory_ref() { - let serialized_bytes = Serialized::serialize(&ret).expect("Should be serializable"); + let ret = (env.source_map.lock()).lookup_char_pos(BytePos(byte_pos)); + let serialized_loc_bytes = Serialized::serialize(&ret).expect("Should be serializable"); - write_into_memory_view(memory, &serialized_bytes, |_| allocated_ptr); + if let Some(alloc_guest_memory) = env.alloc_guest_memory_ref() { + allocate_return_values_into_guest( + memory, + alloc_guest_memory, + allocated_ret_ptr, + &serialized_loc_bytes, + ); + 1 + } else { + 0 + } + } else { + 0 } } diff --git a/crates/swc_plugin_runner/src/lib.rs b/crates/swc_plugin_runner/src/lib.rs index 44915b5dedc..8093282d5ea 100644 --- a/crates/swc_plugin_runner/src/lib.rs +++ b/crates/swc_plugin_runner/src/lib.rs @@ -1,8 +1,8 @@ -use std::path::Path; +use std::{path::Path, sync::Arc}; use anyhow::{Context, Error}; use once_cell::sync::Lazy; -use swc_common::plugin::Serialized; +use swc_common::{plugin::Serialized, SourceMap}; use transform_executor::TransformExecutor; pub mod cache; @@ -13,6 +13,7 @@ mod memory_interop; mod transform_executor; // entrypoint fn swc calls to perform its transform via plugin. +#[allow(clippy::too_many_arguments)] pub fn apply_transform_plugin( plugin_name: &str, path: &Path, @@ -21,9 +22,10 @@ pub fn apply_transform_plugin( config_json: Serialized, context_json: Serialized, should_enable_comments_proxy: bool, + source_map: &Arc, ) -> Result { (|| -> Result<_, Error> { - let mut transform_tracker = TransformExecutor::new(path, cache)?; + let mut transform_tracker = TransformExecutor::new(path, cache, source_map)?; let should_enable_comments_proxy = if should_enable_comments_proxy { 1 } else { 0 }; transform_tracker.transform( &program, diff --git a/crates/swc_plugin_runner/src/load_plugin.rs b/crates/swc_plugin_runner/src/load_plugin.rs index b7b3cedd104..d1b7ace7940 100644 --- a/crates/swc_plugin_runner/src/load_plugin.rs +++ b/crates/swc_plugin_runner/src/load_plugin.rs @@ -2,6 +2,7 @@ use std::{env, sync::Arc}; use anyhow::{Context, Error}; use parking_lot::Mutex; +use swc_common::SourceMap; use wasmer::{ChainableNamedResolver, Instance}; use wasmer_wasi::{is_wasi_module, WasiState}; @@ -11,13 +12,14 @@ use crate::imported_fn::build_import_object; pub fn load_plugin( plugin_path: &std::path::Path, cache: &once_cell::sync::Lazy, + source_map: &Arc, ) -> Result<(Instance, Arc>>), Error> { let module = cache.load_module(plugin_path); return match module { Ok(module) => { let transform_result: Arc>> = Arc::new(Mutex::new(vec![])); - let import_object = build_import_object(&module, &transform_result); + let import_object = build_import_object(&module, &transform_result, source_map.clone()); // Plugin binary can be either wasm32-wasi or wasm32-unknown-unknown. // Wasi specific env need to be initialized if given module targets wasm32-wasi. diff --git a/crates/swc_plugin_runner/src/memory_interop.rs b/crates/swc_plugin_runner/src/memory_interop.rs index ec5e34f322e..49b5a19f548 100644 --- a/crates/swc_plugin_runner/src/memory_interop.rs +++ b/crates/swc_plugin_runner/src/memory_interop.rs @@ -1,5 +1,6 @@ use swc_common::plugin::Serialized; -use wasmer::{Array, Memory, WasmPtr}; +use swc_plugin_proxy::AllocatedBytesPtr; +use wasmer::{Array, Memory, NativeFunc, WasmPtr}; #[tracing::instrument(level = "info", skip_all)] pub fn copy_bytes_into_host(memory: &Memory, bytes_ptr: i32, bytes_ptr_len: i32) -> Vec { @@ -61,3 +62,41 @@ where .expect("Should be able to convert to i32"), ) } + +/// Set `return` value to pass into guest from functions returning values with +/// non-deterministic size like `Vec`. Guest pre-allocates a struct to +/// contain ptr to the value, host in here allocates guest memory for the actual +/// value then returns its ptr with length to the preallocated struct. +pub fn allocate_return_values_into_guest( + memory: &Memory, + alloc_guest_memory: &NativeFunc, + allocated_ret_ptr: i32, + serialized_bytes: &Serialized, +) { + let serialized_bytes_len = serialized_bytes.as_ref().len(); + + let (allocated_ptr, allocated_ptr_len) = + write_into_memory_view(memory, serialized_bytes, |_| { + // In most cases our host-plugin trampoline works in a way that + // plugin pre-allocates + // memory before calling host imported fn. But in case of + // comments return value is Vec which + // guest cannot predetermine size to allocate, instead + // let host allocate by calling guest's alloc via attached + // hostenvironment. + alloc_guest_memory + .call( + serialized_bytes_len + .try_into() + .expect("Should be able to convert size"), + ) + .expect("Should able to allocate memory in the plugin") + }); + + // Retuning (allocated_ptr, len) into caller (plugin) + let comment_ptr_serialized = + Serialized::serialize(&AllocatedBytesPtr(allocated_ptr, allocated_ptr_len)) + .expect("Should be serializable"); + + write_into_memory_view(memory, &comment_ptr_serialized, |_| allocated_ret_ptr); +} diff --git a/crates/swc_plugin_runner/src/transform_executor.rs b/crates/swc_plugin_runner/src/transform_executor.rs index 6cbbd56751b..632eeadc8e9 100644 --- a/crates/swc_plugin_runner/src/transform_executor.rs +++ b/crates/swc_plugin_runner/src/transform_executor.rs @@ -2,7 +2,10 @@ use std::sync::Arc; use anyhow::{anyhow, Error}; use parking_lot::Mutex; -use swc_common::plugin::{PluginError, Serialized}; +use swc_common::{ + plugin::{PluginError, Serialized}, + SourceMap, +}; use wasmer::Instance; use crate::memory_interop::write_into_memory_view; @@ -24,12 +27,14 @@ pub struct TransformExecutor { } impl TransformExecutor { - #[tracing::instrument(level = "info", skip(cache))] + #[tracing::instrument(level = "info", skip(cache, source_map))] pub fn new( path: &std::path::Path, cache: &once_cell::sync::Lazy, + source_map: &Arc, ) -> Result { - let (instance, transform_result) = crate::load_plugin::load_plugin(path, cache)?; + let (instance, transform_result) = + crate::load_plugin::load_plugin(path, cache, source_map)?; let tracker = TransformExecutor { exported_plugin_transform: instance diff --git a/crates/swc_plugin_runner/tests/integration.rs b/crates/swc_plugin_runner/tests/integration.rs index d4c8968412b..2d25406c192 100644 --- a/crates/swc_plugin_runner/tests/integration.rs +++ b/crates/swc_plugin_runner/tests/integration.rs @@ -95,6 +95,7 @@ fn internal() -> Result<(), Error> { config, context, false, + &cm, ) .expect("Plugin should apply transform"); @@ -144,6 +145,7 @@ fn internal() -> Result<(), Error> { config, context, false, + &cm, ) .expect("Plugin should apply transform") }); @@ -179,6 +181,7 @@ fn internal() -> Result<(), Error> { Serialized::serialize(&"{sourceFileName: 'multiple_plugin_test'}".to_string()) .expect("Should serializable"), false, + &cm, ) .expect("Plugin should apply transform"); @@ -192,6 +195,7 @@ fn internal() -> Result<(), Error> { Serialized::serialize(&"{sourceFileName: 'multiple_plugin_test2'}".to_string()) .expect("Should serializable"), false, + &cm, ) .expect("Plugin should apply transform"); diff --git a/tests/rust-plugins/swc_internal_plugin/Cargo.lock b/tests/rust-plugins/swc_internal_plugin/Cargo.lock index a8a590fcbc7..1e9fe919e8b 100644 --- a/tests/rust-plugins/swc_internal_plugin/Cargo.lock +++ b/tests/rust-plugins/swc_internal_plugin/Cargo.lock @@ -183,6 +183,12 @@ dependencies = [ "wasi", ] +[[package]] +name = "hashbrown" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" + [[package]] name = "hashbrown" version = "0.12.0" @@ -209,6 +215,16 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "indexmap" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f647032dfaa1f8b6dc29bd3edb7bbef4861b8b8007ebb118d6db284fd59f6ee" +dependencies = [ + "autocfg", + "hashbrown 0.11.2", +] + [[package]] name = "is-macro" version = "0.2.0" @@ -480,12 +496,12 @@ dependencies = [ [[package]] name = "rkyv" -version = "0.7.36" +version = "0.7.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5230ae2981a885590b0dc84e0b24c0ed23ad24f7adc0eb824b26cafa961f7c36" +checksum = "1f08c8062c1fe1253064043b8fc07bfea1b9702b71b4a86c11ea3588183b12e1" dependencies = [ "bytecheck", - "hashbrown", + "hashbrown 0.12.0", "ptr_meta", "rend", "rkyv_derive", @@ -494,9 +510,9 @@ dependencies = [ [[package]] name = "rkyv_derive" -version = "0.7.36" +version = "0.7.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fc752d5925dbcb324522f3a4c93193d17f107b2e11810913aa3ad352fa01480" +checksum = "e289706df51226e84814bf6ba1a9e1013112ae29bc7a9878f73fce360520c403" dependencies = [ "proc-macro2", "quote", @@ -612,7 +628,7 @@ dependencies = [ [[package]] name = "swc_common" -version = "0.17.19" +version = "0.17.21" dependencies = [ "ahash", "anyhow", @@ -638,7 +654,7 @@ dependencies = [ [[package]] name = "swc_ecma_ast" -version = "0.73.1" +version = "0.76.0" dependencies = [ "is-macro", "num-bigint", @@ -650,9 +666,22 @@ dependencies = [ "unicode-id", ] +[[package]] +name = "swc_ecma_utils" +version = "0.81.0" +dependencies = [ + "indexmap", + "once_cell", + "swc_atoms", + "swc_common", + "swc_ecma_ast", + "swc_ecma_visit", + "tracing", +] + [[package]] name = "swc_ecma_visit" -version = "0.59.0" +version = "0.62.0" dependencies = [ "num-bigint", "swc_atoms", @@ -682,7 +711,7 @@ dependencies = [ [[package]] name = "swc_macros_common" -version = "0.3.3" +version = "0.3.4" dependencies = [ "pmutil", "proc-macro2", @@ -692,23 +721,15 @@ dependencies = [ [[package]] name = "swc_plugin" -version = "0.41.0" +version = "0.47.0" dependencies = [ "swc_atoms", "swc_common", "swc_ecma_ast", + "swc_ecma_utils", "swc_ecma_visit", - "swc_plugin_comments", "swc_plugin_macro", -] - -[[package]] -name = "swc_plugin_comments" -version = "0.1.0" -dependencies = [ - "better_scoped_tls", - "rkyv", - "swc_common", + "swc_plugin_proxy", ] [[package]] @@ -720,6 +741,15 @@ dependencies = [ "syn", ] +[[package]] +name = "swc_plugin_proxy" +version = "0.1.2" +dependencies = [ + "better_scoped_tls", + "rkyv", + "swc_common", +] + [[package]] name = "swc_visit" version = "0.3.0"