swc-project/swc
1
1
Fork 0
mirror of https://github.com/swc-project/swc.git synced 2024-12-24 06:05:02 +03:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

feat(plugin/runner): Support shared wasix runtime (#7504)

Browse Source
This commit is contained in:
OJ Kwon 2023-06-07 19:19:07 -07:00 committed by GitHub
parent 89bee900e4
commit 73929fc43c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 102 additions and 256 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Cargo.lock generated
View File

@ -4721,6 +4721,7 @@ dependencies = [
"swc_node_base",
"swc_plugin_proxy",
"testing",
"tokio",
"tracing",
"wasmer",
"wasmer-cache",
@ -5844,6 +5845,7 @@ dependencies = [
"pin-project",
"rand",
"serde",
"serde_cbor",
"serde_derive",
"serde_json",
"serde_yaml",

10
crates/swc/src/plugin.rs
View File

@ -100,6 +100,15 @@ impl RustPlugins {
.expect("plugin module should be loaded");
let plugin_name = plugin_module_bytes.get_module_name().to_string();
let runtime = swc_plugin_runner::wasix_runtime::build_wasi_runtime(
crate::config::PLUGIN_MODULE_CACHE
.inner
.get()
.unwrap()
.lock()
.get_fs_cache_root()
.map(|v| std::path::PathBuf::from(v)),
);
let mut transform_plugin_executor =
swc_plugin_runner::create_plugin_transform_executor(
&self.source_map,
@ -107,6 +116,7 @@ impl RustPlugins {
&self.metadata_context,
plugin_module_bytes,
Some(p.1),
runtime,
);
let span = tracing::span!(

3
crates/swc_core/Cargo.toml
View File

@ -206,6 +206,9 @@ plugin_transform_host_native_filesystem_cache = [
"swc_plugin_runner/filesystem_cache",
]
plugin_transform_host_native_shared_runtime = [
"swc_plugin_runner/plugin_transform_host_native_shared_runtime",
]
### Internal features that public features are relying on.
### This is not supposed to be used directly, and does not gaurantee
### stability across each versions.

6
crates/swc_plugin_runner/Cargo.toml
View File

@ -30,6 +30,11 @@ plugin_transform_host_native = [
"wasmer-wasix/host-threads",
"wasmer-compiler-cranelift/default",
]
plugin_transform_host_native_shared_runtime = [
"tokio",
"wasmer-wasix/webc_runner",
]
# Supports a cache allow to store compiled bytecode into filesystem location.
# This feature implies in-memory cache support. This is not supported on wasm32 target.
filesystem_cache = ["wasmer-cache"]
@ -51,6 +56,7 @@ once_cell = "1.10.0"
parking_lot = "0.12.0"
serde = { version = "1.0.126", features = ["derive"] }
serde_json = "1.0.64"
tokio = { version = "1", default-features = false, optional = true }
tracing = "0.1.32"
wasmer = { version = "3.3.0", default-features = false }
wasmer-wasix = { version = "0.4.0", default-features = false }

1
crates/swc_plugin_runner/benches/ecma_invoke.rs
View File

@ -91,6 +91,7 @@ fn bench_transform(b: &mut Bencher, plugin_dir: &Path) {
)),
Box::new(plugin_module.clone()),
None,
None,
);
let experimental_metadata: VersionedSerializable<AHashMap<String, String>> =

11
crates/swc_plugin_runner/src/cache.rs
View File

@ -35,6 +35,8 @@ const MODULE_SERIALIZATION_VERSION: &str = "v6";
#[derive(Default)]
pub struct PluginModuleCacheInner {
#[cfg(all(not(target_arch = "wasm32"), feature = "filesystem_cache"))]
fs_cache_root: Option<String>,
#[cfg(all(not(target_arch = "wasm32"), feature = "filesystem_cache"))]
fs_cache_store: Option<FileSystemCache>,
// Stores the string representation of the hash of the plugin module to store into
@ -51,6 +53,13 @@ pub struct PluginModuleCacheInner {
}
impl PluginModuleCacheInner {
pub fn get_fs_cache_root(&self) -> Option<String> {
#[cfg(all(not(target_arch = "wasm32"), feature = "filesystem_cache"))]
return self.fs_cache_root.clone();
None
}
/// Check if the cache contains bytes for the corresponding key.
pub fn contains(&self, key: &str) -> bool {
let is_in_cache = self.memory_cache_store.contains_key(key)
@ -183,6 +192,8 @@ impl PluginModuleCache {
fs_cache_store_root: &Option<String>,
) -> PluginModuleCacheInner {
PluginModuleCacheInner {
#[cfg(all(not(target_arch = "wasm32"), feature = "filesystem_cache"))]
fs_cache_root: fs_cache_store_root.clone(),
#[cfg(all(not(target_arch = "wasm32"), feature = "filesystem_cache"))]
fs_cache_store: if enable_fs_cache_store {
create_filesystem_cache(fs_cache_store_root)

4
crates/swc_plugin_runner/src/lib.rs
View File

@ -13,6 +13,7 @@ mod imported_fn;
mod memory_interop;
pub mod plugin_module_bytes;
mod transform_executor;
pub mod wasix_runtime;
use plugin_module_bytes::PluginModuleBytes;
@ -26,6 +27,7 @@ pub fn create_plugin_transform_executor(
metadata_context: &Arc<TransformPluginMetadataContext>,
plugin_module: Box<dyn PluginModuleBytes>,
plugin_config: Option<serde_json::Value>,
runtime: Option<Arc<dyn wasmer_wasix::WasiRuntime + Send + Sync>>,
) -> TransformExecutor {
TransformExecutor::new(
plugin_module,
@ -33,6 +35,7 @@ pub fn create_plugin_transform_executor(
unresolved_mark,
metadata_context,
plugin_config,
runtime,
)
}
@ -43,6 +46,7 @@ pub fn create_plugin_transform_executor(
metadata_context: &Arc<TransformPluginMetadataContext>,
plugin_module: Box<dyn PluginModuleBytes>,
plugin_config: Option<serde_json::Value>,
runtime: Option<()>,
) -> TransformExecutor {
unimplemented!("Transform plugin cannot be used without serialization support")
}

266
crates/swc_plugin_runner/src/transform_executor.rs
View File

@ -14,7 +14,7 @@ use swc_common::{
SourceMap,
};
use wasmer::{AsStoreMut, FunctionEnv, Instance, Store, TypedFunction};
use wasmer_wasix::{default_fs_backing, is_wasi_module, WasiEnv, WasiFunctionEnv};
use wasmer_wasix::{default_fs_backing, is_wasi_module, WasiEnv, WasiFunctionEnv, WasiRuntime};
use crate::plugin_module_bytes::PluginModuleBytes;
#[cfg(feature = "__rkyv")]
@ -119,13 +119,9 @@ impl PluginTransformState {
guest_program_ptr.1,
)?;
/* [Note]: currently this is disabled: this cleanup is for the multi-threaded
* wasi environment as well as cleaning up file handles which is for the cases
* running wasi binary as standalone. SWC doesn't need neither currently, will
* revisit once we support multithreaded plugin.
if let Some(wasi_env) = &self.wasi_env {
wasi_env.cleanup(&mut self.store, None);
}*/
}
ret
}
@ -177,6 +173,7 @@ pub struct TransformExecutor {
metadata_context: Arc<TransformPluginMetadataContext>,
plugin_config: Option<serde_json::Value>,
module_bytes: Box<dyn PluginModuleBytes>,
runtime: Option<Arc<dyn WasiRuntime + Send + Sync>>,
}
#[cfg(feature = "__rkyv")]
@ -191,6 +188,7 @@ impl TransformExecutor {
unresolved_mark: &swc_common::Mark,
metadata_context: &Arc<TransformPluginMetadataContext>,
plugin_config: Option<serde_json::Value>,
runtime: Option<Arc<dyn WasiRuntime + Send + Sync>>,
) -> Self {
Self {
source_map: source_map.clone(),
@ -198,6 +196,7 @@ impl TransformExecutor {
metadata_context: metadata_context.clone(),
plugin_config,
module_bytes,
runtime,
}
}
@ -260,7 +259,11 @@ impl TransformExecutor {
// TODO: wasm host native runtime throws 'Memory should be set on `WasiEnv`
// first'
let (instance, wasi_env) = if is_wasi_module(&module) {
let builder = WasiEnv::builder(self.module_bytes.get_module_name());
let mut builder = WasiEnv::builder(self.module_bytes.get_module_name());
if let Some(runtime) = &self.runtime {
builder.set_runtime(runtime.clone());
}
// Implicitly enable filesystem access for the wasi plugin to cwd.
//
@ -374,252 +377,3 @@ impl TransformExecutor {
transform_state.run(program, self.unresolved_mark, should_enable_comments_proxy)
}
}
/*
/// A struct encapsule executing a plugin's transform interop to its teardown
pub struct TransformExecutor {
// Main transform interface plugin exports
exported_plugin_transform: TypedFunction<(u32, u32, u32, u32), u32>,
// `__free` function automatically exported via swc_plugin sdk to allow deallocation in guest
// memory space
exported_plugin_free: TypedFunction<(u32, u32), u32>,
// `__alloc` function automatically exported via swc_plugin sdk to allow allocation in guest
// memory space
exported_plugin_alloc: TypedFunction<u32, u32>,
wasi_env: Option<WasiFunctionEnv>,
instance: Instance,
store: Store,
// Reference to the pointers successfully allocated which'll be freed by Drop.
allocated_ptr_vec: Vec<(u32, u32)>,
transform_result: Arc<Mutex<Vec<u8>>>,
// diagnostic metadata for the swc_core plugin binary uses.
pub plugin_core_diag: PluginCorePkgDiagnostics,
}
#[cfg(feature = "__rkyv")]
impl TransformExecutor {
#[tracing::instrument(
level = "info",
skip(source_map, metadata_context, plugin_config, module_bytes)
)]
pub fn new(
source_map: &Arc<SourceMap>,
metadata_context: &Arc<TransformPluginMetadataContext>,
plugin_config: Option<serde_json::Value>,
module_bytes: &dyn PluginModuleBytes,
) -> Self {
unimplemented!()
}
/// Creates a transform executor from a raw bytes.
#[tracing::instrument(
level = "info",
skip(source_map, metadata_context, plugin_config, plugin_module)
)]
pub fn from_raw_bytes(
source_map: &Arc<SourceMap>,
metadata_context: &Arc<TransformPluginMetadataContext>,
plugin_config: Option<serde_json::Value>,
plugin_module: &RawPluginModuleBytes,
) -> Result<Self, Error> {
let (store, module) = plugin_module.compile_module()?;
TransformExecutor::from_module(
source_map,
metadata_context,
plugin_config,
&plugin_module.plugin_name,
store,
module,
)
}
/// Creates a transform executor from a bytes seriaized by runtime (wasmer).
#[tracing::instrument(
level = "info",
skip(source_map, metadata_context, plugin_config, plugin_module)
)]
pub fn from_serialized_bytes(
source_map: &Arc<SourceMap>,
metadata_context: &Arc<TransformPluginMetadataContext>,
plugin_config: Option<serde_json::Value>,
plugin_module: &SerializedPluginModuleBytes,
) -> Result<Self, Error> {
let (store, module) = plugin_module.compile_module()?;
TransformExecutor::from_module(
source_map,
metadata_context,
plugin_config,
&plugin_module.plugin_name,
store,
module,
)
}
/// Creates a transform executor from an already compiled module.
#[tracing::instrument(
level = "info",
skip(source_map, metadata_context, plugin_config, store, module)
)]
pub fn from_module(
source_map: &Arc<SourceMap>,
metadata_context: &Arc<TransformPluginMetadataContext>,
plugin_config: Option<serde_json::Value>,
plugin_name: &str,
store: Store,
module: Module,
) -> Result<Self, Error> {
let (instance, transform_result, diagnostics_buffer, wasi_env) =
crate::load_plugin::load_plugin(
&mut store,
&mut module,
source_map,
metadata_context,
plugin_config,
plugin_name,
)?;
Ok(TransformExecutor {
exported_plugin_transform: instance
.exports
.get_typed_function(&store, "__transform_plugin_process_impl")?,
exported_plugin_free: instance.exports.get_typed_function(&store, "__free")?,
exported_plugin_alloc: instance.exports.get_typed_function(&store, "__alloc")?,
instance,
store,
wasi_env,
allocated_ptr_vec: Vec::with_capacity(3),
transform_result,
plugin_core_diag: diagnostics_buffer,
})
}
/// Copy host's serialized bytes into guest (plugin)'s allocated memory.
/// Once transformation completes, host should free allocated memory.
#[tracing::instrument(level = "info", skip_all)]
fn write_bytes_into_guest(
&mut self,
serialized_bytes: &PluginSerializedBytes,
) -> Result<(u32, u32), Error> {
let memory = self.instance.exports.get_memory("memory")?;
let ptr = write_into_memory_view(
memory,
&mut self.store.as_store_mut(),
serialized_bytes,
|s, serialized_len| {
self.exported_plugin_alloc
.call(s, serialized_len.try_into().expect("booo"))
.unwrap_or_else(|_| {
panic!(
"Should able to allocate memory for the size of {}",
serialized_len
)
})
},
);
self.allocated_ptr_vec.push(ptr);
Ok(ptr)
}
/// Copy guest's memory into host, construct serialized struct from raw
/// bytes.
fn read_transformed_result_bytes_from_guest(
&mut self,
returned_ptr_result: u32,
) -> Result<PluginSerializedBytes, Error> {
let transformed_result = &(*self.transform_result.lock());
let ret = PluginSerializedBytes::from_slice(&transformed_result[..]);
if returned_ptr_result == 0 {
Ok(ret)
} else {
let err: PluginError = ret.deserialize()?.into_inner();
match err {
PluginError::SizeInteropFailure(msg) => Err(anyhow!(
"Failed to convert pointer size to calculate: {}",
msg
)),
PluginError::Deserialize(msg) | PluginError::Serialize(msg) => {
Err(anyhow!("{}", msg))
}
_ => Err(anyhow!(
"Unexpected error occurred while running plugin transform"
)),
}
}
}
/**
* Check compile-time version of AST schema between the plugin and
* the host. Returns true if it's compatible, false otherwise.
*
* Host should appropriately handle if plugin is not compatible to the
* current runtime.
*/
#[allow(unreachable_code)]
pub fn is_transform_schema_compatible(&mut self) -> Result<bool, Error> {
#[cfg(any(
feature = "plugin_transform_schema_v1",
feature = "plugin_transform_schema_vtest"
))]
return {
let host_schema_version = PLUGIN_TRANSFORM_AST_SCHEMA_VERSION;
// TODO: this is incomplete
if host_schema_version >= self.plugin_core_diag.ast_schema_version {
Ok(true)
} else {
Ok(false)
}
};
#[cfg(not(all(
feature = "plugin_transform_schema_v1",
feature = "plugin_transform_schema_vtest"
)))]
anyhow::bail!(
"Plugin runner cannot detect plugin's schema version. Ensure host is compiled with \
proper versions"
)
}
#[tracing::instrument(level = "info", skip_all)]
pub fn transform(
&mut self,
program: &PluginSerializedBytes,
unresolved_mark: swc_common::Mark,
should_enable_comments_proxy: bool,
) -> Result<PluginSerializedBytes, Error> {
let should_enable_comments_proxy = u32::from(should_enable_comments_proxy);
let guest_program_ptr = self.write_bytes_into_guest(program)?;
let result = self.exported_plugin_transform.call(
&mut self.store,
guest_program_ptr.0,
guest_program_ptr.1,
unresolved_mark.as_u32(),
should_enable_comments_proxy,
)?;
self.read_transformed_result_bytes_from_guest(result)
}
}
impl Drop for TransformExecutor {
fn drop(&mut self) {
for ptr in self.allocated_ptr_vec.iter() {
self.exported_plugin_free
.call(&mut self.store, ptr.0, ptr.1)
.expect("Failed to free memory allocated in the plugin");
}
if let Some(wasi_env) = &self.wasi_env {
wasi_env.cleanup(&mut self.store, None);
}
}
}
*/

44
crates/swc_plugin_runner/src/wasix_runtime.rs Normal file
View File

@ -0,0 +1,44 @@
use std::{path::PathBuf, sync::Arc};
use wasmer_wasix::WasiRuntime;
/// Construct a runtime for the wasix engine depends on the compilation
/// features.
///
/// This is mainly for the case if a host already sets up its runtime, which
/// makes wasix initialization fails due to conflicting runtime. When specified,
/// instead of using default runtime it'll try to use shared one.
pub fn build_wasi_runtime(
fs_cache_path: Option<PathBuf>,
) -> Option<Arc<dyn WasiRuntime + Send + Sync>> {
#[cfg(not(feature = "plugin_transform_host_native_shared_runtime"))]
return None;
#[cfg(feature = "plugin_transform_host_native_shared_runtime")]
{
use wasmer_wasix::{
runners::Runner, runtime::task_manager::tokio::TokioTaskManager, PluggableRuntime,
};
let tasks = TokioTaskManager::new(tokio::runtime::Handle::current());
let mut rt = PluggableRuntime::new(Arc::new(tasks));
/* [TODO]: wasmer@4
#[cfg(all(not(target_arch = "wasm32"), feature = "filesystem_cache"))]
let cache = if let Some(fs_cache_path) = fs_cache_path {
SharedCache::default().with_fallback(wasmer_cache::FileSystemCache::new(fs_cache_path))
} else {
SharedCache::default().with_fallback(wasmer_wasix::runtime::module_cache::in_memory())
};
#[cfg(not(feature = "filesystem_cache"))]
let cache = SharedCache::default().with_fallback(in_memory());
*/
rt.set_engine(Some(wasmer::Engine::default()));
//[TODO]: wasmer@4
//rt.set_module_cache(cache);
return Some(Arc::new(rt));
}
}

3
crates/swc_plugin_runner/tests/css_rkyv.rs
View File

@ -107,6 +107,7 @@ fn invoke(input: PathBuf) -> Result<(), Error> {
)),
Box::new(PLUGIN_BYTES.clone()),
Some(json!({ "pluginConfig": "testValue" })),
None,
);
info!("Created transform executor");
@ -158,6 +159,7 @@ fn invoke(input: PathBuf) -> Result<(), Error> {
)),
Box::new(PLUGIN_BYTES.clone()),
Some(json!({ "pluginConfig": "testValue" })),
None,
);
serialized_program = plugin_transform_executor
@ -175,6 +177,7 @@ fn invoke(input: PathBuf) -> Result<(), Error> {
)),
Box::new(PLUGIN_BYTES.clone()),
Some(json!({ "pluginConfig": "testValue" })),
None,
);
serialized_program = plugin_transform_executor

4
crates/swc_plugin_runner/tests/ecma_integration.rs
View File

@ -135,6 +135,7 @@ fn internal() -> Result<(), Error> {
)),
Box::new(PLUGIN_BYTES.clone()),
Some(json!({ "pluginConfig": "testValue" })),
None,
);
/* [TODO]: reenable this later
@ -200,6 +201,7 @@ fn internal() -> Result<(), Error> {
)),
Box::new(PLUGIN_BYTES.clone()),
Some(json!({ "pluginConfig": "testValue" })),
None,
);
plugin_transform_executor
@ -248,6 +250,7 @@ fn internal() -> Result<(), Error> {
)),
Box::new(PLUGIN_BYTES.clone()),
Some(json!({ "pluginConfig": "testValue" })),
None,
);
serialized_program = plugin_transform_executor
@ -265,6 +268,7 @@ fn internal() -> Result<(), Error> {
)),
Box::new(PLUGIN_BYTES.clone()),
Some(json!({ "pluginConfig": "testValue" })),
None,
);
serialized_program = plugin_transform_executor

3
crates/swc_plugin_runner/tests/ecma_rkyv.rs
View File

@ -117,6 +117,7 @@ fn internal(input: PathBuf) -> Result<(), Error> {
)),
Box::new(PLUGIN_BYTES.clone()),
Some(json!({ "pluginConfig": "testValue" })),
None,
);
info!("Created transform executor");
@ -174,6 +175,7 @@ fn internal(input: PathBuf) -> Result<(), Error> {
)),
Box::new(PLUGIN_BYTES.clone()),
Some(json!({ "pluginConfig": "testValue" })),
None,
);
serialized_program = plugin_transform_executor
@ -191,6 +193,7 @@ fn internal(input: PathBuf) -> Result<(), Error> {
)),
Box::new(PLUGIN_BYTES.clone()),
Some(json!({ "pluginConfig": "testValue" })),
None,
);
serialized_program = plugin_transform_executor

1
crates/swc_plugin_runner/tests/issues.rs
View File

@ -134,6 +134,7 @@ fn issue_6404() -> Result<(), Error> {
)),
Box::new(plugin_module),
Some(json!({ "pluginConfig": "testValue" })),
None,
);
/* [TODO]: reenable this test