feat(wasm/plugin): Implement initial plugin interface for wasm runtimes (#4123)

Cargo.lock

@@ -3080,6 +3080,7 @@ dependencies = [
  "serde_json",
  "swc",
  "swc_common",
+ "swc_plugin_runner",
  "swc_trace_macro",
  "tracing",
  "tracing-chrome",
@@ -4511,6 +4512,7 @@ dependencies = [
  "swc_common",
  "swc_ecma_lints",
  "swc_ecmascript",
+ "swc_plugin_runner",
  "tracing",
  "wasm-bindgen",
  "wasmer",

crates/swc/src/config/mod.rs

@@ -1,3 +1,4 @@
+#![cfg_attr(any(not(feature = "plugin"), target_arch = "wasm32"), allow(unused))]
 use std::{
     cell::RefCell,
     collections::{HashMap, HashSet},
@@ -438,26 +439,58 @@ impl Options {
                 comments,
             );
 
-        let plugin_resolver = CachingResolver::new(
-            40,
-            NodeModulesResolver::new(TargetEnv::Node, Default::default(), true),
-        );
+        let keep_import_assertions = experimental.keep_import_assertions;
 
-        let transform_filename = match base {
-            FileName::Real(path) => path.as_os_str().to_str().map(String::from),
-            FileName::Custom(filename) => Some(filename.to_owned()),
-            _ => None,
+        // Embedded runtime plugin target, based on assumption we have
+        // 1. filesystem access for the cache
+        // 2. embedded runtime can compiles & execute wasm
+        #[cfg(all(feature = "plugin", not(target_arch = "wasm32")))]
+        let plugins = {
+            let plugin_resolver = CachingResolver::new(
+                40,
+                NodeModulesResolver::new(TargetEnv::Node, Default::default(), true),
+            );
+
+            let transform_filename = match base {
+                FileName::Real(path) => path.as_os_str().to_str().map(String::from),
+                FileName::Custom(filename) => Some(filename.to_owned()),
+                _ => None,
+            };
+
+            let plugin_context = PluginContext {
+                filename: transform_filename,
+                env_name: self.env_name.to_owned(),
+            };
+
+            if experimental.plugins.is_some() {
+                swc_plugin_runner::cache::init_plugin_module_cache_once(&experimental.cache_root);
+            }
+
+            crate::plugin::plugins(plugin_resolver, experimental, plugin_context)
         };
 
-        let plugin_context = PluginContext {
-            filename: transform_filename,
-            env_name: self.env_name.to_owned(),
+        // Native runtime plugin target, based on assumption we have
+        // 1. no filesystem access, loading binary / cache management should be
+        // performed externally
+        // 2. native runtime compiles & execute wasm (i.e v8 on node, chrome)
+        #[cfg(all(feature = "plugin", target_arch = "wasm32"))]
+        let plugins = {
+            let transform_filename = match base {
+                FileName::Real(path) => path.as_os_str().to_str().map(String::from),
+                FileName::Custom(filename) => Some(filename.to_owned()),
+                _ => None,
+            };
+
+            let plugin_context = PluginContext {
+                filename: transform_filename,
+                env_name: self.env_name.to_owned(),
+            };
+
+            crate::plugin::plugins(experimental, plugin_context)
         };
 
-        #[cfg(feature = "plugin")]
-        if experimental.plugins.is_some() {
-            swc_plugin_runner::cache::init_plugin_module_cache_once(&experimental.cache_root);
-        }
+        #[cfg(not(feature = "plugin"))]
+        let plugins = crate::plugin::plugins();
 
         let pass = chain!(
             lint_to_fold(swc_ecma_lints::rules::all(LintParams {
@@ -477,7 +510,7 @@ impl Options {
             ),
             // The transform strips import assertions, so it's only enabled if
             // keep_import_assertions is false.
-            Optional::new(import_assertions(), !experimental.keep_import_assertions),
+            Optional::new(import_assertions(), !keep_import_assertions),
             Optional::new(
                 typescript::strip_with_jsx(
                     cm.clone(),
@@ -496,7 +529,7 @@ impl Options {
                 ),
                 syntax.typescript()
             ),
-            crate::plugin::plugins(plugin_resolver, experimental, plugin_context),
+            plugins,
             custom_before_pass(&program),
             // handle jsx
             Optional::new(

crates/swc/src/plugin.rs

@@ -41,12 +41,12 @@ pub struct PluginContext {
     pub env_name: String,
 }
 
+#[cfg(all(feature = "plugin", not(target_arch = "wasm32")))]
 pub fn plugins(
     resolver: CachingResolver<NodeModulesResolver>,
     config: crate::config::JscExperimental,
     plugin_context: PluginContext,
 ) -> impl Fold {
-    #[cfg(feature = "plugin")]
     {
         RustPlugins {
             resolver,
@@ -54,11 +54,16 @@ pub fn plugins(
             plugin_context,
         }
     }
+}
 
-    #[cfg(not(feature = "plugin"))]
-    {
-        noop()
-    }
+#[cfg(all(feature = "plugin", target_arch = "wasm32"))]
+pub fn plugins(config: crate::config::JscExperimental, plugin_context: PluginContext) -> impl Fold {
+    swc_ecma_transforms::pass::noop()
+}
+
+#[cfg(not(feature = "plugin"))]
+pub fn plugins() -> impl Fold {
+    noop()
 }
 
 struct RustPlugins {

crates/swc_cli/Cargo.toml

@@ -14,7 +14,7 @@ path = "./src/main.rs"
 
 [features]
 default = []
-plugin = ["swc/plugin", "wasmer/default", "wasmer-wasi/default"]
+plugin = ["swc/plugin", "swc_plugin_runner/filesystem_cache", "wasmer/default", "wasmer-wasi/default"]
 
 [dependencies]
 anyhow = "1.0.53"
@@ -27,6 +27,7 @@ serde_json = { version = "1", features = ["unbounded_depth"] }
 swc = { version = "0.159.0", path = "../swc" }
 swc_common = { version = "0.17.5", path = "../swc_common" }
 swc_trace_macro = { version = "0.1.0", path = "../swc_trace_macro" }
+swc_plugin_runner = { version = "0.46.0", path = "../swc_plugin_runner", default-features = false, optional = true }
 tracing = "0.1.32"
 tracing-chrome = "0.4.0"
 tracing-futures = "0.2.5"

crates/swc_plugin_runner/Cargo.toml

@@ -10,7 +10,12 @@ version = "0.46.0"
 
 [features]
 default = ["filesystem_cache"]
+# Supports a cache allow to store compiled bytecode into filesystem location.
+# This feature implies in-memory cache enabled always.
 filesystem_cache = ["wasmer-cache"]
+# Supports a cache allow to store wasm module in-memory. This avoids recompilation
+# to the same module in a single procress lifecycle.
+memory_cache = []
 
 [dependencies]
 anyhow = "1.0.42"

crates/swc_plugin_runner/src/cache.rs

@@ -13,6 +13,15 @@ use wasmer::{Module, Store};
 #[cfg(all(not(target_arch = "wasm32"), feature = "filesystem_cache"))]
 use wasmer_cache::{Cache as WasmerCache, FileSystemCache, Hash};
 
+#[cfg(all(not(feature = "filesystem_cache"), not(feature = "memory_cache")))]
+compile_error!("Plugin_runner should enable either filesystem, or memory cache");
+
+#[cfg(all(feature = "filesystem_cache", feature = "memory_cache"))]
+compile_error!(
+    "Only one cache feature should be enabled. If you enabled filesystem_cache, it activates its \
+     memory cache as well."
+);
+
 /// Version for bytecode cache stored in local filesystem.
 ///
 /// This MUST be updated when bump up wasmer.
@@ -24,30 +33,32 @@ use wasmer_cache::{Cache as WasmerCache, FileSystemCache, Hash};
 const MODULE_SERIALIZATION_VERSION: &str = "v2";
 
 /// A shared instance to plugin's module bytecode cache.
-/// TODO: it is unclear how we'll support plugin itself in wasm target of
-/// swc, as well as cache.
 pub static PLUGIN_MODULE_CACHE: Lazy<PluginModuleCache> = Lazy::new(Default::default);
 
+#[cfg(feature = "filesystem_cache")]
 #[derive(Default)]
 pub struct CacheInner {
-    #[cfg(feature = "filesystem_cache")]
     fs_cache: Option<FileSystemCache>,
-    #[cfg(feature = "filesystem_cache")]
-    memory_cache: InMemoryCache,
+    // A naive hashmap to the compiled plugin modules.
+    // Current it doesn't have any invalidation or expiration logics like lru,
+    // having a lot of plugins may create some memory pressure.
+    loaded_module_bytes: AHashMap<PathBuf, Module>,
 }
 
-/// Lightweight in-memory cache to hold plugin module instances.
-/// Current it doesn't have any invalidation or expiration logics like lru,
-/// having a lot of plugins may create some memory pressure.
+#[cfg(feature = "memory_cache")]
 #[derive(Default)]
-pub struct InMemoryCache {
-    modules: AHashMap<PathBuf, Module>,
+pub struct CacheInner {
+    // Unlike sys::Module, we'll keep raw bytes from the module instead of js::Module which
+    // implies bindgen's JsValue
+    loaded_module_bytes: AHashMap<PathBuf, Vec<u8>>,
 }
 
 #[derive(Default)]
 pub struct PluginModuleCache {
     inner: OnceCell<Mutex<CacheInner>>,
-    /// To prevent concurrent access to `WasmerInstance::new`
+    /// To prevent concurrent access to `WasmerInstance::new`.
+    /// This is a precaution only yet, for the preparation of wasm thread
+    /// support in the future.
     instantiation_lock: Mutex<()>,
 }
 
@@ -86,14 +97,16 @@ pub fn init_plugin_module_cache_once(filesystem_cache_root: &Option<String>) {
     PLUGIN_MODULE_CACHE.inner.get_or_init(|| {
         Mutex::new(CacheInner {
             fs_cache: create_filesystem_cache(filesystem_cache_root),
-            memory_cache: Default::default(),
+            loaded_module_bytes: Default::default(),
         })
     });
 
     #[cfg(target_arch = "wasm32")]
-    PLUGIN_MODULE_CACHE
-        .inner
-        .get_or_init(|| Mutex::new(CacheInner {}));
+    PLUGIN_MODULE_CACHE.inner.get_or_init(|| {
+        Mutex::new(CacheInner {
+            loaded_module_bytes: Default::default(),
+        })
+    });
 }
 
 impl PluginModuleCache {
@@ -125,7 +138,7 @@ impl PluginModuleCache {
         // if constructed Module is available in-memory, directly return it.
         // Note we do not invalidate in-memory cache currently: if wasm binary is
         // replaced in-process lifecycle (i.e devserver) it won't be reflected.
-        let in_memory_module = inner_cache.memory_cache.modules.get(&binary_path);
+        let in_memory_module = inner_cache.loaded_module_bytes.get(&binary_path);
         if let Some(module) = in_memory_module {
             return Ok(module.clone());
         }
@@ -166,8 +179,7 @@ impl PluginModuleCache {
         };
 
         inner_cache
-            .memory_cache
-            .modules
+            .loaded_module_bytes
             .insert(binary_path, module.clone());
 
         Ok(module)
@@ -175,6 +187,41 @@ impl PluginModuleCache {
 
     #[cfg(target_arch = "wasm32")]
     pub fn load_module(&self, binary_path: &Path) -> Result<Module, Error> {
+        let binary_path = binary_path.to_path_buf();
+        let mut inner_cache = self.inner.get().expect("Cache should be available").lock();
+
+        // if constructed Module is available in-memory, directly return it.
+        // Note we do not invalidate in-memory cache currently: if wasm binary is
+        // replaced in-process lifecycle (i.e devserver) it won't be reflected.
+        let in_memory_module_bytes = inner_cache.loaded_module_bytes.get(&binary_path);
+        if let Some(module) = in_memory_module_bytes {
+            //TODO: In native runtime we have to reconstruct module using raw bytes in
+            // memory cache. requires https://github.com/wasmerio/wasmer/pull/2821
+            unimplemented!("Not implemented yet");
+        }
+
         unimplemented!("Not implemented yet");
     }
+
+    /// An experimental interface to store externally loaded module bytes into
+    /// cache. This is primarily to support swc/wasm-* target, which does
+    /// not have way to access system, especially filesystem by default.
+    ///
+    /// Currently this doesn't do any validation or expiration: once a bytes set
+    /// with specific id, subsequent call will noop.
+    ///
+    /// This interface is not a public, but also will likely change anytime
+    /// while stablizing plugin interface.
+    #[cfg(target_arch = "wasm32")]
+    pub fn store_once(&self, module_name: &str, module_bytes: Vec<u8>) {
+        // We use path as canonical id for the cache
+        let binary_path = PathBuf::from(module_name);
+        let mut inner_cache = self.inner.get().expect("Cache should be available").lock();
+
+        if !inner_cache.loaded_module_bytes.contains_key(&binary_path) {
+            inner_cache
+                .loaded_module_bytes
+                .insert(binary_path, module_bytes);
+        }
+    }
 }

crates/wasm/Cargo.toml

@@ -18,6 +18,7 @@ swc_v2 = []
 # This feature exists to allow cargo operations
 plugin = [
   "swc/plugin",
+  "swc_plugin_runner/memory_cache",
   "wasmer",
   "wasmer-wasi",
   "wasmer/js-default",
@@ -38,6 +39,7 @@ swc_ecma_lints = { path = "../swc_ecma_lints", features = [
   "non_critical_lints",
 ] }
 swc_ecmascript = { path = "../swc_ecmascript" }
+swc_plugin_runner = { version = "0.46.0", path = "../swc_plugin_runner", default-features = false, optional = true }
 tracing = { version = "0.1.32", features = ["release_max_level_off"] }
 wasm-bindgen = { version = "0.2", features = ["serde-serialize"] }
 wasmer = { version = "2.2.1", optional = true, default-features = false }

crates/wasm/scripts/build.sh

@@ -1 +1 @@
-wasm-pack build --debug --scope swc -t nodejs $@
+wasm-pack build --debug --scope swc -t nodejs --features plugin $@

crates/wasm/scripts/build_nodejs_release.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 
 # run this script from the wasm folder ./scripts/build_nodejs_release.sh
-npx wasm-pack build --scope swc -t nodejs
+npx wasm-pack build --scope swc -t nodejs --features plugin

crates/wasm/scripts/build_web_release.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 
 # run this script from the wasm folder ./scripts/build_web_release.sh
-npx wasm-pack build --scope swc 
+npx wasm-pack build --scope swc --features plugin

crates/wasm/src/lib.rs

@@ -124,12 +124,55 @@ pub fn print_sync(s: JsValue, opts: JsValue) -> Result<JsValue, JsValue> {
     .map_err(convert_err)
 }
 
-#[wasm_bindgen(js_name = "transformSync")]
-pub fn transform_sync(s: &str, opts: JsValue) -> Result<JsValue, JsValue> {
+#[wasm_bindgen(typescript_custom_section)]
+const TRANSFORM_SYNC_DEFINITION: &'static str = r#"
+/**
+* @param {string} code
+* @param {any} opts
+* @param {Record<string, ArrayBuffer>} experimental_plugin_bytes_resolver An object contains bytes array for the plugin
+* specified in config. Key of record represents the name of the plugin specified in config. Note this is an experimental
+* interface, likely will change.
+* @returns {any}
+*/
+export function transformSync(code: string, opts: any, experimental_plugin_bytes_resolver?: any): any;
+"#;
+
+#[wasm_bindgen(
+    js_name = "transformSync",
+    typescript_type = "transformSync",
+    skip_typescript
+)]
+#[allow(unused_variables)]
+pub fn transform_sync(
+    s: &str,
+    opts: JsValue,
+    experimental_plugin_bytes_resolver: JsValue,
+) -> Result<JsValue, JsValue> {
     console_error_panic_hook::set_once();
 
     let c = compiler();
 
+    #[cfg(feature = "plugin")]
+    {
+        // TODO: This is probably very inefficient, including each transform
+        // deserializes plugin bytes.
+        let plugin_bytes = if experimental_plugin_bytes_resolver.is_object() {
+            JsValue::into_serde::<std::collections::HashMap<String, Vec<u8>>>(
+                &experimental_plugin_bytes_resolver,
+            )
+            .expect("Object should be available")
+        } else {
+            Default::default()
+        };
+
+        // In here we 'inject' externally loaded bytes into the cache, so remaining
+        // plugin_runner execution path works as much as similar between embedded
+        // runtime.
+        plugin_bytes.into_iter().for_each(|(key, bytes)| {
+            swc_plugin_runner::cache::PLUGIN_MODULE_CACHE.store_once(&key, bytes.clone())
+        });
+    }
+
     try_with_handler(
         c.cm.clone(),
         swc::HandlerOpts {