zed-industries/zed
1
1
Fork 0
mirror of https://github.com/zed-industries/zed.git synced 2024-09-16 00:47:39 +03:00
Code Issues Packages Projects Releases Wiki Activity

Remove unused plugin crates (#8350)

Browse Source 
This PR removes the unused crates for plugin support.

We're currently exploring Wasm-based extensions, and it's unlikely that
we'll be reusing any of this existing work.

Release Notes:

- N/A
This commit is contained in:
Marshall Bowers 2024-02-24 19:05:18 -05:00 committed by GitHub
parent 35bec9803a
commit 401798d9b8
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
24 changed files with 0 additions and 2090 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
Cargo.lock generated
View File

@ -6793,28 +6793,6 @@ dependencies = [
"time",
]
[[package]]
name = "plugin"
version = "0.1.0"
dependencies = [
"bincode",
"plugin_macros",
"serde",
"serde_derive",
]
[[package]]
name = "plugin_macros"
version = "0.1.0"
dependencies = [
"bincode",
"proc-macro2",
"quote",
"serde",
"serde_derive",
"syn 1.0.109",
]
[[package]]
name = "png"
version = "0.16.8"

2
Cargo.toml
View File

@ -50,8 +50,6 @@ members = [
"crates/notifications",
"crates/outline",
"crates/picker",
"crates/plugin",
"crates/plugin_macros",
"crates/prettier",
"crates/project",
"crates/project_panel",

1
crates/languages/Cargo.toml
View File

@ -4,7 +4,6 @@ version = "0.1.0"
edition = "2021"
publish = false
license = "GPL-3.0-or-later"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
anyhow.workspace = true

168
crates/languages/src/language_plugin.rs
View File

@ -1,168 +0,0 @@
use anyhow::{anyhow, Result};
use async_trait::async_trait;
use collections::HashMap;
use futures::lock::Mutex;
use gpui::executor::Background;
use language::{LanguageServerName, LspAdapter, LspAdapterDelegate};
use lsp2::LanguageServerBinary;
use plugin_runtime::{Plugin, PluginBinary, PluginBuilder, WasiFn};
use std::{any::Any, path::PathBuf, sync::Arc};
use util::ResultExt;
#[allow(dead_code)]
pub async fn new_json(executor: Arc<Background>) -> Result<PluginLspAdapter> {
let plugin = PluginBuilder::new_default()?
.host_function_async("command", |command: String| async move {
let mut args = command.split(' ');
let command = args.next().unwrap();
smol::process::Command::new(command)
.args(args)
.output()
.await
.log_err()
.map(|output| output.stdout)
})?
.init(PluginBinary::Precompiled(include_bytes!(
"../../../../plugins/bin/json_language.wasm.pre",
)))
.await?;
PluginLspAdapter::new(plugin, executor).await
}
pub struct PluginLspAdapter {
name: WasiFn<(), String>,
fetch_latest_server_version: WasiFn<(), Option<String>>,
fetch_server_binary: WasiFn<(PathBuf, String), Result<LanguageServerBinary, String>>,
cached_server_binary: WasiFn<PathBuf, Option<LanguageServerBinary>>,
initialization_options: WasiFn<(), String>,
language_ids: WasiFn<(), Vec<(String, String)>>,
executor: Arc<Background>,
runtime: Arc<Mutex<Plugin>>,
}
impl PluginLspAdapter {
#[allow(unused)]
pub async fn new(mut plugin: Plugin, executor: Arc<Background>) -> Result<Self> {
Ok(Self {
name: plugin.function("name")?,
fetch_latest_server_version: plugin.function("fetch_latest_server_version")?,
fetch_server_binary: plugin.function("fetch_server_binary")?,
cached_server_binary: plugin.function("cached_server_binary")?,
initialization_options: plugin.function("initialization_options")?,
language_ids: plugin.function("language_ids")?,
executor,
runtime: Arc::new(Mutex::new(plugin)),
})
}
}
#[async_trait]
impl LspAdapter for PluginLspAdapter {
async fn name(&self) -> LanguageServerName {
let name: String = self
.runtime
.lock()
.await
.call(&self.name, ())
.await
.unwrap();
LanguageServerName(name.into())
}
fn short_name(&self) -> &'static str {
"PluginLspAdapter"
}
async fn fetch_latest_server_version(
&self,
_: &dyn LspAdapterDelegate,
) -> Result<Box<dyn 'static + Send + Any>> {
let runtime = self.runtime.clone();
let function = self.fetch_latest_server_version;
self.executor
.spawn(async move {
let mut runtime = runtime.lock().await;
let versions: Result<Option<String>> =
runtime.call::<_, Option<String>>(&function, ()).await;
versions
.map_err(|e| anyhow!("{}", e))?
.ok_or_else(|| anyhow!("Could not fetch latest server version"))
.map(|v| Box::new(v) as Box<_>)
})
.await
}
async fn fetch_server_binary(
&self,
version: Box<dyn 'static + Send + Any>,
container_dir: PathBuf,
_: &dyn LspAdapterDelegate,
) -> Result<LanguageServerBinary> {
let version = *version.downcast::<String>().unwrap();
let runtime = self.runtime.clone();
let function = self.fetch_server_binary;
self.executor
.spawn(async move {
let mut runtime = runtime.lock().await;
let handle = runtime.attach_path(&container_dir)?;
let result: Result<LanguageServerBinary, String> =
runtime.call(&function, (container_dir, version)).await?;
runtime.remove_resource(handle)?;
result.map_err(|e| anyhow!("{}", e))
})
.await
}
async fn cached_server_binary(
&self,
container_dir: PathBuf,
_: &dyn LspAdapterDelegate,
) -> Option<LanguageServerBinary> {
let runtime = self.runtime.clone();
let function = self.cached_server_binary;
self.executor
.spawn(async move {
let mut runtime = runtime.lock().await;
let handle = runtime.attach_path(&container_dir).ok()?;
let result: Option<LanguageServerBinary> =
runtime.call(&function, container_dir).await.ok()?;
runtime.remove_resource(handle).ok()?;
result
})
.await
}
fn can_be_reinstalled(&self) -> bool {
false
}
async fn installation_test_binary(&self, _: PathBuf) -> Option<LanguageServerBinary> {
None
}
async fn initialization_options(&self) -> Option<serde_json::Value> {
let string: String = self
.runtime
.lock()
.await
.call(&self.initialization_options, ())
.await
.log_err()?;
serde_json::from_str(&string).ok()
}
async fn language_ids(&self) -> HashMap<String, String> {
self.runtime
.lock()
.await
.call(&self.language_ids, ())
.await
.log_err()
.unwrap_or_default()
.into_iter()
.collect()
}
}

2
crates/languages/src/lib.rs
View File

@ -25,8 +25,6 @@ mod go;
mod haskell;
mod html;
mod json;
#[cfg(feature = "plugin_runtime")]
mod language_plugin;
mod lua;
mod nu;
mod ocaml;

12
crates/plugin/Cargo.toml
View File

@ -1,12 +0,0 @@
[package]
name = "plugin"
version = "0.1.0"
edition = "2021"
publish = false
license = "GPL-3.0-or-later"
[dependencies]
bincode = "1.3"
plugin_macros.workspace = true
serde.workspace = true
serde_derive.workspace = true

1
crates/plugin/LICENSE-GPL
View File

@ -1 +0,0 @@
../../LICENSE-GPL

61
crates/plugin/src/lib.rs
View File

@ -1,61 +0,0 @@
pub use bincode;
pub use serde;
/// This is the buffer that is used Wasm side.
/// Note that it mirrors the functionality of
/// the `WasiBuffer` found in `plugin_runtime/src/plugin.rs`,
/// But has a few different methods.
pub struct __Buffer {
pub ptr: u32, // *const u8,
pub len: u32, // usize,
}
impl __Buffer {
pub fn into_u64(self) -> u64 {
((self.ptr as u64) << 32) | (self.len as u64)
}
pub fn from_u64(packed: u64) -> Self {
__Buffer {
ptr: (packed >> 32) as u32,
len: packed as u32,
}
}
}
/// Allocates a buffer with an exact size.
/// We don't return the size because it has to be passed in anyway.
#[no_mangle]
pub extern "C" fn __alloc_buffer(len: u32) -> u32 {
let vec = vec![0; len as usize];
let buffer = unsafe { __Buffer::from_vec(vec) };
buffer.ptr
}
/// Frees a given buffer, requires the size.
#[no_mangle]
pub extern "C" fn __free_buffer(buffer: u64) {
let vec = unsafe { __Buffer::from_u64(buffer).to_vec() };
std::mem::drop(vec);
}
impl __Buffer {
#[inline(always)]
pub unsafe fn to_vec(&self) -> Vec<u8> {
core::slice::from_raw_parts(self.ptr as *const u8, self.len as usize).to_vec()
}
#[inline(always)]
pub unsafe fn from_vec(mut vec: Vec<u8>) -> __Buffer {
vec.shrink_to(0);
let ptr = vec.as_ptr() as u32;
let len = vec.len() as u32;
std::mem::forget(vec);
__Buffer { ptr, len }
}
}
pub mod prelude {
pub use super::{__Buffer, __alloc_buffer};
pub use plugin_macros::{export, import};
}

17
crates/plugin_macros/Cargo.toml
View File

@ -1,17 +0,0 @@
[package]
name = "plugin_macros"
version = "0.1.0"
edition = "2021"
publish = false
license = "GPL-3.0-or-later"
[lib]
proc-macro = true
[dependencies]
bincode = "1.3"
proc-macro2 = "1.0"
quote = "1.0"
serde.workspace = true
serde_derive.workspace = true
syn = { version = "1.0", features = ["full", "extra-traits"] }

1
crates/plugin_macros/LICENSE-GPL
View File

@ -1 +0,0 @@
../../LICENSE-GPL

168
crates/plugin_macros/src/lib.rs
View File

@ -1,168 +0,0 @@
use core::panic;
use proc_macro::TokenStream;
use quote::{format_ident, quote};
use syn::{parse_macro_input, Block, FnArg, ForeignItemFn, Ident, ItemFn, Pat, Type, Visibility};
/// Attribute macro to be used guest-side within a plugin.
/// ```ignore
/// #[export]
/// pub fn say_hello() -> String {
/// "Hello from Wasm".into()
/// }
/// ```
/// This macro makes a function defined guest-side available host-side.
/// Note that all arguments and return types must be `serde`.
#[proc_macro_attribute]
pub fn export(args: TokenStream, function: TokenStream) -> TokenStream {
if !args.is_empty() {
panic!("The export attribute does not take any arguments");
}
let inner_fn = parse_macro_input!(function as ItemFn);
if !inner_fn.sig.generics.params.is_empty() {
panic!("Exported functions can not take generic parameters");
}
if let Visibility::Public(_) = inner_fn.vis {
} else {
panic!("The export attribute only works for public functions");
}
let inner_fn_name = format_ident!("{}", inner_fn.sig.ident);
let outer_fn_name = format_ident!("__{}", inner_fn_name);
let variadic = inner_fn.sig.inputs.len();
let i = (0..variadic).map(syn::Index::from);
let t: Vec<Type> = inner_fn
.sig
.inputs
.iter()
.map(|x| match x {
FnArg::Receiver(_) => {
panic!("All arguments must have specified types, no `self` allowed")
}
FnArg::Typed(item) => *item.ty.clone(),
})
.collect();
// this is cursed...
let (args, ty) = if variadic != 1 {
(
quote! {
#( data.#i ),*
},
quote! {
( #( #t ),* )
},
)
} else {
let ty = &t[0];
(quote! { data }, quote! { #ty })
};
TokenStream::from(quote! {
#[no_mangle]
#inner_fn
#[no_mangle]
pub extern "C" fn #outer_fn_name(packed_buffer: u64) -> u64 {
// setup
let data = unsafe { ::plugin::__Buffer::from_u64(packed_buffer).to_vec() };
// operation
let data: #ty = match ::plugin::bincode::deserialize(&data) {
Ok(d) => d,
Err(e) => panic!("Data passed to function not deserializable."),
};
let result = #inner_fn_name(#args);
let new_data: Result<Vec<u8>, _> = ::plugin::bincode::serialize(&result);
let new_data = new_data.unwrap();
// teardown
let new_buffer = unsafe { ::plugin::__Buffer::from_vec(new_data) }.into_u64();
return new_buffer;
}
})
}
/// Attribute macro to be used guest-side within a plugin.
/// ```ignore
/// #[import]
/// pub fn operating_system_name() -> String;
/// ```
/// This macro makes a function defined host-side available guest-side.
/// Note that all arguments and return types must be `serde`.
/// All that's provided is a signature, as the function is implemented host-side.
#[proc_macro_attribute]
pub fn import(args: TokenStream, function: TokenStream) -> TokenStream {
if !args.is_empty() {
panic!("The import attribute does not take any arguments");
}
let fn_declare = parse_macro_input!(function as ForeignItemFn);
if !fn_declare.sig.generics.params.is_empty() {
panic!("Exported functions can not take generic parameters");
}
// let inner_fn_name = format_ident!("{}", fn_declare.sig.ident);
let extern_fn_name = format_ident!("__{}", fn_declare.sig.ident);
let (args, tys): (Vec<Ident>, Vec<Type>) = fn_declare
.sig
.inputs
.clone()
.into_iter()
.map(|x| match x {
FnArg::Receiver(_) => {
panic!("All arguments must have specified types, no `self` allowed")
}
FnArg::Typed(t) => {
if let Pat::Ident(i) = *t.pat {
(i.ident, *t.ty)
} else {
panic!("All function arguments must be identifiers");
}
}
})
.unzip();
let body = TokenStream::from(quote! {
{
// setup
let data: (#( #tys ),*) = (#( #args ),*);
let data = ::plugin::bincode::serialize(&data).unwrap();
let buffer = unsafe { ::plugin::__Buffer::from_vec(data) };
// operation
let new_buffer = unsafe { #extern_fn_name(buffer.into_u64()) };
let new_data = unsafe { ::plugin::__Buffer::from_u64(new_buffer).to_vec() };
// teardown
match ::plugin::bincode::deserialize(&new_data) {
Ok(d) => d,
Err(e) => panic!("Data returned from function not deserializable."),
}
}
});
let block = parse_macro_input!(body as Block);
let inner_fn = ItemFn {
attrs: fn_declare.attrs,
vis: fn_declare.vis,
sig: fn_declare.sig,
block: Box::new(block),
};
TokenStream::from(quote! {
extern "C" {
fn #extern_fn_name(buffer: u64) -> u64;
}
#[no_mangle]
#inner_fn
})
}

21
crates/plugin_runtime/Cargo.toml
View File

@ -1,21 +0,0 @@
[package]
name = "plugin_runtime"
version = "0.1.0"
edition = "2021"
publish = false
license = "GPL-3.0-or-later"
[dependencies]
anyhow.workspace = true
bincode = "1.3"
pollster = "0.2.5"
serde.workspace = true
serde_derive.workspace = true
serde_json.workspace = true
smol.workspace = true
wasi-common = "2.0"
wasmtime = "2.0"
wasmtime-wasi = "2.0"
[build-dependencies]
wasmtime = { version = "2.0", features = ["all-arch"] }

1
crates/plugin_runtime/LICENSE-GPL
View File

@ -1 +0,0 @@
../../LICENSE-GPL

188
crates/plugin_runtime/OPAQUE.md
View File

@ -1,188 +0,0 @@
# Opaque handles to resources
Currently, Zed's plugin system only supports moving *data* (e.g. things you can serialize) across the boundary between guest-side plugin and host-side runtime. Resources, things you can't just copy, have been set aside for now. Given how important this is to Zed, I think it's about time we address this.
Managing resources is very important to Zed, because a lot of what Zed does is exactly that—managing resources. Each open buffer you're editing is a resource, as is the language server you're querying, or the collaboration session you're currently in. Therefore, writing a plugin system with deep integration with Zed requires some mechanism to manage resources.
The reason resources are problematic is because, unlike data, we can't pass resources across the ABI boundary. Wasm can't take references to host memory (and even if it could, that doesn't mean that it's a good idea). To add support for resources to plugins, we'd need three things:
1. Some sort of way for the host-side runtime to hang onto **references** to a resource. If the plugin requests to modify a resource, but we don't even know where that resource is, that's kinda bad, isn't it?
2. Some sort of way for the guest-side runtime to hang onto **handles** to a resource. We can't reference the resource directly from a plugin, but if a resource *has* been registered with the runtime, we can at least take a runtime-provided handle to that resource so that we may request that the runtime modify it in the future.
3. Some sort of way to **modify the resources** we're holding onto. This requires two things: some way for a plugin to request a modification, and some for the runtime to apply that modification. Here I'm using 'modification' in the most general sense, which includes, e.g. reading or writing to the resource, i.e. calling a method on it.
Luckily for us, managing resources across boundaries is a problem that languages have had to deal with for eons. File descriptors referencing resources managed by the kernel quintessentially defines of resource management, but this pattern is oft repeated in games, scripting languages, or surprise surprise, when writing plugins.
To see what managing resources in plugins could look like in Rust, we need look no further than Rhai. Rhai is a scripting language powered by a tree-walk interpreter written in Rust. It's pretty neat, but what we care about is not the language itself, but how it interfaces with Rust types.
In its [guide](https://rhai.rs/book/rust/custom-types.html), Rhai claims the following:
> Rhai works seamlessly with any Rust type, as long as it implements `Clone` as this allows the `Engine` to pass by value.
This doesn't mean that the underlying resources themselves need to be copied:
> \[Because Rhai works with types implementing `Clone`\] it is extremely easy to use Rhai with data types such as `Rc<...>`, `Arc<...>`, `Rc<RefCell<...>>`, `Arc<Mutex<...>>` etc.
Given that we have to register a resource with our plugin runtime before we use it, requiring the resource to be behind a shared reference makes sense, so I think the `Clone` bound is reasonable. So how does `Rhai` represent types under the hood?
> A custom type is stored in Rhai as a Rust trait object (specifically, a `dyn rhai::Variant`), with no restrictions other than being `Clone` (plus `Send + Sync` under the `sync` feature).
I'd be interested to know how Rhai disambiguates between different types if everything's a trait object under the hood.
Rhai actually exposes a pretty nice interface for working with native Rust types. We can register a type using `Engine::register_type::<T: Variant + Clone>()`. Internally, this just grabs the string name of the type for future reference.
> **Note**: Rhai uses strings, but I wonder if you could get away with something more compact using `TypeIds`. Maybe not, given that `TypeId`s are not deterministic across builds, and we'd need matching IDs both host-side and guest side.
In Rhai, we can alternatively use the method `Engine::register_type_with_name::<T: Variant + Clone>(name: &str)` if we have a different type name host-side (in Rust) and guest-side (in Rhai).
With respect to Wasm plugins, I think an interface like this is fairly important, because we don't know whether the original plugin was written in Rust. (This may not be true now, because we write all the plugins Zed uses, but once we allow packaging and shipping plugins, it's important to maintain a consistent interface, because even Rust changes over time.)
Once we've registered a type, we can begin using this type in functions. We can add new function using the standard `Engine::register_fn` function, which has the following signature:
```rust
pub fn register_fn<N, A, F>(&mut self, name: N, func: F) -> &mut Self
where
N: AsRef<str> + Into<Identifier>,
F: RegisterNativeFunction<A, ()>,
```
This is quite complex, but under the hood it's fairly similar to our own `PluginBuilder::host_function` async method. Looking at `RegisterNativeFunction`, it seems as though this trait essentially provides methods that expose the `TypeID`s and type/param names of the arguments and return types of the function.
So once we register a function, what happens when we call it? Well, let me introduce you to my friend `Engine::call_native_fn`, whose type signature is too complex to list here.
> **Note**: Finding this function took like 7 levels of indirection from `eval`. It's surprising how much shuffling of data Rhai does under the hood, I bet you could probably make it a lot faster.
This takes and returns, like everything else in Rhai, an object of type `Dynamic`. We know that we can use native Rust types, so how does Rhai perform the conversion to and from `Dynamic`?
The secret lies in `Dynamic::try_cast::<T: Any>(self) -> Option<T>`. Like most dynamic scripting languages, Rhai uses a tagged `Union` to represent types. Remember `dyn Variant` from earlier? Rhai's `Union` has a variant, `Variant`, to hold the dynamic native types:
```rust
/// Any type as a trait object.
#[allow(clippy::redundant_allocation)]
Variant(Box<Box<dyn Variant>>, Tag, AccessMode),
```
Redundant allocations aside, To `try_cast` a `Dynamic` type to `T: Any`thing, we pattern match on `Union`. In the case of variant, we:
```rust
Union::Variant(v, ..) => (*v).as_boxed_any().downcast().ok().map(|x| *x),
```
Now Rhai can do this because it's implemented in Rust. In other words, unlike Wasm, Rhai scripts can, indirectly, hold references to places in host memory. For us to implement something like this for Wasm plugins, we'd have to keep track of a "`ResourcePool`"—alive for the duration of each function call—that we can check rust types into and out of.
I think I've got a handle on how Rhai works now, so let's stop talking about Rhai and discuss what this opaque object system would look like if we implemented it in Rust.
# Design Sketch
First things first, we'd have to generalize the arguments we can pass to and return from functions host-side. Currently, we support anything that's `serde`able. We'd have to create a new trait, say `Value`, that has blanket implementations for both `serde` and `Clone` (or something like this; if a type is both `serde` and `clone`, we'd have to figure out a way to disambiguate).
We'd also create a `ResourcePool` struct that essentially is a `Vec` of `Box<dyn Any>`. When calling a function, all `Value` arguments that are resources (e.g. `Clone` instead of `serde`) would be typecasted to `dyn Any` and stored in the `ResourcePool`.
We'd probably also need a `Resource` trait that defines an associated handle for a resource. Something like this:
```rust
pub trait Resource {
type Handle: Serialize + DeserializeOwned;
fn handle(index: u32) -> Self;
fn index(handle: Self) -> u32;
}
```
Where a handle is just a dead-simple wrapper around a `u32`:
```rust
#[derive(Serialize, Deserialize)]
pub struct CoolHandle(u32);
```
It's important that this handle be accessible *both* host-side and plugin side. I don't know if this means that we have another crate, like `plugin_handles`, that contains a bunch of u32 wrappers, or something else. Because a `Resource::Handle` is just a u32, it's trivially `serde`, and can cross the ABI boundary.
So when we add each `T: Resource` to the `ResourcePool`, the resource pool typecasts it to `Any`, appends it to the `Vec`, and returns the associated `Resource::Handle`. This handle is what we pass through to Wasm.
```rust
// Implementations and attributes omitted
pub struct Rope { ... };
pub struct RopeHandle(u32);
impl Resource for Arc<RwLock<Rope>> { ... }
let builder: PluginBuilder = ...;
let builder = builder
.host_fn_async(
"append",
|(rope, string): (Arc<RwLock<Rope>>, &str)| async move {
rope.write().await.append(Rope::from(string))
}
)
// ...
```
He're we're providing a host function, `append` that can be called from Wasm. To import this function into a plugin, we'd do something like the following:
```rust
use plugin::prelude::*;
use plugin_handles::RopeHandle;
#[import]
pub fn append(rope: RopeHandle, string: &str);
```
This allows us to perform an operation on a `Rope`, but how do we get a `RopeHandle` into a plugin? Well, as plugins, we can only acquire resources to handles we're given, so we'd need to expose a function that takes a handle.
To illustrate that point, here's an example. First, we'd define a plugin-side function as follows:
```rust
// same file as above ...
#[export]
pub fn append_newline(rope: RopeHandle){
append(rope, "\n");
}
```
Host-side, we'd treat this function like any other:
```rust
pub struct NewlineAppenderPlugin {
append_newline: WasiFn<Arc<RwLock<Rope>>, ()>,
runtime: Arc<Mutex<Plugin>>,
}
```
To call this function, we'd do the following:
```rust
let plugin: NewlineAppenderPlugin = ...;
let rope = Arc::new(RwLock::new(Rope::from("Hello World")));
plugin.lock().await.call(
&plugin.append_newline,
rope.clone(),
).await?;
// `rope` is now "Hello World\n"
```
So here's what calling `append_newline` would do, from the top:
1. First, we'd create a new `ResourcePool`, and insert the `Arc<RwLock<Rope>>`, creating a `RopeHandle` in the process. (We could also reuse a resource pool across calls, but the idea is that the pool only keeps track of resources for the duration of the call).
2. Then, we'd call the Wasm plugin function `append_newline`, passing in the `RopeHandle` we created, which easily crosses the ABI boundary.
3. Next, in Wasm, we call the native imported function `append`. This sends the `RopeHandle` back over the boundary, to Rust.
4. Looking in the `Plugin`'s `ResourcePool`, we'd convert the handle into an index, grab and downcast the `dyn Any` back into the type we need, and then call the async Rust callback with an `Arc<RwLock<Rope>>`.
5. The Rust async callback actually acquires a lock and appends the newline.
6. And from here on out we return up the callstack, through Wasm, to Rust all the way back to where we started. Right before we return, we clear out the `ResourcePool`, so that we're no longer holding onto the underlying resource.
Throughout this entire chain of calls, the resource remain host-side. By temporarily checking it into a `ResourcePool`, we're able to keep a reference to the resource that we can use, while avoiding copying the uncopyable resource.
## Final Notes
Using this approach, it should be possible to add fairly good support for resources to Wasm. I've only done a little rough prototyping, so we're bound to run into some issues along the way, but I think this should be a good first approximation.
This next week, I'll try to get a production-ready version of this working, using the `Language` resource required by some Language Server Adapters.
Hope this guide made sense!

320
crates/plugin_runtime/README.md
View File

@ -1,320 +0,0 @@
# Zed's Plugin Runner
This is a short guide that aims to answer the following questions:
- How do plugins work in Zed?
- How can I create a new plugin?
- How can I integrate plugins into a part of Zed?
### Nomenclature
- Host-side: The native Rust runtime managing plugins, e.g. Zed.
- Guest-side: The wasm-based runtime that plugins use.
## How plugins work
Zed's plugins are WebAssembly (Wasm) based, and have access to the WebAssembly System Interface (WASI), which allows for permissions-based access to subsets of system resources, like the filesystem.
To execute plugins, Zed's plugin system uses the sandboxed [`wasmtime`](https://wasmtime.dev/) runtime, which is Open Source and developed by the [Bytecode Alliance](https://bytecodealliance.org/). Wasmtime uses the [Cranelift](https://docs.rs/cranelift/latest/cranelift/) codegen library to compile plugins to native code.
Zed has three `plugin` crates that implement different things:
1. `plugin_runtime` is a host-side library that loads and runs compiled `Wasm` plugins, in addition to setting up system bindings. This crate should be used host-side
2. `plugin` contains a prelude for guest-side plugins to depend on. It re-exports some required crates (e.g. `serde`, `bincode`) and provides some necessary macros for generating bindings that `plugin_runtime` can hook into.
3. `plugin_macros` implements the proc macros required by `plugin`, like the `#[import]` and `#[export]` attribute macros, and should also be used guest-side.
### ABI
The interface between the host Rust runtime ('Runtime') and plugins implemented in Wasm ('Plugin') is pretty simple.
When calling a guest-side function, all arguments are serialized to bytes and passed through `Buffer`s. We currently use `serde` + [`bincode`](https://docs.rs/bincode/latest/bincode/) to do this serialization. This means that any type that can be serialized using serde can be passed across the ABI boundary. For types that represent resources that cannot pass the ABI boundary (e.g. `Rope`), we are working on an opaque callback-based system.
> **Note**: It's important to note that there is a draft ABI standard for Wasm called WebAssembly Interface Types (often abbreviated `WITX`). This standard is currently not stable and only experimentally supported in some runtimes. Once this proposal becomes stable, it would be a good idea to transition towards using WITX as the ABI, rather than the rather rudimentary `bincode` ABI we have now.
All `Buffer`s are stored in Wasm linear memory (Wasm memory). A `Buffer` is a pointer, length pair to a byte array somewhere in Wasm memory. A `Buffer` itself is represented as a pair of two 4-byte (`u32`) fields:
```rust
struct Buffer {
ptr: u32,
len: u32,
}
```
Which we encode as a single `u64` when crossing the ABI boundary:
```
+-------+-------+
| ptr | len |
+-------+-------+
|
~ ~ ~ ~ | ~ ~ ~ ~ spOoky ABI boundary O.o
V
+---------------+
| u64 |
+---------------+
```
All functions that a plugin exports or imports have the following properties:
- A function signature of `fn(u64) -> u64`, where both the argument (input) and return type (output) are a `Buffer`:
- The input `Buffer` will contain the input arguments serialized to `bincode`.
- The output `Buffer` will contain the output arguments serialized to `bincode`.
- Have a name starting with two underscores.
Luckily for us, we don't have to worry about mangling names or writing serialization code. The `plugin::prelude::*` defines a couple of macros—aptly named `#[import]` and `#[export]`—that generate all serialization code and perform all mangling of names requisite for crossing the ABI boundary.
There are also a couple important things every plugin must have:
- `__alloc_buffer` function that, given a `u32` length, returns a `u32` pointer to a buffer of that length.
- `__free_buffer` function that, given a buffer encoded as a `u64`, frees the buffer at the given location, and does not return anything.
Luckily enough for us yet again, the `plugin` prelude defines two ready-made versions of these functions, so you don't have to worry about implementing them yourselves.
So, what does importing and exporting functions from a plugin look like in practice? I'm glad you asked...
## Creating new plugins
Since Zed's plugin system uses Wasm + WASI, in theory any language that compiles to Wasm can be used to write plugins. In practice, and out of practicality, however, we currently only really support plugins written in Rust.
A plugin is just a rust crate like any other. All plugins embedded in Zed are located in the `plugins` folder in the root. These plugins will automatically be compiled, optimized, and recompiled on change, so it's recommended that when creating a new plugin you create it there.
As plugins are compiled to Wasm + WASI, you need to have the `wasm32-wasi` toolchain installed on your system. If you don't have it already, a little rustup magick will do the trick:
```bash
rustup target add wasm32-wasi
```
### Configuring a plugin
After you've created a new plugin in `plugins` using `cargo new --lib`, edit your `Cargo.toml` to ensure that it looks something like this:
```toml
[package]
name = "my_very_cool_incredible_plugin_with_a_short_name_of_course"
version = "0.1.0"
edition = "2021"
[lib]
crate-type = ["cdylib"]
[dependencies]
plugin = { path = "../../crates/plugin" }
[profile.release]
opt-level = "z"
lto = true
```
Here's a quick explainer of what we're doing:
- `crate-type = ["cdylib"]` is used because a plugin essentially acts a *library*, exposing functions with specific signatures that perform certain tasks. This key ensures that the library is generated in a reproducible manner with a layout `plugin_runtime` knows how to hook into.
- `plugin = { path = "../../crates/plugin" }` is used so we have access to the prelude, which has a few useful functions and can automatically generate serialization glue code for us.
- `[profile.release]` these options wholistically optimize for size, which will become increasingly important as we add more plugins.
### Importing and Exporting functions
To import or export a function, all you need are two things:
1. Make sure that you've imported `plugin::prelude::*`
2. Annotate your function or signature with `#[export]` or `#[import]` respectively.
Here's an example plugin that doubles the value of every float in a `Vec<f64>` passed into it:
```rust
use plugin::prelude::*;
#[export]
pub fn double(mut x: Vec<f64>) -> Vec<f64> {
x.into_iter().map(|x| x * 2.0).collect()
}
```
All the serialization code is automatically generated by `#[export]`.
You can specify functions that must be defined host-side by using the `#[import]` attribute. This attribute must be attached to a function signature:
```rust
use plugin::prelude::*;
#[import]
fn run(command: String) -> Vec<u8>;
```
The `#[import]` macro will generate a function body that performs the proper serialization/deserialization needed to call out to the host rust runtime. Note that the same `serde` + `bincode` + `Buffer` ABI is used for both `#[import]` and `#[export]`.
> **Note**: If you'd like to see an example of importing and exporting functions, check out the `test_plugin`, which can be found in the `plugins` directory.
## Integrating plugins into Zed
Currently, plugins are used to add support for language servers to Zed. Plugins should be fairly simple to integrate for library-like applications. Here's a quick overview of how plugins work:
### Normal vs Precompiled plugins
Plugins in the `plugins` directory are automatically recompiled and serialized to disk when compiling Zed. The resulting artifacts can be found in the `plugins/bin` directory. For each `plugin`, you should see two files:
- `plugin.wasm` is the plugin compiled to Wasm. As a baseline, this should be about 4MB for debug builds and 2MB for release builds, but it depends on the specific plugin being built.
- `plugin.wasm.pre` is the plugin compiled to Wasm *and additionally* precompiled to host-platform-specific native code, determined by the `TARGET` cargo exposes at compile-time. This should be about 700KB for debug builds and 500KB in release builds. Each plugin takes about 1 or 2 seconds to compile to native code using cranelift, so precompiling plugins drastically reduces the startup time required to begin to run a plugin.
For all intents and purposes, it is *highly recommended* that you use precompiled plugins where possible, as they are much more lightweight and take much less time to instantiate.
### Instantiating a plugin
So you have something you'd like to add a plugin for. What now? The general pattern for adding support for plugins is as follows:
#### 1. Create a struct to hold the plugin
To call the functions that a plugin exports host-side, you need to have 'handles' to those functions. Each handle is typed and stored in `WasiFn<A, R>` where `A: Serialize` and `R: DeserializeOwned`.
For example, let's suppose we're creating a plugin that:
1. formats a message
2. processes a list of numbers somehow
We could create a struct for this plugin as follows:
```rust
use plugin_runtime::{WasiFn, Plugin};
pub struct CoolPlugin {
format: WasiFn<String, String>,
process: WasiFn<Vec<f64>, f64>,
runtime: Plugin,
}
```
Note that this plugin also holds an owned reference to the runtime, which is stored in the `Plugin` type. In asynchronous or multithreaded contexts, it may be required to put `Plugin` behind an `Arc<Mutex<Plugin>>`. Although plugins expose an asynchronous interface, the underlying Wasm engine can only execute a single function at a time.
> **Note**: This is a limitation of the WebAssembly standard itself. In the future, to work around this, we've been considering starting a pool of plugins, or instantiating a new plugin per call (this isn't as bad as it sounds, as instantiating a new plugin only takes about 30µs).
In the following steps, we're going to build a plugin and extract handles to fill this struct we've created.
#### 2. Bind all imported functions
While a plugin can export functions, it can also import them. We'll refer to the host-side functions that a plugin imports as 'native' functions. Native functions are represented using callbacks, and both synchronous and asynchronous callbacks are supported.
To bind imported functions, the first thing we need to do is create a new plugin using `PluginBuilder`. `PluginBuilder` uses the builder pattern to configure a new plugin, after which calling the `init` method will instantiate the `Plugin`.
You can create a new plugin builder as follows:
```rust
let builder = PluginBuilder::new_with_default_ctx();
```
This creates a plugin with a sensible default set of WASI permissions, namely the ability to write to `stdout` and `stderr` (note that, by default, plugins do not have access to `stdin`). For more control, you can use `PluginBuilder::new` and pass in a `WasiCtx` manually.
##### Synchronous Functions
To add a sync native function to a plugin, use the `.host_function` method:
```rust
let builder = builder.host_function(
"add_f64",
|(a, b): (f64, f64)| a + b,
).unwrap();
```
The `.host_function` method takes two arguments: the name of the function, and a sync callback that implements it. Note that this name must match the name of the function declared in the plugin exactly. For example, to use the `add_f64` from a plugin, you must include the following `#[import]` signature:
```rust
use plugin::prelude::*;
#[import]
fn add_f64(a: f64, b: f64) -> f64;
```
Note that the specific names of the arguments do not matter, as long as they are unique. Once a function has been imported, it may be used in the plugin as any other Rust function.
##### Asynchronous Functions
To add an async native function to a plugin, use the `.host_function_async` method:
```rust
let builder = builder.host_function_async(
"half",
|n: f64| async move { n / 2.0 },
).unwrap();
```
This method works exactly the same as the `.host_function` method, but requires a callback that returns an async future. On the plugin side, there is no distinction made between sync and async functions (as Wasm has no built-in notion of sync vs. async), so the required import signature should *not* use the `async` keyword:
```rust
use plugin::prelude::*;
#[import]
fn half(n: f64) -> f64;
```
All functions declared by the builder must be imported by the Wasm plugin, otherwise an error will be raised.
#### 3. Get the compiled plugin
Once all imports are marked, we can instantiate the plugin. To instantiate the plugin, simply call the `.init` method on a `PluginBuilder`:
```rust
let plugin = builder
.init(
PluginBinary::Precompiled(bytes),
)
.await
.unwrap();
```
The `.init` method takes a single argument containing the plugin binary.
1. If not precompiled, use `PluginBinary::Wasm(bytes)`. This supports both the WebAssembly Textual format (`.wat`) and the WebAssembly Binary format (`.wasm`).
2. If precompiled, use `PluginBinary::Precompiled(bytes)`. This supports precompiled plugins ending in `.wasm.pre`. You need to be extra-careful when using precompiled plugins to ensure that the plugin target matches the target of the binary you are compiling.
The `.init` method is asynchronous, and must be `.await`ed upon. If the plugin is malformed or doesn't import the right functions, an error will be raised.
#### 4. Get handles to all exported functions
Once the plugin has been compiled, it's time to start filling in the plugin struct defined earlier. In the case of `CoolPlugin` from earlier, this can be done as follows:
```rust
let mut cool_plugin = CoolPlugin {
format: plugin.function("format").unwrap(),
process: plugin.function("process").unwrap(),
runtime: plugin,
};
```
Because the struct definition defines the types of functions we're grabbing handles to, it's not required to specify the types of the functions here.
Note that, yet again, the names of guest-side functions we import must match exactly. Here's an example of what that implementation might look like:
```rust
use plugin::prelude::*;
#[export]
pub fn format(message: String) -> String {
format!("Cool Plugin says... '{}!'", message)
}
#[export]
pub fn process(numbers: Vec<f64>) -> f64 {
// Process by calculating the average
let mut total = 0.0;
for number in numbers.into_iter() {
total += number;
}
total / numbers.len()
}
```
That's it! Now you have a struct that holds an instance of a plugin. The last thing you need to know is how to call out the plugin you've defined...
### Using a plugin
To call a plugin function, use the async `.call` method on `Plugin`:
```rust
let average = cool_plugin.runtime
.call(
&cool_plugin.process,
vec![1.0, 2.0, 3.0],
)
.await
.unwrap();
```
The `.call` method takes two arguments:
1. A reference to the handle of the function we want to call.
2. The input argument to this function.
This method is async, and must be `.await`ed. If something goes wrong (e.g. the plugin panics, or there is a type mismatch between the plugin and `WasiFn`), then this method will return an error.
## Last Notes
This has been a brief overview of how the plugin system currently works in Zed. We hope to implement higher-level affordances as time goes on, to make writing plugins easier, and providing tooling so that users of Zed may also write plugins to extend their own editors.

97
crates/plugin_runtime/build.rs
View File

@ -1,97 +0,0 @@
use std::{io::Write, path::Path};
use wasmtime::{Config, Engine};
fn main() {
let base = Path::new("../../plugins");
// Find all files and folders that don't change when rebuilt
let crates = std::fs::read_dir(base).expect("Could not find plugin directory");
for dir in crates {
let path = dir.unwrap().path();
let name = path.file_name().and_then(|x| x.to_str());
let is_dir = path.is_dir();
if is_dir && name != Some("target") && name != Some("bin") {
println!("cargo:rerun-if-changed={}", path.display());
}
}
// Clear out and recreate the plugin bin directory
let _ = std::fs::remove_dir_all(base.join("bin"));
std::fs::create_dir_all(base.join("bin")).expect("Could not make plugins bin directory");
// Compile the plugins using the same profile as the current Zed build
let (profile_flags, profile_target) = match std::env::var("PROFILE").unwrap().as_str() {
"debug" => (&[][..], "debug"),
"release" => (&["--release"][..], "release"),
unknown => panic!("unknown profile `{}`", unknown),
};
// Invoke cargo to build the plugins
let build_successful = std::process::Command::new("cargo")
.args([
"build",
"--target",
"wasm32-wasi",
"--manifest-path",
base.join("Cargo.toml").to_str().unwrap(),
])
.args(profile_flags)
.status()
.expect("Could not build plugins")
.success();
assert!(build_successful);
// Get the target architecture for pre-cross-compilation of plugins
// and create and engine with the appropriate config
let target_triple = std::env::var("TARGET").unwrap();
println!("cargo:rerun-if-env-changed=TARGET");
let engine = create_default_engine(&target_triple);
// Find all compiled binaries
let binaries = std::fs::read_dir(base.join("target/wasm32-wasi").join(profile_target))
.expect("Could not find compiled plugins in target");
// Copy and precompile all compiled plugins we can find
for file in binaries {
let is_wasm = || {
let path = file.ok()?.path();
if path.extension()? == "wasm" {
Some(path)
} else {
None
}
};
if let Some(path) = is_wasm() {
let out_path = base.join("bin").join(path.file_name().unwrap());
std::fs::copy(&path, &out_path).expect("Could not copy compiled plugin to bin");
precompile(&out_path, &engine);
}
}
}
/// Creates an engine with the default configuration.
/// N.B. This must create an engine with the same config as the one
/// in `plugin_runtime/src/plugin.rs`.
fn create_default_engine(target_triple: &str) -> Engine {
let mut config = Config::default();
config
.target(target_triple)
.unwrap_or_else(|_| panic!("Could not set target to `{}`", target_triple));
config.async_support(true);
config.consume_fuel(true);
Engine::new(&config).expect("Could not create precompilation engine")
}
fn precompile(path: &Path, engine: &Engine) {
let bytes = std::fs::read(path).expect("Could not read wasm module");
let compiled = engine
.precompile_module(&bytes)
.expect("Could not precompile module");
let out_path = path.parent().unwrap().join(&format!(
"{}.pre",
path.file_name().unwrap().to_string_lossy(),
));
let mut out_file = std::fs::File::create(out_path)
.expect("Could not create output file for precompiled module");
out_file.write_all(&compiled).unwrap();
}

92
crates/plugin_runtime/src/lib.rs
View File

@ -1,92 +0,0 @@
pub mod plugin;
pub use plugin::*;
#[cfg(test)]
mod tests {
use super::*;
use pollster::FutureExt as _;
#[test]
pub fn test_plugin() {
pub struct TestPlugin {
noop: WasiFn<(), ()>,
constant: WasiFn<(), u32>,
identity: WasiFn<u32, u32>,
add: WasiFn<(u32, u32), u32>,
swap: WasiFn<(u32, u32), (u32, u32)>,
sort: WasiFn<Vec<u32>, Vec<u32>>,
print: WasiFn<String, ()>,
and_back: WasiFn<u32, u32>,
imports: WasiFn<u32, u32>,
half_async: WasiFn<u32, u32>,
echo_async: WasiFn<String, String>,
}
async {
let mut runtime = PluginBuilder::new_default()
.unwrap()
.host_function("mystery_number", |input: u32| input + 7)
.unwrap()
.host_function("import_noop", |_: ()| ())
.unwrap()
.host_function("import_identity", |input: u32| input)
.unwrap()
.host_function("import_swap", |(a, b): (u32, u32)| (b, a))
.unwrap()
.host_function_async("import_half", |a: u32| async move { a / 2 })
.unwrap()
.host_function_async("command_async", |command: String| async move {
let mut args = command.split(' ');
let command = args.next().unwrap();
smol::process::Command::new(command)
.args(args)
.output()
.await
.ok()
.map(|output| output.stdout)
})
.unwrap()
.init(PluginBinary::Wasm(
include_bytes!("../../../plugins/bin/test_plugin.wasm").as_ref(),
))
.await
.unwrap();
let plugin = TestPlugin {
noop: runtime.function("noop").unwrap(),
constant: runtime.function("constant").unwrap(),
identity: runtime.function("identity").unwrap(),
add: runtime.function("add").unwrap(),
swap: runtime.function("swap").unwrap(),
sort: runtime.function("sort").unwrap(),
print: runtime.function("print").unwrap(),
and_back: runtime.function("and_back").unwrap(),
imports: runtime.function("imports").unwrap(),
half_async: runtime.function("half_async").unwrap(),
echo_async: runtime.function("echo_async").unwrap(),
};
let unsorted = vec![1, 3, 4, 2, 5];
let sorted = vec![1, 2, 3, 4, 5];
runtime.call(&plugin.noop, ()).await.unwrap();
assert_eq!(runtime.call(&plugin.constant, ()).await.unwrap(), 27);
assert_eq!(runtime.call(&plugin.identity, 58).await.unwrap(), 58);
assert_eq!(runtime.call(&plugin.add, (3, 4)).await.unwrap(), 7);
assert_eq!(runtime.call(&plugin.swap, (1, 2)).await.unwrap(), (2, 1));
assert_eq!(runtime.call(&plugin.sort, unsorted).await.unwrap(), sorted);
runtime.call(&plugin.print, "Hi!".into()).await.unwrap();
assert_eq!(runtime.call(&plugin.and_back, 1).await.unwrap(), 8);
assert_eq!(runtime.call(&plugin.imports, 1).await.unwrap(), 8);
assert_eq!(runtime.call(&plugin.half_async, 4).await.unwrap(), 2);
assert_eq!(
runtime
.call(&plugin.echo_async, "eko".into())
.await
.unwrap(),
"eko\n"
);
}
.block_on()
}
}

584
crates/plugin_runtime/src/plugin.rs
View File

@ -1,584 +0,0 @@
use std::future::Future;
use std::{fs::File, marker::PhantomData, path::Path};
use anyhow::{anyhow, Error};
use serde::{de::DeserializeOwned, Serialize};
use wasi_common::{dir, file};
use wasmtime::Memory;
use wasmtime::{
AsContext, AsContextMut, Caller, Config, Engine, Extern, Instance, Linker, Module, Store, Trap,
TypedFunc,
};
use wasmtime_wasi::{Dir, WasiCtx, WasiCtxBuilder};
/// Represents a resource currently managed by the plugin, like a file descriptor.
pub struct PluginResource(u32);
/// This is the buffer that is used Host side.
/// Note that it mirrors the functionality of
/// the `__Buffer` found in the `plugin/src/lib.rs` prelude.
struct WasiBuffer {
ptr: u32,
len: u32,
}
impl WasiBuffer {
pub fn into_u64(self) -> u64 {
((self.ptr as u64) << 32) | (self.len as u64)
}
pub fn from_u64(packed: u64) -> Self {
WasiBuffer {
ptr: (packed >> 32) as u32,
len: packed as u32,
}
}
}
/// Represents a typed WebAssembly function.
pub struct WasiFn<A: Serialize, R: DeserializeOwned> {
function: TypedFunc<u64, u64>,
_function_type: PhantomData<fn(A) -> R>,
}
impl<A: Serialize, R: DeserializeOwned> Copy for WasiFn<A, R> {}
impl<A: Serialize, R: DeserializeOwned> Clone for WasiFn<A, R> {
fn clone(&self) -> Self {
Self {
function: self.function,
_function_type: PhantomData,
}
}
}
pub struct Metering {
initial: u64,
refill: u64,
}
impl Default for Metering {
fn default() -> Self {
Metering {
initial: 1000,
refill: 1000,
}
}
}
/// This struct is used to build a new [`Plugin`], using the builder pattern.
/// Creates a new default plugin with `PluginBuilder::new_with_default_ctx`,
/// and add host-side exported functions using `host_function` and `host_function_async`.
/// Finalize the plugin by calling [`init`].
pub struct PluginBuilder {
wasi_ctx: WasiCtx,
engine: Engine,
linker: Linker<WasiCtxAlloc>,
metering: Metering,
}
/// Creates an engine with the default configuration.
/// N.B. This must create an engine with the same config as the one
/// in `plugin_runtime/build.rs`.
fn create_default_engine() -> Result<Engine, Error> {
let mut config = Config::default();
config.async_support(true);
config.consume_fuel(true);
Engine::new(&config)
}
impl PluginBuilder {
/// Creates a new [`PluginBuilder`] with the given WASI context.
/// Using the default context is a safe bet, see [`new_with_default_context`].
/// This plugin will yield after a configurable amount of fuel is consumed.
pub fn new(wasi_ctx: WasiCtx, metering: Metering) -> Result<Self, Error> {
let engine = create_default_engine()?;
let linker = Linker::new(&engine);
Ok(PluginBuilder {
wasi_ctx,
engine,
linker,
metering,
})
}
/// Creates a new `PluginBuilder` with the default `WasiCtx` (see [`default_ctx`]).
/// This plugin will yield after a configurable amount of fuel is consumed.
pub fn new_default() -> Result<Self, Error> {
let default_ctx = WasiCtxBuilder::new()
.inherit_stdout()
.inherit_stderr()
.build();
let metering = Metering::default();
Self::new(default_ctx, metering)
}
/// Add an `async` host function. See [`host_function`] for details.
pub fn host_function_async<F, A, R, Fut>(
mut self,
name: &str,
function: F,
) -> Result<Self, Error>
where
F: Fn(A) -> Fut + Send + Sync + 'static,
Fut: Future<Output = R> + Send + 'static,
A: DeserializeOwned + Send + 'static,
R: Serialize + Send + Sync + 'static,
{
self.linker.func_wrap1_async(
"env",
&format!("__{}", name),
move |mut caller: Caller<'_, WasiCtxAlloc>, packed_buffer: u64| {
// TODO: use try block once available
let result: Result<(WasiBuffer, Memory, _), Trap> = (|| {
// grab a handle to the memory
let plugin_memory = match caller.get_export("memory") {
Some(Extern::Memory(mem)) => mem,
_ => return Err(Trap::new("Could not grab slice of plugin memory"))?,
};
let buffer = WasiBuffer::from_u64(packed_buffer);
// get the args passed from Guest
let args =
Plugin::buffer_to_bytes(&plugin_memory, caller.as_context(), &buffer)?;
let args: A = Plugin::deserialize_to_type(args)?;
// Call the Host-side function
let result = function(args);
Ok((buffer, plugin_memory, result))
})();
Box::new(async move {
let (buffer, mut plugin_memory, future) = result?;
let result: R = future.await;
let result: Result<Vec<u8>, Error> = Plugin::serialize_to_bytes(result)
.map_err(|_| {
Trap::new("Could not serialize value returned from function").into()
});
let result = result?;
Plugin::buffer_to_free(caller.data().free_buffer(), &mut caller, buffer)
.await?;
let buffer = Plugin::bytes_to_buffer(
caller.data().alloc_buffer(),
&mut plugin_memory,
&mut caller,
result,
)
.await?;
Ok(buffer.into_u64())
})
},
)?;
Ok(self)
}
/// Add a new host function to the given `PluginBuilder`.
/// A host function is a function defined host-side, in Rust,
/// that is accessible guest-side, in WebAssembly.
/// You can specify host-side functions to import using
/// the `#[input]` macro attribute:
/// ```ignore
/// #[input]
/// fn total(counts: Vec<f64>) -> f64;
/// ```
/// When loading a plugin, you need to provide all host functions the plugin imports:
/// ```ignore
/// let plugin = PluginBuilder::new_with_default_context()
/// .host_function("total", |counts| counts.iter().fold(0.0, |tot, n| tot + n))
/// // and so on...
/// ```
/// And that's a wrap!
pub fn host_function<A, R>(
mut self,
name: &str,
function: impl Fn(A) -> R + Send + Sync + 'static,
) -> Result<Self, Error>
where
A: DeserializeOwned + Send,
R: Serialize + Send + Sync,
{
self.linker.func_wrap1_async(
"env",
&format!("__{}", name),
move |mut caller: Caller<'_, WasiCtxAlloc>, packed_buffer: u64| {
// TODO: use try block once available
let result: Result<(WasiBuffer, Memory, Vec<u8>), Trap> = (|| {
// grab a handle to the memory
let plugin_memory = match caller.get_export("memory") {
Some(Extern::Memory(mem)) => mem,
_ => return Err(Trap::new("Could not grab slice of plugin memory"))?,
};
let buffer = WasiBuffer::from_u64(packed_buffer);
// get the args passed from Guest
let args = Plugin::buffer_to_type(&plugin_memory, &mut caller, &buffer)?;
// Call the Host-side function
let result: R = function(args);
// Serialize the result back to guest
let result = Plugin::serialize_to_bytes(result).map_err(|_| {
Trap::new("Could not serialize value returned from function")
})?;
Ok((buffer, plugin_memory, result))
})();
Box::new(async move {
let (buffer, mut plugin_memory, result) = result?;
Plugin::buffer_to_free(caller.data().free_buffer(), &mut caller, buffer)
.await?;
let buffer = Plugin::bytes_to_buffer(
caller.data().alloc_buffer(),
&mut plugin_memory,
&mut caller,
result,
)
.await?;
Ok(buffer.into_u64())
})
},
)?;
Ok(self)
}
/// Initializes a [`Plugin`] from a given compiled Wasm module.
/// Both binary (`.wasm`) and text (`.wat`) module formats are supported.
pub async fn init(self, binary: PluginBinary<'_>) -> Result<Plugin, Error> {
Plugin::init(binary, self).await
}
}
#[derive(Copy, Clone)]
struct WasiAlloc {
alloc_buffer: TypedFunc<u32, u32>,
free_buffer: TypedFunc<u64, ()>,
}
struct WasiCtxAlloc {
wasi_ctx: WasiCtx,
alloc: Option<WasiAlloc>,
}
impl WasiCtxAlloc {
fn alloc_buffer(&self) -> TypedFunc<u32, u32> {
self.alloc
.expect("allocator has been not initialized, cannot allocate buffer!")
.alloc_buffer
}
fn free_buffer(&self) -> TypedFunc<u64, ()> {
self.alloc
.expect("allocator has been not initialized, cannot free buffer!")
.free_buffer
}
fn init_alloc(&mut self, alloc: WasiAlloc) {
self.alloc = Some(alloc)
}
}
pub enum PluginBinary<'a> {
Wasm(&'a [u8]),
Precompiled(&'a [u8]),
}
/// Represents a WebAssembly plugin, with access to the WebAssembly System Interface.
/// Build a new plugin using [`PluginBuilder`].
pub struct Plugin {
store: Store<WasiCtxAlloc>,
instance: Instance,
}
impl Plugin {
/// Dumps the *entirety* of Wasm linear memory to `stdout`.
/// Don't call this unless you're debugging a memory issue!
pub fn dump_memory(data: &[u8]) {
for (i, byte) in data.iter().enumerate() {
if i % 32 == 0 {
println!();
}
if i % 4 == 0 {
print!("|");
}
if *byte == 0 {
print!("__")
} else {
print!("{:02x}", byte);
}
}
println!();
}
async fn init(binary: PluginBinary<'_>, plugin: PluginBuilder) -> Result<Self, Error> {
// initialize the WebAssembly System Interface context
let engine = plugin.engine;
let mut linker = plugin.linker;
wasmtime_wasi::add_to_linker(&mut linker, |s| &mut s.wasi_ctx)?;
// create a store, note that we can't initialize the allocator,
// because we can't grab the functions until initialized.
let mut store: Store<WasiCtxAlloc> = Store::new(
&engine,
WasiCtxAlloc {
wasi_ctx: plugin.wasi_ctx,
alloc: None,
},
);
let module = match binary {
PluginBinary::Precompiled(bytes) => unsafe { Module::deserialize(&engine, bytes)? },
PluginBinary::Wasm(bytes) => Module::new(&engine, bytes)?,
};
// set up automatic yielding based on configuration
store.add_fuel(plugin.metering.initial).unwrap();
store.out_of_fuel_async_yield(u64::MAX, plugin.metering.refill);
// load the provided module into the asynchronous runtime
linker.module_async(&mut store, "", &module).await?;
let instance = linker.instantiate_async(&mut store, &module).await?;
// now that the module is initialized,
// we can initialize the store's allocator
let alloc_buffer = instance.get_typed_func(&mut store, "__alloc_buffer")?;
let free_buffer = instance.get_typed_func(&mut store, "__free_buffer")?;
store.data_mut().init_alloc(WasiAlloc {
alloc_buffer,
free_buffer,
});
Ok(Plugin { store, instance })
}
/// Attaches a file or directory the the given system path to the runtime.
/// Note that the resource must be freed by calling `remove_resource` afterwards.
pub fn attach_path<T: AsRef<Path>>(&mut self, path: T) -> Result<PluginResource, Error> {
// grab the WASI context
let ctx = self.store.data_mut();
// open the file we want, and convert it into the right type
// this is a footgun and a half
let file = File::open(&path).unwrap();
let dir = Dir::from_std_file(file);
let dir = Box::new(wasmtime_wasi::dir::Dir::from_cap_std(dir));
// grab an empty file descriptor, specify capabilities
let fd = ctx.wasi_ctx.table().push(Box::new(()))?;
let caps = dir::DirCaps::all();
let file_caps = file::FileCaps::all();
// insert the directory at the given fd,
// return a handle to the resource
ctx.wasi_ctx
.insert_dir(fd, dir, caps, file_caps, path.as_ref().to_path_buf());
Ok(PluginResource(fd))
}
/// Returns `true` if the resource existed and was removed.
/// Currently the only resource we support is adding scoped paths (e.g. folders and files)
/// to plugins using [`attach_path`].
pub fn remove_resource(&mut self, resource: PluginResource) -> Result<(), Error> {
self.store
.data_mut()
.wasi_ctx
.table()
.delete(resource.0)
.ok_or_else(|| anyhow!("Resource did not exist, but a valid handle was passed in"))?;
Ok(())
}
// So this call function is kinda a dance, I figured it'd be a good idea to document it.
// the high level is we take a serde type, serialize it to a byte array,
// (we're doing this using bincode for now)
// then toss that byte array into webassembly.
// webassembly grabs that byte array, does some magic,
// and serializes the result into yet another byte array.
// we then grab *that* result byte array and deserialize it into a result.
//
// phew...
//
// now the problem is, webassembly doesn't support buffers.
// only really like i32s, that's it (yeah, it's sad. Not even unsigned!)
// (ok, I'm exaggerating a bit).
//
// the Wasm function that this calls must have a very specific signature:
//
// fn(pointer to byte array: i32, length of byte array: i32)
// -> pointer to (
// pointer to byte_array: i32,
// length of byte array: i32,
// ): i32
//
// This pair `(pointer to byte array, length of byte array)` is called a `Buffer`
// and can be found in the cargo_test plugin.
//
// so on the wasm side, we grab the two parameters to the function,
// stuff them into a `Buffer`,
// and then pray to the `unsafe` Rust gods above that a valid byte array pops out.
//
// On the flip side, when returning from a wasm function,
// we convert whatever serialized result we get into byte array,
// which we stuff into a Buffer and allocate on the heap,
// which pointer to we then return.
// Note the double indirection!
//
// So when returning from a function, we actually leak memory *twice*:
//
// 1) once when we leak the byte array
// 2) again when we leak the allocated `Buffer`
//
// This isn't a problem because Wasm stops executing after the function returns,
// so the heap is still valid for our inspection when we want to pull things out.
/// Serializes a given type to bytes.
fn serialize_to_bytes<A: Serialize>(item: A) -> Result<Vec<u8>, Error> {
// serialize the argument using bincode
let bytes = bincode::serialize(&item)?;
Ok(bytes)
}
/// Deserializes a given type from bytes.
fn deserialize_to_type<R: DeserializeOwned>(bytes: &[u8]) -> Result<R, Error> {
// serialize the argument using bincode
let bytes = bincode::deserialize(bytes)?;
Ok(bytes)
}
// fn deserialize<R: DeserializeOwned>(
// plugin_memory: &mut Memory,
// mut store: impl AsContextMut<Data = WasiCtxAlloc>,
// buffer: WasiBuffer,
// ) -> Result<R, Error> {
// let buffer_start = buffer.ptr as usize;
// let buffer_end = buffer_start + buffer.len as usize;
// // read the buffer at this point into a byte array
// // deserialize the byte array into the provided serde type
// let item = &plugin_memory.data(store.as_context())[buffer_start..buffer_end];
// let item = bincode::deserialize(bytes)?;
// Ok(item)
// }
/// Takes an item, allocates a buffer, serializes the argument to that buffer,
/// and returns a (ptr, len) pair to that buffer.
async fn bytes_to_buffer(
alloc_buffer: TypedFunc<u32, u32>,
plugin_memory: &mut Memory,
mut store: impl AsContextMut<Data = WasiCtxAlloc>,
item: Vec<u8>,
) -> Result<WasiBuffer, Error> {
// allocate a buffer and write the argument to that buffer
let len = item.len() as u32;
let ptr = alloc_buffer.call_async(&mut store, len).await?;
plugin_memory.write(&mut store, ptr as usize, &item)?;
Ok(WasiBuffer { ptr, len })
}
/// Takes a `(ptr, len)` pair and returns the corresponding deserialized buffer.
fn buffer_to_type<R: DeserializeOwned>(
plugin_memory: &Memory,
store: impl AsContext<Data = WasiCtxAlloc>,
buffer: &WasiBuffer,
) -> Result<R, Error> {
let buffer_start = buffer.ptr as usize;
let buffer_end = buffer_start + buffer.len as usize;
// read the buffer at this point into a byte array
// deserialize the byte array into the provided serde type
let result = &plugin_memory.data(store.as_context())[buffer_start..buffer_end];
let result = bincode::deserialize(result)?;
Ok(result)
}
/// Takes a `(ptr, len)` pair and returns the corresponding deserialized buffer.
fn buffer_to_bytes<'a>(
plugin_memory: &'a Memory,
store: wasmtime::StoreContext<'a, WasiCtxAlloc>,
buffer: &'a WasiBuffer,
) -> Result<&'a [u8], Error> {
let buffer_start = buffer.ptr as usize;
let buffer_end = buffer_start + buffer.len as usize;
// read the buffer at this point into a byte array
// deserialize the byte array into the provided serde type
let result = &plugin_memory.data(store)[buffer_start..buffer_end];
Ok(result)
}
async fn buffer_to_free(
free_buffer: TypedFunc<u64, ()>,
mut store: impl AsContextMut<Data = WasiCtxAlloc>,
buffer: WasiBuffer,
) -> Result<(), Error> {
// deallocate the argument buffer
Ok(free_buffer
.call_async(&mut store, buffer.into_u64())
.await?)
}
/// Retrieves the handle to a function of a given type.
pub fn function<A: Serialize, R: DeserializeOwned, T: AsRef<str>>(
&mut self,
name: T,
) -> Result<WasiFn<A, R>, Error> {
let fun_name = format!("__{}", name.as_ref());
let fun = self
.instance
.get_typed_func::<u64, u64, _>(&mut self.store, &fun_name)?;
Ok(WasiFn {
function: fun,
_function_type: PhantomData,
})
}
/// Asynchronously calls a function defined Guest-side.
pub async fn call<A: Serialize, R: DeserializeOwned>(
&mut self,
handle: &WasiFn<A, R>,
arg: A,
) -> Result<R, Error> {
let mut plugin_memory = self
.instance
.get_memory(&mut self.store, "memory")
.ok_or_else(|| anyhow!("Could not grab slice of plugin memory"))?;
// write the argument to linear memory
// this returns a (ptr, length) pair
let arg_buffer = Self::bytes_to_buffer(
self.store.data().alloc_buffer(),
&mut plugin_memory,
&mut self.store,
Self::serialize_to_bytes(arg)?,
)
.await?;
// call the function, passing in the buffer and its length
// this returns a ptr to a (ptr, length) pair
let result_buffer = handle
.function
.call_async(&mut self.store, arg_buffer.into_u64())
.await?;
Self::buffer_to_type(
&plugin_memory,
&mut self.store,
&WasiBuffer::from_u64(result_buffer),
)
}
}

129
plugins/Cargo.lock generated
View File

@ -1,129 +0,0 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 3
[[package]]
name = "bincode"
version = "1.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
dependencies = [
"serde",
]
[[package]]
name = "itoa"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d"
[[package]]
name = "json_language"
version = "0.1.0"
dependencies = [
"plugin",
"serde",
"serde_derive",
"serde_json",
]
[[package]]
name = "plugin"
version = "0.1.0"
dependencies = [
"bincode",
"plugin_macros",
"serde",
"serde_derive",
]
[[package]]
name = "plugin_macros"
version = "0.1.0"
dependencies = [
"bincode",
"proc-macro2",
"quote",
"serde",
"serde_derive",
"syn",
]
[[package]]
name = "proc-macro2"
version = "1.0.39"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c54b25569025b7fc9651de43004ae593a75ad88543b17178aa5e1b9c4f15f56f"
dependencies = [
"unicode-ident",
]
[[package]]
name = "quote"
version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1"
dependencies = [
"proc-macro2",
]
[[package]]
name = "ryu"
version = "1.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3f6f92acf49d1b98f7a81226834412ada05458b7364277387724a237f062695"
[[package]]
name = "serde"
version = "1.0.137"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61ea8d54c77f8315140a05f4c7237403bf38b72704d031543aa1d16abbf517d1"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.137"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1f26faba0c3959972377d3b2d306ee9f71faee9714294e41bb777f83f88578be"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "serde_json"
version = "1.0.82"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "82c2c1fdcd807d1098552c5b9a36e425e42e9fbd7c6a37a8425f390f781f7fa7"
dependencies = [
"itoa",
"ryu",
"serde",
]
[[package]]
name = "syn"
version = "1.0.96"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0748dd251e24453cb8717f0354206b91557e4ec8703673a4b30208f2abaf1ebf"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "test_plugin"
version = "0.1.0"
dependencies = [
"plugin",
]
[[package]]
name = "unicode-ident"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d22af068fba1eb5edcb4aea19d382b2a3deb4c8f9d475c589b6ada9e0fd493ee"

2
plugins/Cargo.toml
View File

@ -1,2 +0,0 @@
[workspace]
members = ["./json_language", "./test_plugin"]

13
plugins/json_language/Cargo.toml
View File

@ -1,13 +0,0 @@
[package]
name = "json_language"
version = "0.1.0"
edition = "2021"
[dependencies]
plugin = { path = "../../crates/plugin" }
serde = { version = "1.0", features = ["derive", "rc"] }
serde_derive = { version = "1.0", features = ["deserialize_in_place"] }
serde_json = "1.0"
[lib]
crate-type = ["cdylib"]

96
plugins/json_language/src/lib.rs
View File

@ -1,96 +0,0 @@
use std::{fs, path::PathBuf};
use plugin::prelude::*;
use serde::Deserialize;
#[import]
fn command(string: &str) -> Option<Vec<u8>>;
const SERVER_PATH: &str = "node_modules/vscode-json-languageserver/bin/vscode-json-languageserver";
#[export]
pub fn name() -> &'static str {
"vscode-json-languageserver"
}
#[export]
pub fn server_args() -> Vec<String> {
vec!["--stdio".into()]
}
#[export]
pub fn fetch_latest_server_version() -> Option<String> {
#[derive(Deserialize)]
struct NpmInfo {
versions: Vec<String>,
}
let output =
command("npm info vscode-json-languageserver --json").expect("could not run command");
let output = String::from_utf8(output).unwrap();
let mut info: NpmInfo = serde_json::from_str(&output).ok()?;
info.versions.pop()
}
#[export]
pub fn fetch_server_binary(container_dir: PathBuf, version: String) -> Result<PathBuf, String> {
let version_dir = container_dir.join(version.as_str());
fs::create_dir_all(&version_dir)
.map_err(|_| "failed to create version directory".to_string())?;
let binary_path = version_dir.join(SERVER_PATH);
if fs::metadata(&binary_path).is_err() {
let output = command(&format!(
"npm install vscode-json-languageserver@{}",
version
));
let output = output.map(String::from_utf8);
if output.is_none() {
return Err("failed to install vscode-json-languageserver".to_string());
}
if let Ok(entries) = fs::read_dir(&container_dir) {
for entry in entries.flatten() {
let entry_path = entry.path();
if entry_path.as_path() != version_dir {
fs::remove_dir_all(&entry_path).ok();
}
}
}
}
Ok(binary_path)
}
#[export]
pub fn cached_server_binary(container_dir: PathBuf) -> Option<PathBuf> {
let mut last_version_dir = None;
let entries = fs::read_dir(&container_dir).ok()?;
for entry in entries {
let entry = entry.ok()?;
if entry.file_type().ok()?.is_dir() {
last_version_dir = Some(entry.path());
}
}
let last_version_dir = last_version_dir?;
let server_path = last_version_dir.join(SERVER_PATH);
if server_path.exists() {
Some(server_path)
} else {
println!("no binary found");
None
}
}
#[export]
pub fn initialization_options() -> Option<String> {
Some("{ \"provideFormatter\": true }".to_string())
}
#[export]
pub fn language_ids() -> Vec<(String, String)> {
vec![("JSON".into(), "jsonc".into())]
}

10
plugins/test_plugin/Cargo.toml
View File

@ -1,10 +0,0 @@
[package]
name = "test_plugin"
version = "0.1.0"
edition = "2021"
[dependencies]
plugin = { path = "../../crates/plugin" }
[lib]
crate-type = ["cdylib"]

82
plugins/test_plugin/src/lib.rs
View File

@ -1,82 +0,0 @@
use plugin::prelude::*;
#[export]
pub fn noop() {}
#[export]
pub fn constant() -> u32 {
27
}
#[export]
pub fn identity(i: u32) -> u32 {
i
}
#[export]
pub fn add(a: u32, b: u32) -> u32 {
a + b
}
#[export]
pub fn swap(a: u32, b: u32) -> (u32, u32) {
(b, a)
}
#[export]
pub fn sort(mut list: Vec<u32>) -> Vec<u32> {
list.sort();
list
}
#[export]
pub fn print(string: String) {
println!("to stdout: {}", string);
eprintln!("to stderr: {}", string);
}
#[import]
fn mystery_number(input: u32) -> u32;
#[export]
pub fn and_back(secret: u32) -> u32 {
mystery_number(secret)
}
#[import]
fn import_noop() -> ();
#[import]
fn import_identity(i: u32) -> u32;
#[import]
fn import_swap(a: u32, b: u32) -> (u32, u32);
#[export]
pub fn imports(x: u32) -> u32 {
let a = import_identity(7);
import_noop();
let (b, c) = import_swap(a, x);
assert_eq!(a, c);
assert_eq!(x, b);
a + b // should be 7 + x
}
#[import]
fn import_half(a: u32) -> u32;
#[export]
pub fn half_async(a: u32) -> u32 {
import_half(a)
}
#[import]
fn command_async(command: String) -> Option<Vec<u8>>;
#[export]
pub fn echo_async(message: String) -> String {
let command = format!("echo {}", message);
let result = command_async(command);
let result = result.expect("Could not run command");
String::from_utf8_lossy(&result).to_string()
}