From fd11bd68f2f4a0ee3cf53e52a270fcc6d70939f1 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Tue, 19 Mar 2024 14:26:06 -0700 Subject: [PATCH] Perform extension packaging in extension-cli (#9549) Release Notes: - N/A --------- Co-authored-by: Marshall --- Cargo.lock | 3 + crates/extension/src/extension_store.rs | 5 +- crates/extension_cli/Cargo.toml | 7 +- crates/extension_cli/src/main.rs | 254 ++++++++++++++++++------ crates/rpc/src/extension.rs | 10 + crates/rpc/src/rpc.rs | 2 + extensions/gleam/extension.toml | 1 + extensions/uiua/extension.toml | 1 + 8 files changed, 221 insertions(+), 62 deletions(-) create mode 100644 crates/rpc/src/extension.rs diff --git a/Cargo.lock b/Cargo.lock index 1e80a494ce..cb7d0e7ad6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3501,6 +3501,9 @@ dependencies = [ "fs", "language", "log", + "rpc", + "serde", + "serde_json", "theme", "tokio", "toml 0.8.10", diff --git a/crates/extension/src/extension_store.rs b/crates/extension/src/extension_store.rs index 1b80c251ea..e627d8b9f3 100644 --- a/crates/extension/src/extension_store.rs +++ b/crates/extension/src/extension_store.rs @@ -12,7 +12,6 @@ use async_compression::futures::bufread::GzipDecoder; use async_tar::Archive; use collections::{hash_map, BTreeMap, HashMap, HashSet}; use extension_builder::{CompileExtensionOptions, ExtensionBuilder}; -use extension_manifest::ExtensionLibraryKind; use fs::{Fs, RemoveOptions}; use futures::{ channel::{ @@ -43,7 +42,9 @@ use util::{ }; use wasm_host::{WasmExtension, WasmHost}; -pub use extension_manifest::{ExtensionManifest, GrammarManifestEntry, OldExtensionManifest}; +pub use extension_manifest::{ + ExtensionLibraryKind, ExtensionManifest, GrammarManifestEntry, OldExtensionManifest, +}; const RELOAD_DEBOUNCE_DURATION: Duration = Duration::from_millis(200); const FS_WATCH_LATENCY: Duration = Duration::from_millis(100); diff --git a/crates/extension_cli/Cargo.toml b/crates/extension_cli/Cargo.toml index 5012494eac..1b56a7e87d 100644 --- a/crates/extension_cli/Cargo.toml +++ b/crates/extension_cli/Cargo.toml @@ -8,10 +8,6 @@ license = "GPL-3.0-or-later" [lints] workspace = true -[[bin]] -name = "zed-extension" -path = "src/main.rs" - [dependencies] anyhow.workspace = true clap = { workspace = true, features = ["derive"] } @@ -20,6 +16,9 @@ fs.workspace = true extension.workspace = true language.workspace = true log.workspace = true +rpc.workspace = true +serde.workspace = true +serde_json.workspace = true theme.workspace = true tokio.workspace = true toml.workspace = true diff --git a/crates/extension_cli/src/main.rs b/crates/extension_cli/src/main.rs index 608aeaffd0..b3884dce9c 100644 --- a/crates/extension_cli/src/main.rs +++ b/crates/extension_cli/src/main.rs @@ -1,16 +1,17 @@ use std::{ collections::HashMap, - fs, + env, fs, path::{Path, PathBuf}, + process::Command, sync::Arc, }; -use ::fs::{Fs, RealFs}; -use anyhow::{anyhow, Context, Result}; +use ::fs::{copy_recursive, CopyOptions, Fs, RealFs}; +use anyhow::{anyhow, bail, Context, Result}; use clap::Parser; use extension::{ extension_builder::{CompileExtensionOptions, ExtensionBuilder}, - ExtensionStore, + ExtensionLibraryKind, ExtensionManifest, ExtensionStore, }; use language::LanguageConfig; use theme::ThemeRegistry; @@ -20,10 +21,11 @@ use tree_sitter::{Language, Query, WasmStore}; #[command(name = "zed-extension")] struct Args { /// The path to the extension directory - extension_path: PathBuf, - /// Whether to compile with optimizations #[arg(long)] - release: bool, + source_dir: PathBuf, + /// The output directory to place the packaged extension. + #[arg(long)] + output_dir: PathBuf, /// The path to a directory where build dependencies are downloaded #[arg(long)] scratch_dir: PathBuf, @@ -39,69 +41,214 @@ async fn main() -> Result<()> { let mut wasm_store = WasmStore::new(engine)?; let extension_path = args - .extension_path + .source_dir .canonicalize() - .context("can't canonicalize extension_path")?; + .context("failed to canonicalize source_dir")?; let scratch_dir = args .scratch_dir .canonicalize() - .context("can't canonicalize scratch_dir")?; + .context("failed to canonicalize scratch_dir")?; + let output_dir = if args.output_dir.is_relative() { + env::current_dir()?.join(&args.output_dir) + } else { + args.output_dir + }; + + let mut manifest = ExtensionStore::load_extension_manifest(fs.clone(), &extension_path).await?; + populate_default_paths(&mut manifest, &extension_path)?; - let manifest = ExtensionStore::load_extension_manifest(fs.clone(), &extension_path).await?; let builder = ExtensionBuilder::new(scratch_dir); builder .compile_extension( &extension_path, &manifest, - CompileExtensionOptions { - release: args.release, - }, + CompileExtensionOptions { release: true }, ) - .await?; + .await + .context("failed to compile extension")?; - let grammars = test_grammars(&extension_path, &mut wasm_store)?; - test_languages(&extension_path, &grammars)?; - test_themes(&extension_path, fs.clone()).await?; + let grammars = test_grammars(&manifest, &extension_path, &mut wasm_store)?; + test_languages(&manifest, &extension_path, &grammars)?; + test_themes(&manifest, &extension_path, fs.clone()).await?; + + let archive_dir = output_dir.join("archive"); + fs::remove_dir_all(&archive_dir).ok(); + copy_extension_resources(&manifest, &extension_path, &archive_dir, fs.clone()) + .await + .context("failed to copy extension resources")?; + + let tar_output = Command::new("tar") + .current_dir(&output_dir) + .args(&["-czvf", "archive.tar.gz", "-C", "archive", "."]) + .output() + .context("failed to run tar")?; + if !tar_output.status.success() { + bail!( + "failed to create archive.tar.gz: {}", + String::from_utf8_lossy(&tar_output.stderr) + ); + } + + let manifest_json = serde_json::to_string(&rpc::ExtensionApiManifest { + name: manifest.name, + version: manifest.version.to_string(), + description: manifest.description, + authors: manifest.authors, + repository: manifest + .repository + .ok_or_else(|| anyhow!("missing repository in extension manifest"))?, + })?; + fs::remove_dir_all(&archive_dir)?; + fs::write(output_dir.join("manifest.json"), manifest_json.as_bytes())?; + + Ok(()) +} + +fn populate_default_paths(manifest: &mut ExtensionManifest, extension_path: &Path) -> Result<()> { + let cargo_toml_path = extension_path.join("Cargo.toml"); + if cargo_toml_path.exists() { + manifest.lib.kind = Some(ExtensionLibraryKind::Rust); + } + + let languages_dir = extension_path.join("languages"); + if languages_dir.exists() { + for entry in fs::read_dir(&languages_dir).context("failed to list languages dir")? { + let entry = entry?; + let language_dir = entry.path(); + let config_path = language_dir.join("config.toml"); + if config_path.exists() { + let relative_language_dir = + language_dir.strip_prefix(extension_path)?.to_path_buf(); + if !manifest.languages.contains(&relative_language_dir) { + manifest.languages.push(relative_language_dir); + } + } + } + } + + let themes_dir = extension_path.join("themes"); + if themes_dir.exists() { + for entry in fs::read_dir(&themes_dir).context("failed to list themes dir")? { + let entry = entry?; + let theme_path = entry.path(); + if theme_path.extension() == Some("json".as_ref()) { + let relative_theme_path = theme_path.strip_prefix(extension_path)?.to_path_buf(); + if !manifest.themes.contains(&relative_theme_path) { + manifest.themes.push(relative_theme_path); + } + } + } + } + + Ok(()) +} + +async fn copy_extension_resources( + manifest: &ExtensionManifest, + extension_path: &Path, + output_dir: &Path, + fs: Arc, +) -> Result<()> { + fs::create_dir_all(&output_dir).context("failed to create output dir")?; + + let manifest_toml = toml::to_string(&manifest).context("failed to serialize manifest")?; + fs::write(output_dir.join("extension.toml"), &manifest_toml) + .context("failed to write extension.toml")?; + + if manifest.lib.kind.is_some() { + fs::copy( + extension_path.join("extension.wasm"), + output_dir.join("extension.wasm"), + ) + .context("failed to copy extension.wasm")?; + } + + if !manifest.grammars.is_empty() { + let source_grammars_dir = extension_path.join("grammars"); + let output_grammars_dir = output_dir.join("grammars"); + fs::create_dir_all(&output_grammars_dir)?; + for grammar_name in manifest.grammars.keys() { + let mut grammar_filename = PathBuf::from(grammar_name.as_ref()); + grammar_filename.set_extension("wasm"); + fs::copy( + &source_grammars_dir.join(&grammar_filename), + &output_grammars_dir.join(&grammar_filename), + ) + .with_context(|| format!("failed to copy grammar '{}'", grammar_filename.display()))?; + } + } + + if !manifest.themes.is_empty() { + let output_themes_dir = output_dir.join("themes"); + fs::create_dir_all(&output_themes_dir)?; + for theme_path in &manifest.themes { + fs::copy( + extension_path.join(theme_path), + output_themes_dir.join( + theme_path + .file_name() + .ok_or_else(|| anyhow!("invalid theme path"))?, + ), + ) + .with_context(|| format!("failed to copy theme '{}'", theme_path.display()))?; + } + } + + if !manifest.languages.is_empty() { + let output_languages_dir = output_dir.join("languages"); + fs::create_dir_all(&output_languages_dir)?; + for language_path in &manifest.languages { + copy_recursive( + fs.as_ref(), + &extension_path.join(language_path), + &output_languages_dir.join( + language_path + .file_name() + .ok_or_else(|| anyhow!("invalid language path"))?, + ), + CopyOptions { + overwrite: true, + ignore_if_exists: false, + }, + ) + .await + .with_context(|| { + format!("failed to copy language dir '{}'", language_path.display()) + })?; + } + } Ok(()) } fn test_grammars( + manifest: &ExtensionManifest, extension_path: &Path, wasm_store: &mut WasmStore, ) -> Result> { let mut grammars = HashMap::default(); let grammars_dir = extension_path.join("grammars"); - if !grammars_dir.exists() { - return Ok(grammars); - } - let entries = fs::read_dir(&grammars_dir)?; - for entry in entries { - let entry = entry?; - let grammar_path = entry.path(); - let grammar_name = grammar_path.file_stem().unwrap().to_str().unwrap(); - if grammar_path.extension() == Some("wasm".as_ref()) { - let wasm = fs::read(&grammar_path)?; - let language = wasm_store.load_language(grammar_name, &wasm)?; - log::info!("loaded grammar {grammar_name}"); - grammars.insert(grammar_name.into(), language); - } + for grammar_name in manifest.grammars.keys() { + let mut grammar_path = grammars_dir.join(grammar_name.as_ref()); + grammar_path.set_extension("wasm"); + + let wasm = fs::read(&grammar_path)?; + let language = wasm_store.load_language(grammar_name, &wasm)?; + log::info!("loaded grammar {grammar_name}"); + grammars.insert(grammar_name.to_string(), language); } Ok(grammars) } -fn test_languages(extension_path: &Path, grammars: &HashMap) -> Result<()> { - let languages_dir = extension_path.join("languages"); - if !languages_dir.exists() { - return Ok(()); - } - - let entries = fs::read_dir(&languages_dir)?; - for entry in entries { - let entry = entry?; - let language_dir = entry.path(); +fn test_languages( + manifest: &ExtensionManifest, + extension_path: &Path, + grammars: &HashMap, +) -> Result<()> { + for relative_language_dir in &manifest.languages { + let language_dir = extension_path.join(relative_language_dir); let config_path = language_dir.join("config.toml"); let config_content = fs::read_to_string(&config_path)?; let config: LanguageConfig = toml::from_str(&config_content)?; @@ -139,20 +286,15 @@ fn test_languages(extension_path: &Path, grammars: &HashMap) - Ok(()) } -async fn test_themes(extension_path: &Path, fs: Arc) -> Result<()> { - let themes_dir = extension_path.join("themes"); - if !themes_dir.exists() { - return Ok(()); - } - - let entries = fs::read_dir(&themes_dir)?; - for entry in entries { - let entry = entry?; - let theme_path = entry.path(); - if theme_path.extension() == Some("json".as_ref()) { - let theme_family = ThemeRegistry::read_user_theme(&entry.path(), fs.clone()).await?; - log::info!("loaded theme family {}", theme_family.name); - } +async fn test_themes( + manifest: &ExtensionManifest, + extension_path: &Path, + fs: Arc, +) -> Result<()> { + for relative_theme_path in &manifest.themes { + let theme_path = extension_path.join(relative_theme_path); + let theme_family = ThemeRegistry::read_user_theme(&theme_path, fs.clone()).await?; + log::info!("loaded theme family {}", theme_family.name); } Ok(()) diff --git a/crates/rpc/src/extension.rs b/crates/rpc/src/extension.rs new file mode 100644 index 0000000000..59176b976f --- /dev/null +++ b/crates/rpc/src/extension.rs @@ -0,0 +1,10 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize)] +pub struct ExtensionApiManifest { + pub name: String, + pub version: String, + pub description: Option, + pub authors: Vec, + pub repository: String, +} diff --git a/crates/rpc/src/rpc.rs b/crates/rpc/src/rpc.rs index 2beb7614e8..880102e8d3 100644 --- a/crates/rpc/src/rpc.rs +++ b/crates/rpc/src/rpc.rs @@ -1,12 +1,14 @@ pub mod auth; mod conn; mod error; +mod extension; mod notification; mod peer; pub mod proto; pub use conn::Connection; pub use error::*; +pub use extension::*; pub use notification::*; pub use peer::*; mod macros; diff --git a/extensions/gleam/extension.toml b/extensions/gleam/extension.toml index 76cce0f291..76684f9504 100644 --- a/extensions/gleam/extension.toml +++ b/extensions/gleam/extension.toml @@ -3,6 +3,7 @@ name = "Gleam" description = "Gleam support for Zed" version = "0.0.1" authors = ["Marshall Bowers "] +repository = "https://github.com/zed-industries/zed" [language_servers.gleam] name = "Gleam LSP" diff --git a/extensions/uiua/extension.toml b/extensions/uiua/extension.toml index e8448c425c..9241483ac2 100644 --- a/extensions/uiua/extension.toml +++ b/extensions/uiua/extension.toml @@ -3,6 +3,7 @@ name = "Uiua" description = "Uiua support for Zed" version = "0.0.1" authors = ["Max Brunsfeld "] +repository = "https://github.com/zed-industries/zed" [language_servers.uiua] name = "Uiua LSP"