From 868616d62ea0151a14fe9ece72c7d0ab5f52d8a0 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Tue, 19 Mar 2024 07:50:21 -0700 Subject: [PATCH] Introduce `extension-cli` binary, for packaging extensions in CI (#9523) This will be used in the [extensions](https://github.com/zed-industries/extensions) repository for packaging the extensions that users submit. Release Notes: - N/A --------- Co-authored-by: Marshall Bowers --- Cargo.lock | 18 ++ Cargo.toml | 2 + crates/collab/Cargo.toml | 2 +- ...uild_extension.rs => extension_builder.rs} | 31 ++-- crates/extension/src/extension_store.rs | 11 +- crates/extension_cli/Cargo.toml | 23 +++ crates/extension_cli/LICENSE-GPL | 1 + crates/extension_cli/src/main.rs | 159 ++++++++++++++++++ 8 files changed, 229 insertions(+), 18 deletions(-) rename crates/extension/src/{build_extension.rs => extension_builder.rs} (95%) create mode 100644 crates/extension_cli/Cargo.toml create mode 120000 crates/extension_cli/LICENSE-GPL create mode 100644 crates/extension_cli/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index b6141435cc..9112dc4c10 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3514,6 +3514,24 @@ dependencies = [ "wit-component 0.20.3", ] +[[package]] +name = "extension_cli" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap 4.4.4", + "env_logger", + "extension", + "fs", + "language", + "log", + "theme", + "tokio", + "toml 0.8.10", + "tree-sitter", + "wasmtime", +] + [[package]] name = "extensions_ui" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 1dc3c6cbfd..ff1fe93e4f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,6 +24,7 @@ members = [ "crates/editor", "crates/extension", "crates/extension_api", + "crates/extension_cli", "crates/extensions_ui", "crates/feature_flags", "crates/feedback", @@ -273,6 +274,7 @@ time = { version = "0.3", features = [ "formatting", ] } toml = "0.8" +tokio = { version = "1", features = ["full"] } tower-http = "0.4.4" tree-sitter = { version = "0.20", features = ["wasm"] } tree-sitter-astro = { git = "https://github.com/virchau13/tree-sitter-astro.git", rev = "e924787e12e8a03194f36a113290ac11d6dc10f3" } diff --git a/crates/collab/Cargo.toml b/crates/collab/Cargo.toml index 0f4b706b5a..49cdcebbff 100644 --- a/crates/collab/Cargo.toml +++ b/crates/collab/Cargo.toml @@ -54,7 +54,7 @@ rustc-demangle.workspace = true telemetry_events.workspace = true text.workspace = true time.workspace = true -tokio = { version = "1", features = ["full"] } +tokio.workspace = true toml.workspace = true tower = "0.4" tower-http = { workspace = true, features = ["trace"] } diff --git a/crates/extension/src/build_extension.rs b/crates/extension/src/extension_builder.rs similarity index 95% rename from crates/extension/src/build_extension.rs rename to crates/extension/src/extension_builder.rs index 0e1408d886..2de6c1057c 100644 --- a/crates/extension/src/build_extension.rs +++ b/crates/extension/src/extension_builder.rs @@ -6,9 +6,8 @@ use async_tar::Archive; use futures::io::BufReader; use futures::AsyncReadExt; use serde::Deserialize; -use std::mem; use std::{ - env, fs, + env, fs, mem, path::{Path, PathBuf}, process::{Command, Stdio}, sync::Arc, @@ -72,22 +71,27 @@ impl ExtensionBuilder { pub async fn compile_extension( &self, extension_dir: &Path, + extension_manifest: &ExtensionManifest, options: CompileExtensionOptions, ) -> Result<()> { + if extension_dir.is_relative() { + bail!( + "extension dir {} is not an absolute path", + extension_dir.display() + ); + } + fs::create_dir_all(&self.cache_dir)?; - let extension_toml_path = extension_dir.join("extension.toml"); - let extension_toml_content = fs::read_to_string(&extension_toml_path)?; - let extension_toml: ExtensionManifest = toml::from_str(&extension_toml_content)?; let cargo_toml_path = extension_dir.join("Cargo.toml"); - if extension_toml.lib.kind == Some(ExtensionLibraryKind::Rust) + if extension_manifest.lib.kind == Some(ExtensionLibraryKind::Rust) || fs::metadata(&cargo_toml_path)?.is_file() { self.compile_rust_extension(extension_dir, options).await?; } - for (grammar_name, grammar_metadata) in extension_toml.grammars { - self.compile_grammar(extension_dir, grammar_name, grammar_metadata) + for (grammar_name, grammar_metadata) in &extension_manifest.grammars { + self.compile_grammar(extension_dir, grammar_name.as_ref(), grammar_metadata) .await?; } @@ -157,13 +161,13 @@ impl ExtensionBuilder { async fn compile_grammar( &self, extension_dir: &Path, - grammar_name: Arc, - grammar_metadata: GrammarManifestEntry, + grammar_name: &str, + grammar_metadata: &GrammarManifestEntry, ) -> Result<()> { let clang_path = self.install_wasi_sdk_if_needed().await?; let mut grammar_repo_dir = extension_dir.to_path_buf(); - grammar_repo_dir.extend(["grammars", grammar_name.as_ref()]); + grammar_repo_dir.extend(["grammars", grammar_name]); let mut grammar_wasm_path = grammar_repo_dir.clone(); grammar_wasm_path.set_extension("wasm"); @@ -277,9 +281,10 @@ impl ExtensionBuilder { ); } bail!( - "failed to checkout revision {} in directory '{}'", + "failed to checkout revision {} in directory '{}': {}", rev, - directory.display() + directory.display(), + String::from_utf8_lossy(&checkout_output.stderr) ); } diff --git a/crates/extension/src/extension_store.rs b/crates/extension/src/extension_store.rs index 5b53b509a7..cb7190eb4d 100644 --- a/crates/extension/src/extension_store.rs +++ b/crates/extension/src/extension_store.rs @@ -1,4 +1,4 @@ -mod build_extension; +pub mod extension_builder; mod extension_lsp_adapter; mod extension_manifest; mod wasm_host; @@ -10,8 +10,8 @@ use crate::{extension_lsp_adapter::ExtensionLspAdapter, wasm_host::wit}; use anyhow::{anyhow, bail, Context as _, Result}; use async_compression::futures::bufread::GzipDecoder; use async_tar::Archive; -use build_extension::{CompileExtensionOptions, ExtensionBuilder}; use collections::{hash_map, BTreeMap, HashMap, HashSet}; +use extension_builder::{CompileExtensionOptions, ExtensionBuilder}; use extension_manifest::ExtensionLibraryKind; use fs::{Fs, RemoveOptions}; use futures::{ @@ -545,6 +545,7 @@ impl ExtensionStore { builder .compile_extension( &extension_source_path, + &extension_manifest, CompileExtensionOptions { release: false }, ) .await @@ -580,6 +581,7 @@ impl ExtensionStore { pub fn rebuild_dev_extension(&mut self, extension_id: Arc, cx: &mut ModelContext) { let path = self.installed_dir.join(extension_id.as_ref()); let builder = self.builder.clone(); + let fs = self.fs.clone(); match self.outstanding_operations.entry(extension_id.clone()) { hash_map::Entry::Occupied(_) => return, @@ -588,8 +590,9 @@ impl ExtensionStore { cx.notify(); let compile = cx.background_executor().spawn(async move { + let manifest = Self::load_extension_manifest(fs, &path).await?; builder - .compile_extension(&path, CompileExtensionOptions { release: true }) + .compile_extension(&path, &manifest, CompileExtensionOptions { release: true }) .await }); @@ -1000,7 +1003,7 @@ impl ExtensionStore { Ok(()) } - async fn load_extension_manifest( + pub async fn load_extension_manifest( fs: Arc, extension_dir: &Path, ) -> Result { diff --git a/crates/extension_cli/Cargo.toml b/crates/extension_cli/Cargo.toml new file mode 100644 index 0000000000..74e4d115f3 --- /dev/null +++ b/crates/extension_cli/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "extension_cli" +version = "0.1.0" +edition = "2021" +publish = false +license = "GPL-3.0-or-later" + +[lints] +workspace = true + +[dependencies] +anyhow.workspace = true +clap = { workspace = true, features = ["derive"] } +env_logger.workspace = true +fs.workspace = true +extension.workspace = true +language.workspace = true +log.workspace = true +theme.workspace = true +tokio.workspace = true +toml.workspace = true +tree-sitter.workspace = true +wasmtime.workspace = true diff --git a/crates/extension_cli/LICENSE-GPL b/crates/extension_cli/LICENSE-GPL new file mode 120000 index 0000000000..89e542f750 --- /dev/null +++ b/crates/extension_cli/LICENSE-GPL @@ -0,0 +1 @@ +../../LICENSE-GPL \ No newline at end of file diff --git a/crates/extension_cli/src/main.rs b/crates/extension_cli/src/main.rs new file mode 100644 index 0000000000..608aeaffd0 --- /dev/null +++ b/crates/extension_cli/src/main.rs @@ -0,0 +1,159 @@ +use std::{ + collections::HashMap, + fs, + path::{Path, PathBuf}, + sync::Arc, +}; + +use ::fs::{Fs, RealFs}; +use anyhow::{anyhow, Context, Result}; +use clap::Parser; +use extension::{ + extension_builder::{CompileExtensionOptions, ExtensionBuilder}, + ExtensionStore, +}; +use language::LanguageConfig; +use theme::ThemeRegistry; +use tree_sitter::{Language, Query, WasmStore}; + +#[derive(Parser, Debug)] +#[command(name = "zed-extension")] +struct Args { + /// The path to the extension directory + extension_path: PathBuf, + /// Whether to compile with optimizations + #[arg(long)] + release: bool, + /// The path to a directory where build dependencies are downloaded + #[arg(long)] + scratch_dir: PathBuf, +} + +#[tokio::main] +async fn main() -> Result<()> { + env_logger::init(); + + let args = Args::parse(); + let fs = Arc::new(RealFs); + let engine = wasmtime::Engine::default(); + let mut wasm_store = WasmStore::new(engine)?; + + let extension_path = args + .extension_path + .canonicalize() + .context("can't canonicalize extension_path")?; + let scratch_dir = args + .scratch_dir + .canonicalize() + .context("can't canonicalize scratch_dir")?; + + let manifest = ExtensionStore::load_extension_manifest(fs.clone(), &extension_path).await?; + let builder = ExtensionBuilder::new(scratch_dir); + builder + .compile_extension( + &extension_path, + &manifest, + CompileExtensionOptions { + release: args.release, + }, + ) + .await?; + + let grammars = test_grammars(&extension_path, &mut wasm_store)?; + test_languages(&extension_path, &grammars)?; + test_themes(&extension_path, fs.clone()).await?; + + Ok(()) +} + +fn test_grammars( + extension_path: &Path, + wasm_store: &mut WasmStore, +) -> Result> { + let mut grammars = HashMap::default(); + let grammars_dir = extension_path.join("grammars"); + if !grammars_dir.exists() { + return Ok(grammars); + } + + let entries = fs::read_dir(&grammars_dir)?; + for entry in entries { + let entry = entry?; + let grammar_path = entry.path(); + let grammar_name = grammar_path.file_stem().unwrap().to_str().unwrap(); + if grammar_path.extension() == Some("wasm".as_ref()) { + let wasm = fs::read(&grammar_path)?; + let language = wasm_store.load_language(grammar_name, &wasm)?; + log::info!("loaded grammar {grammar_name}"); + grammars.insert(grammar_name.into(), language); + } + } + + Ok(grammars) +} + +fn test_languages(extension_path: &Path, grammars: &HashMap) -> Result<()> { + let languages_dir = extension_path.join("languages"); + if !languages_dir.exists() { + return Ok(()); + } + + let entries = fs::read_dir(&languages_dir)?; + for entry in entries { + let entry = entry?; + let language_dir = entry.path(); + let config_path = language_dir.join("config.toml"); + let config_content = fs::read_to_string(&config_path)?; + let config: LanguageConfig = toml::from_str(&config_content)?; + let grammar = if let Some(name) = &config.grammar { + Some( + grammars + .get(name.as_ref()) + .ok_or_else(|| anyhow!("language"))?, + ) + } else { + None + }; + + let query_entries = fs::read_dir(&language_dir)?; + for entry in query_entries { + let entry = entry?; + let query_path = entry.path(); + if query_path.extension() == Some("scm".as_ref()) { + let grammar = grammar.ok_or_else(|| { + anyhow!( + "language {} provides query {} but no grammar", + config.name, + query_path.display() + ) + })?; + + let query_source = fs::read_to_string(&query_path)?; + let _query = Query::new(grammar, &query_source)?; + } + } + + log::info!("loaded language {}", config.name); + } + + Ok(()) +} + +async fn test_themes(extension_path: &Path, fs: Arc) -> Result<()> { + let themes_dir = extension_path.join("themes"); + if !themes_dir.exists() { + return Ok(()); + } + + let entries = fs::read_dir(&themes_dir)?; + for entry in entries { + let entry = entry?; + let theme_path = entry.path(); + if theme_path.extension() == Some("json".as_ref()) { + let theme_family = ThemeRegistry::read_user_theme(&entry.path(), fs.clone()).await?; + log::info!("loaded theme family {}", theme_family.name); + } + } + + Ok(()) +}