Perform extension packaging in extension-cli (#9549)

Release Notes:

- N/A

---------

Co-authored-by: Marshall <marshall@zed.dev>
This commit is contained in:
Max Brunsfeld 2024-03-19 14:26:06 -07:00 committed by GitHub
parent 85c294da9a
commit fd11bd68f2
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 221 additions and 62 deletions

3
Cargo.lock generated
View File

@ -3501,6 +3501,9 @@ dependencies = [
"fs",
"language",
"log",
"rpc",
"serde",
"serde_json",
"theme",
"tokio",
"toml 0.8.10",

View File

@ -12,7 +12,6 @@ use async_compression::futures::bufread::GzipDecoder;
use async_tar::Archive;
use collections::{hash_map, BTreeMap, HashMap, HashSet};
use extension_builder::{CompileExtensionOptions, ExtensionBuilder};
use extension_manifest::ExtensionLibraryKind;
use fs::{Fs, RemoveOptions};
use futures::{
channel::{
@ -43,7 +42,9 @@ use util::{
};
use wasm_host::{WasmExtension, WasmHost};
pub use extension_manifest::{ExtensionManifest, GrammarManifestEntry, OldExtensionManifest};
pub use extension_manifest::{
ExtensionLibraryKind, ExtensionManifest, GrammarManifestEntry, OldExtensionManifest,
};
const RELOAD_DEBOUNCE_DURATION: Duration = Duration::from_millis(200);
const FS_WATCH_LATENCY: Duration = Duration::from_millis(100);

View File

@ -8,10 +8,6 @@ license = "GPL-3.0-or-later"
[lints]
workspace = true
[[bin]]
name = "zed-extension"
path = "src/main.rs"
[dependencies]
anyhow.workspace = true
clap = { workspace = true, features = ["derive"] }
@ -20,6 +16,9 @@ fs.workspace = true
extension.workspace = true
language.workspace = true
log.workspace = true
rpc.workspace = true
serde.workspace = true
serde_json.workspace = true
theme.workspace = true
tokio.workspace = true
toml.workspace = true

View File

@ -1,16 +1,17 @@
use std::{
collections::HashMap,
fs,
env, fs,
path::{Path, PathBuf},
process::Command,
sync::Arc,
};
use ::fs::{Fs, RealFs};
use anyhow::{anyhow, Context, Result};
use ::fs::{copy_recursive, CopyOptions, Fs, RealFs};
use anyhow::{anyhow, bail, Context, Result};
use clap::Parser;
use extension::{
extension_builder::{CompileExtensionOptions, ExtensionBuilder},
ExtensionStore,
ExtensionLibraryKind, ExtensionManifest, ExtensionStore,
};
use language::LanguageConfig;
use theme::ThemeRegistry;
@ -20,10 +21,11 @@ use tree_sitter::{Language, Query, WasmStore};
#[command(name = "zed-extension")]
struct Args {
/// The path to the extension directory
extension_path: PathBuf,
/// Whether to compile with optimizations
#[arg(long)]
release: bool,
source_dir: PathBuf,
/// The output directory to place the packaged extension.
#[arg(long)]
output_dir: PathBuf,
/// The path to a directory where build dependencies are downloaded
#[arg(long)]
scratch_dir: PathBuf,
@ -39,69 +41,214 @@ async fn main() -> Result<()> {
let mut wasm_store = WasmStore::new(engine)?;
let extension_path = args
.extension_path
.source_dir
.canonicalize()
.context("can't canonicalize extension_path")?;
.context("failed to canonicalize source_dir")?;
let scratch_dir = args
.scratch_dir
.canonicalize()
.context("can't canonicalize scratch_dir")?;
.context("failed to canonicalize scratch_dir")?;
let output_dir = if args.output_dir.is_relative() {
env::current_dir()?.join(&args.output_dir)
} else {
args.output_dir
};
let mut manifest = ExtensionStore::load_extension_manifest(fs.clone(), &extension_path).await?;
populate_default_paths(&mut manifest, &extension_path)?;
let manifest = ExtensionStore::load_extension_manifest(fs.clone(), &extension_path).await?;
let builder = ExtensionBuilder::new(scratch_dir);
builder
.compile_extension(
&extension_path,
&manifest,
CompileExtensionOptions {
release: args.release,
CompileExtensionOptions { release: true },
)
.await
.context("failed to compile extension")?;
let grammars = test_grammars(&manifest, &extension_path, &mut wasm_store)?;
test_languages(&manifest, &extension_path, &grammars)?;
test_themes(&manifest, &extension_path, fs.clone()).await?;
let archive_dir = output_dir.join("archive");
fs::remove_dir_all(&archive_dir).ok();
copy_extension_resources(&manifest, &extension_path, &archive_dir, fs.clone())
.await
.context("failed to copy extension resources")?;
let tar_output = Command::new("tar")
.current_dir(&output_dir)
.args(&["-czvf", "archive.tar.gz", "-C", "archive", "."])
.output()
.context("failed to run tar")?;
if !tar_output.status.success() {
bail!(
"failed to create archive.tar.gz: {}",
String::from_utf8_lossy(&tar_output.stderr)
);
}
let manifest_json = serde_json::to_string(&rpc::ExtensionApiManifest {
name: manifest.name,
version: manifest.version.to_string(),
description: manifest.description,
authors: manifest.authors,
repository: manifest
.repository
.ok_or_else(|| anyhow!("missing repository in extension manifest"))?,
})?;
fs::remove_dir_all(&archive_dir)?;
fs::write(output_dir.join("manifest.json"), manifest_json.as_bytes())?;
Ok(())
}
fn populate_default_paths(manifest: &mut ExtensionManifest, extension_path: &Path) -> Result<()> {
let cargo_toml_path = extension_path.join("Cargo.toml");
if cargo_toml_path.exists() {
manifest.lib.kind = Some(ExtensionLibraryKind::Rust);
}
let languages_dir = extension_path.join("languages");
if languages_dir.exists() {
for entry in fs::read_dir(&languages_dir).context("failed to list languages dir")? {
let entry = entry?;
let language_dir = entry.path();
let config_path = language_dir.join("config.toml");
if config_path.exists() {
let relative_language_dir =
language_dir.strip_prefix(extension_path)?.to_path_buf();
if !manifest.languages.contains(&relative_language_dir) {
manifest.languages.push(relative_language_dir);
}
}
}
}
let themes_dir = extension_path.join("themes");
if themes_dir.exists() {
for entry in fs::read_dir(&themes_dir).context("failed to list themes dir")? {
let entry = entry?;
let theme_path = entry.path();
if theme_path.extension() == Some("json".as_ref()) {
let relative_theme_path = theme_path.strip_prefix(extension_path)?.to_path_buf();
if !manifest.themes.contains(&relative_theme_path) {
manifest.themes.push(relative_theme_path);
}
}
}
}
Ok(())
}
async fn copy_extension_resources(
manifest: &ExtensionManifest,
extension_path: &Path,
output_dir: &Path,
fs: Arc<dyn Fs>,
) -> Result<()> {
fs::create_dir_all(&output_dir).context("failed to create output dir")?;
let manifest_toml = toml::to_string(&manifest).context("failed to serialize manifest")?;
fs::write(output_dir.join("extension.toml"), &manifest_toml)
.context("failed to write extension.toml")?;
if manifest.lib.kind.is_some() {
fs::copy(
extension_path.join("extension.wasm"),
output_dir.join("extension.wasm"),
)
.context("failed to copy extension.wasm")?;
}
if !manifest.grammars.is_empty() {
let source_grammars_dir = extension_path.join("grammars");
let output_grammars_dir = output_dir.join("grammars");
fs::create_dir_all(&output_grammars_dir)?;
for grammar_name in manifest.grammars.keys() {
let mut grammar_filename = PathBuf::from(grammar_name.as_ref());
grammar_filename.set_extension("wasm");
fs::copy(
&source_grammars_dir.join(&grammar_filename),
&output_grammars_dir.join(&grammar_filename),
)
.with_context(|| format!("failed to copy grammar '{}'", grammar_filename.display()))?;
}
}
if !manifest.themes.is_empty() {
let output_themes_dir = output_dir.join("themes");
fs::create_dir_all(&output_themes_dir)?;
for theme_path in &manifest.themes {
fs::copy(
extension_path.join(theme_path),
output_themes_dir.join(
theme_path
.file_name()
.ok_or_else(|| anyhow!("invalid theme path"))?,
),
)
.with_context(|| format!("failed to copy theme '{}'", theme_path.display()))?;
}
}
if !manifest.languages.is_empty() {
let output_languages_dir = output_dir.join("languages");
fs::create_dir_all(&output_languages_dir)?;
for language_path in &manifest.languages {
copy_recursive(
fs.as_ref(),
&extension_path.join(language_path),
&output_languages_dir.join(
language_path
.file_name()
.ok_or_else(|| anyhow!("invalid language path"))?,
),
CopyOptions {
overwrite: true,
ignore_if_exists: false,
},
)
.await?;
let grammars = test_grammars(&extension_path, &mut wasm_store)?;
test_languages(&extension_path, &grammars)?;
test_themes(&extension_path, fs.clone()).await?;
.await
.with_context(|| {
format!("failed to copy language dir '{}'", language_path.display())
})?;
}
}
Ok(())
}
fn test_grammars(
manifest: &ExtensionManifest,
extension_path: &Path,
wasm_store: &mut WasmStore,
) -> Result<HashMap<String, Language>> {
let mut grammars = HashMap::default();
let grammars_dir = extension_path.join("grammars");
if !grammars_dir.exists() {
return Ok(grammars);
}
let entries = fs::read_dir(&grammars_dir)?;
for entry in entries {
let entry = entry?;
let grammar_path = entry.path();
let grammar_name = grammar_path.file_stem().unwrap().to_str().unwrap();
if grammar_path.extension() == Some("wasm".as_ref()) {
for grammar_name in manifest.grammars.keys() {
let mut grammar_path = grammars_dir.join(grammar_name.as_ref());
grammar_path.set_extension("wasm");
let wasm = fs::read(&grammar_path)?;
let language = wasm_store.load_language(grammar_name, &wasm)?;
log::info!("loaded grammar {grammar_name}");
grammars.insert(grammar_name.into(), language);
}
grammars.insert(grammar_name.to_string(), language);
}
Ok(grammars)
}
fn test_languages(extension_path: &Path, grammars: &HashMap<String, Language>) -> Result<()> {
let languages_dir = extension_path.join("languages");
if !languages_dir.exists() {
return Ok(());
}
let entries = fs::read_dir(&languages_dir)?;
for entry in entries {
let entry = entry?;
let language_dir = entry.path();
fn test_languages(
manifest: &ExtensionManifest,
extension_path: &Path,
grammars: &HashMap<String, Language>,
) -> Result<()> {
for relative_language_dir in &manifest.languages {
let language_dir = extension_path.join(relative_language_dir);
let config_path = language_dir.join("config.toml");
let config_content = fs::read_to_string(&config_path)?;
let config: LanguageConfig = toml::from_str(&config_content)?;
@ -139,21 +286,16 @@ fn test_languages(extension_path: &Path, grammars: &HashMap<String, Language>) -
Ok(())
}
async fn test_themes(extension_path: &Path, fs: Arc<dyn Fs>) -> Result<()> {
let themes_dir = extension_path.join("themes");
if !themes_dir.exists() {
return Ok(());
}
let entries = fs::read_dir(&themes_dir)?;
for entry in entries {
let entry = entry?;
let theme_path = entry.path();
if theme_path.extension() == Some("json".as_ref()) {
let theme_family = ThemeRegistry::read_user_theme(&entry.path(), fs.clone()).await?;
async fn test_themes(
manifest: &ExtensionManifest,
extension_path: &Path,
fs: Arc<dyn Fs>,
) -> Result<()> {
for relative_theme_path in &manifest.themes {
let theme_path = extension_path.join(relative_theme_path);
let theme_family = ThemeRegistry::read_user_theme(&theme_path, fs.clone()).await?;
log::info!("loaded theme family {}", theme_family.name);
}
}
Ok(())
}

View File

@ -0,0 +1,10 @@
use serde::{Deserialize, Serialize};
#[derive(Serialize, Deserialize)]
pub struct ExtensionApiManifest {
pub name: String,
pub version: String,
pub description: Option<String>,
pub authors: Vec<String>,
pub repository: String,
}

View File

@ -1,12 +1,14 @@
pub mod auth;
mod conn;
mod error;
mod extension;
mod notification;
mod peer;
pub mod proto;
pub use conn::Connection;
pub use error::*;
pub use extension::*;
pub use notification::*;
pub use peer::*;
mod macros;

View File

@ -3,6 +3,7 @@ name = "Gleam"
description = "Gleam support for Zed"
version = "0.0.1"
authors = ["Marshall Bowers <elliott.codes@gmail.com>"]
repository = "https://github.com/zed-industries/zed"
[language_servers.gleam]
name = "Gleam LSP"

View File

@ -3,6 +3,7 @@ name = "Uiua"
description = "Uiua support for Zed"
version = "0.0.1"
authors = ["Max Brunsfeld <max@zed.dev>"]
repository = "https://github.com/zed-industries/zed"
[language_servers.uiua]
name = "Uiua LSP"