zed-industries/zed
1
1
Fork 0
mirror of https://github.com/zed-industries/zed.git synced 2024-12-26 18:03:02 +03:00
Code Issues Packages Projects Releases Wiki Activity

Add telemetry events for loading extensions (#9793)

Browse Source 
* Store extensions versions' wasm API version in the database
* Share a common struct for extension API responses between collab and
client
* Add wasm API version and schema version to extension API responses

Release Notes:

- N/A

Co-authored-by: Marshall <marshall@zed.dev>
This commit is contained in:
Max Brunsfeld 2024-03-25 14:30:48 -07:00 committed by GitHub
parent 9b62e461ed
commit 5adc51f113
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
22 changed files with 531 additions and 306 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Cargo.lock generated
View File

@ -3472,6 +3472,7 @@ dependencies = [
"async-tar",
"async-trait",
"cap-std",
"client",
"collections",
"ctor",
"env_logger",
@ -7799,6 +7800,7 @@ dependencies = [
"anyhow",
"async-tungstenite",
"base64 0.13.1",
"chrono",
"collections",
"env_logger",
"futures 0.3.28",

10
crates/client/src/telemetry.rs
View File

@ -15,7 +15,8 @@ use std::{env, mem, path::PathBuf, sync::Arc, time::Duration};
use sysinfo::{CpuRefreshKind, MemoryRefreshKind, Pid, ProcessRefreshKind, RefreshKind, System};
use telemetry_events::{
ActionEvent, AppEvent, AssistantEvent, AssistantKind, CallEvent, CopilotEvent, CpuEvent,
EditEvent, EditorEvent, Event, EventRequestBody, EventWrapper, MemoryEvent, SettingEvent,
EditEvent, EditorEvent, Event, EventRequestBody, EventWrapper, ExtensionEvent, MemoryEvent,
SettingEvent,
};
use tempfile::NamedTempFile;
use util::http::{self, HttpClient, HttpClientWithUrl, Method};
@ -326,6 +327,13 @@ impl Telemetry {
self.report_event(event)
}
pub fn report_extension_event(self: &Arc<Self>, extension_id: Arc<str>, version: Arc<str>) {
self.report_event(Event::Extension(ExtensionEvent {
extension_id,
version,
}))
}
pub fn log_edit_event(self: &Arc<Self>, environment: &'static str) {
let mut state = self.state.lock();
let period_data = state.event_coalescer.log_event(environment);

1
crates/collab/migrations.sqlite/20221109000000_test_schema.sql
View File

@ -374,6 +374,7 @@ CREATE TABLE extension_versions (
repository TEXT NOT NULL,
description TEXT NOT NULL,
schema_version INTEGER NOT NULL DEFAULT 0,
wasm_api_version TEXT,
download_count INTEGER NOT NULL DEFAULT 0,
PRIMARY KEY (extension_id, version)
);

1
crates/collab/migrations/20240335123500_add_extension_wasm_api_version.sql Normal file
View File

@ -0,0 +1 @@
ALTER TABLE extension_versions ADD COLUMN wasm_api_version TEXT;

99
crates/collab/src/api/events.rs
View File

@ -1,5 +1,5 @@
use std::sync::{Arc, OnceLock};
use super::ips_file::IpsFile;
use crate::{api::slack, AppState, Error, Result};
use anyhow::{anyhow, Context};
use aws_sdk_s3::primitives::ByteStream;
use axum::{
@ -9,18 +9,16 @@ use axum::{
routing::post,
Extension, Router, TypedHeader,
};
use rpc::ExtensionMetadata;
use serde::{Serialize, Serializer};
use sha2::{Digest, Sha256};
use std::sync::{Arc, OnceLock};
use telemetry_events::{
ActionEvent, AppEvent, AssistantEvent, CallEvent, CopilotEvent, CpuEvent, EditEvent,
EditorEvent, Event, EventRequestBody, EventWrapper, MemoryEvent, SettingEvent,
EditorEvent, Event, EventRequestBody, EventWrapper, ExtensionEvent, MemoryEvent, SettingEvent,
};
use util::SemanticVersion;
use crate::{api::slack, AppState, Error, Result};
use super::ips_file::IpsFile;
pub fn router() -> Router {
Router::new()
.route("/telemetry/events", post(post_events))
@ -331,6 +329,21 @@ pub async fn post_events(
&request_body,
first_event_at,
)),
Event::Extension(event) => {
let metadata = app
.db
.get_extension_version(&event.extension_id, &event.version)
.await?;
to_upload
.extension_events
.push(ExtensionEventRow::from_event(
event.clone(),
&wrapper,
&request_body,
metadata,
first_event_at,
))
}
}
}
@ -352,6 +365,7 @@ struct ToUpload {
memory_events: Vec<MemoryEventRow>,
app_events: Vec<AppEventRow>,
setting_events: Vec<SettingEventRow>,
extension_events: Vec<ExtensionEventRow>,
edit_events: Vec<EditEventRow>,
action_events: Vec<ActionEventRow>,
}
@ -410,6 +424,15 @@ impl ToUpload {
.await
.with_context(|| format!("failed to upload to table '{SETTING_EVENTS_TABLE}'"))?;
const EXTENSION_EVENTS_TABLE: &str = "extension_events";
Self::upload_to_table(
EXTENSION_EVENTS_TABLE,
&self.extension_events,
clickhouse_client,
)
.await
.with_context(|| format!("failed to upload to table '{EXTENSION_EVENTS_TABLE}'"))?;
const EDIT_EVENTS_TABLE: &str = "edit_events";
Self::upload_to_table(EDIT_EVENTS_TABLE, &self.edit_events, clickhouse_client)
.await
@ -861,6 +884,68 @@ impl SettingEventRow {
}
}
#[derive(Serialize, Debug, clickhouse::Row)]
pub struct ExtensionEventRow {
// AppInfoBase
app_version: String,
major: Option<i32>,
minor: Option<i32>,
patch: Option<i32>,
release_channel: String,
// ClientEventBase
installation_id: Option<String>,
session_id: Option<String>,
is_staff: Option<bool>,
time: i64,
// ExtensionEventRow
extension_id: Arc<str>,
extension_version: Arc<str>,
dev: bool,
schema_version: Option<i32>,
wasm_api_version: Option<String>,
}
impl ExtensionEventRow {
fn from_event(
event: ExtensionEvent,
wrapper: &EventWrapper,
body: &EventRequestBody,
extension_metadata: Option<ExtensionMetadata>,
first_event_at: chrono::DateTime<chrono::Utc>,
) -> Self {
let semver = body.semver();
let time =
first_event_at + chrono::Duration::milliseconds(wrapper.milliseconds_since_first_event);
Self {
app_version: body.app_version.clone(),
major: semver.map(|s| s.major as i32),
minor: semver.map(|s| s.minor as i32),
patch: semver.map(|s| s.patch as i32),
release_channel: body.release_channel.clone().unwrap_or_default(),
installation_id: body.installation_id.clone(),
session_id: body.session_id.clone(),
is_staff: body.is_staff,
time: time.timestamp_millis(),
extension_id: event.extension_id,
extension_version: event.version,
dev: extension_metadata.is_none(),
schema_version: extension_metadata
.as_ref()
.and_then(|metadata| metadata.manifest.schema_version),
wasm_api_version: extension_metadata.as_ref().and_then(|metadata| {
metadata
.manifest
.wasm_api_version
.as_ref()
.map(|version| version.to_string())
}),
}
}
}
#[derive(Serialize, Debug, clickhouse::Row)]
pub struct EditEventRow {
// AppInfoBase

10
crates/collab/src/api/extensions.rs
View File

@ -1,7 +1,4 @@
use crate::{
db::{ExtensionMetadata, NewExtensionVersion},
AppState, Error, Result,
};
use crate::{db::NewExtensionVersion, AppState, Error, Result};
use anyhow::{anyhow, Context as _};
use aws_sdk_s3::presigning::PresigningConfig;
use axum::{
@ -12,7 +9,7 @@ use axum::{
Extension, Json, Router,
};
use collections::HashMap;
use rpc::ExtensionApiManifest;
use rpc::{ExtensionApiManifest, ExtensionMetadata};
use serde::{Deserialize, Serialize};
use std::{sync::Arc, time::Duration};
use time::PrimitiveDateTime;
@ -78,7 +75,7 @@ async fn download_latest_extension(
Extension(app),
Path(DownloadExtensionParams {
extension_id: params.extension_id,
version: extension.version,
version: extension.manifest.version,
}),
)
.await
@ -285,6 +282,7 @@ async fn fetch_extension_manifest(
authors: manifest.authors,
repository: manifest.repository,
schema_version: manifest.schema_version.unwrap_or(0),
wasm_api_version: manifest.wasm_api_version,
published_at,
})
}

16
crates/collab/src/db.rs
View File

@ -12,7 +12,7 @@ use futures::StreamExt;
use rand::{prelude::StdRng, Rng, SeedableRng};
use rpc::{
proto::{self},
ConnectionId,
ConnectionId, ExtensionMetadata,
};
use sea_orm::{
entity::prelude::*,
@ -726,22 +726,10 @@ pub struct NewExtensionVersion {
pub authors: Vec<String>,
pub repository: String,
pub schema_version: i32,
pub wasm_api_version: Option<String>,
pub published_at: PrimitiveDateTime,
}
#[derive(Debug, Serialize, PartialEq)]
pub struct ExtensionMetadata {
pub id: String,
pub name: String,
pub version: String,
pub authors: Vec<String>,
pub description: String,
pub repository: String,
#[serde(serialize_with = "serialize_iso8601")]
pub published_at: PrimitiveDateTime,
pub download_count: u64,
}
pub fn serialize_iso8601<S: Serializer>(
datetime: &PrimitiveDateTime,
serializer: S,

90
crates/collab/src/db/queries/extensions.rs
View File

@ -1,3 +1,5 @@
use chrono::Utc;
use super::*;
impl Database {
@ -31,22 +33,8 @@ impl Database {
Ok(extensions
.into_iter()
.filter_map(|(extension, latest_version)| {
let version = latest_version?;
Some(ExtensionMetadata {
id: extension.external_id,
name: extension.name,
version: version.version,
authors: version
.authors
.split(',')
.map(|author| author.trim().to_string())
.collect::<Vec<_>>(),
description: version.description,
repository: version.repository,
published_at: version.published_at,
download_count: extension.total_download_count as u64,
})
.filter_map(|(extension, version)| {
Some(metadata_from_extension_and_version(extension, version?))
})
.collect())
})
@ -67,22 +55,29 @@ impl Database {
.one(&*tx)
.await?;
Ok(extension.and_then(|(extension, latest_version)| {
let version = latest_version?;
Some(ExtensionMetadata {
id: extension.external_id,
name: extension.name,
version: version.version,
authors: version
.authors
.split(',')
.map(|author| author.trim().to_string())
.collect::<Vec<_>>(),
description: version.description,
repository: version.repository,
published_at: version.published_at,
download_count: extension.total_download_count as u64,
Ok(extension.and_then(|(extension, version)| {
Some(metadata_from_extension_and_version(extension, version?))
}))
})
.await
}
pub async fn get_extension_version(
&self,
extension_id: &str,
version: &str,
) -> Result<Option<ExtensionMetadata>> {
self.transaction(|tx| async move {
let extension = extension::Entity::find()
.filter(extension::Column::ExternalId.eq(extension_id))
.filter(extension_version::Column::Version.eq(version))
.inner_join(extension_version::Entity)
.select_also(extension_version::Entity)
.one(&*tx)
.await?;
Ok(extension.and_then(|(extension, version)| {
Some(metadata_from_extension_and_version(extension, version?))
}))
})
.await
@ -172,6 +167,7 @@ impl Database {
repository: ActiveValue::Set(version.repository.clone()),
description: ActiveValue::Set(version.description.clone()),
schema_version: ActiveValue::Set(version.schema_version),
wasm_api_version: ActiveValue::Set(version.wasm_api_version.clone()),
download_count: ActiveValue::NotSet,
}
}))
@ -241,3 +237,35 @@ impl Database {
.await
}
}
fn metadata_from_extension_and_version(
extension: extension::Model,
version: extension_version::Model,
) -> ExtensionMetadata {
ExtensionMetadata {
id: extension.external_id,
manifest: rpc::ExtensionApiManifest {
name: extension.name,
version: version.version,
authors: version
.authors
.split(',')
.map(|author| author.trim().to_string())
.collect::<Vec<_>>(),
description: Some(version.description),
repository: version.repository,
schema_version: Some(version.schema_version),
wasm_api_version: version.wasm_api_version,
},
published_at: convert_time_to_chrono(version.published_at),
download_count: extension.total_download_count as u64,
}
}
pub fn convert_time_to_chrono(time: time::PrimitiveDateTime) -> chrono::DateTime<Utc> {
chrono::DateTime::from_naive_utc_and_offset(
chrono::NaiveDateTime::from_timestamp_opt(time.assume_utc().unix_timestamp(), 0).unwrap(),
Utc,
)
}

1
crates/collab/src/db/tables/extension_version.rs
View File

@ -14,6 +14,7 @@ pub struct Model {
pub repository: String,
pub description: String,
pub schema_version: i32,
pub wasm_api_version: Option<String>,
pub download_count: i64,
}

70
crates/collab/src/db/tests/extension_tests.rs
View File

@ -1,10 +1,9 @@
use super::Database;
use crate::{
db::{ExtensionMetadata, NewExtensionVersion},
db::{queries::extensions::convert_time_to_chrono, ExtensionMetadata, NewExtensionVersion},
test_both_dbs,
};
use std::sync::Arc;
use time::{OffsetDateTime, PrimitiveDateTime};
test_both_dbs!(
test_extensions,
@ -19,8 +18,10 @@ async fn test_extensions(db: &Arc<Database>) {
let extensions = db.get_extensions(None, 1, 5).await.unwrap();
assert!(extensions.is_empty());
let t0 = OffsetDateTime::from_unix_timestamp_nanos(0).unwrap();
let t0 = PrimitiveDateTime::new(t0.date(), t0.time());
let t0 = time::OffsetDateTime::from_unix_timestamp_nanos(0).unwrap();
let t0 = time::PrimitiveDateTime::new(t0.date(), t0.time());
let t0_chrono = convert_time_to_chrono(t0);
db.insert_extension_versions(
&[
@ -34,6 +35,7 @@ async fn test_extensions(db: &Arc<Database>) {
authors: vec!["max".into()],
repository: "ext1/repo".into(),
schema_version: 1,
wasm_api_version: None,
published_at: t0,
},
NewExtensionVersion {
@ -43,6 +45,7 @@ async fn test_extensions(db: &Arc<Database>) {
authors: vec!["max".into(), "marshall".into()],
repository: "ext1/repo".into(),
schema_version: 1,
wasm_api_version: None,
published_at: t0,
},
],
@ -56,6 +59,7 @@ async fn test_extensions(db: &Arc<Database>) {
authors: vec!["marshall".into()],
repository: "ext2/repo".into(),
schema_version: 0,
wasm_api_version: None,
published_at: t0,
}],
),
@ -84,22 +88,30 @@ async fn test_extensions(db: &Arc<Database>) {
&[
ExtensionMetadata {
id: "ext1".into(),
manifest: rpc::ExtensionApiManifest {
name: "Extension One".into(),
version: "0.0.2".into(),
authors: vec!["max".into(), "marshall".into()],
description: "a good extension".into(),
description: Some("a good extension".into()),
repository: "ext1/repo".into(),
published_at: t0,
schema_version: Some(1),
wasm_api_version: None,
},
published_at: t0_chrono,
download_count: 0,
},
ExtensionMetadata {
id: "ext2".into(),
manifest: rpc::ExtensionApiManifest {
name: "Extension Two".into(),
version: "0.2.0".into(),
authors: vec!["marshall".into()],
description: "a great extension".into(),
description: Some("a great extension".into()),
repository: "ext2/repo".into(),
published_at: t0,
schema_version: Some(0),
wasm_api_version: None,
},
published_at: t0_chrono,
download_count: 0
},
]
@ -111,12 +123,16 @@ async fn test_extensions(db: &Arc<Database>) {
extensions,
&[ExtensionMetadata {
id: "ext2".into(),
manifest: rpc::ExtensionApiManifest {
name: "Extension Two".into(),
version: "0.2.0".into(),
authors: vec!["marshall".into()],
description: "a great extension".into(),
description: Some("a great extension".into()),
repository: "ext2/repo".into(),
published_at: t0,
schema_version: Some(0),
wasm_api_version: None,
},
published_at: t0_chrono,
download_count: 0
},]
);
@ -147,22 +163,30 @@ async fn test_extensions(db: &Arc<Database>) {
&[
ExtensionMetadata {
id: "ext2".into(),
manifest: rpc::ExtensionApiManifest {
name: "Extension Two".into(),
version: "0.2.0".into(),
authors: vec!["marshall".into()],
description: "a great extension".into(),
description: Some("a great extension".into()),
repository: "ext2/repo".into(),
published_at: t0,
schema_version: Some(0),
wasm_api_version: None,
},
published_at: t0_chrono,
download_count: 7
},
ExtensionMetadata {
id: "ext1".into(),
manifest: rpc::ExtensionApiManifest {
name: "Extension One".into(),
version: "0.0.2".into(),
authors: vec!["max".into(), "marshall".into()],
description: "a good extension".into(),
description: Some("a good extension".into()),
repository: "ext1/repo".into(),
published_at: t0,
schema_version: Some(1),
wasm_api_version: None,
},
published_at: t0_chrono,
download_count: 5,
},
]
@ -181,6 +205,7 @@ async fn test_extensions(db: &Arc<Database>) {
authors: vec!["max".into(), "marshall".into()],
repository: "ext1/repo".into(),
schema_version: 1,
wasm_api_version: None,
published_at: t0,
}],
),
@ -193,6 +218,7 @@ async fn test_extensions(db: &Arc<Database>) {
authors: vec!["marshall".into()],
repository: "ext2/repo".into(),
schema_version: 0,
wasm_api_version: None,
published_at: t0,
}],
),
@ -223,22 +249,30 @@ async fn test_extensions(db: &Arc<Database>) {
&[
ExtensionMetadata {
id: "ext2".into(),
manifest: rpc::ExtensionApiManifest {
name: "Extension Two".into(),
version: "0.2.0".into(),
authors: vec!["marshall".into()],
description: "a great extension".into(),
description: Some("a great extension".into()),
repository: "ext2/repo".into(),
published_at: t0,
schema_version: Some(0),
wasm_api_version: None,
},
published_at: t0_chrono,
download_count: 7
},
ExtensionMetadata {
id: "ext1".into(),
manifest: rpc::ExtensionApiManifest {
name: "Extension One".into(),
version: "0.0.3".into(),
authors: vec!["max".into(), "marshall".into()],
description: "a real good extension".into(),
description: Some("a real good extension".into()),
repository: "ext1/repo".into(),
published_at: t0,
schema_version: Some(1),
wasm_api_version: None,
},
published_at: t0_chrono,
download_count: 5,
},
]

1
crates/extension/Cargo.toml
View File

@ -22,6 +22,7 @@ async-compression.workspace = true
async-tar.workspace = true
async-trait.workspace = true
cap-std.workspace = true
client.workspace = true
collections.workspace = true
fs.workspace = true
futures.workspace = true

101
crates/extension/src/extension_builder.rs
View File

@ -1,3 +1,4 @@
use crate::wasm_host::parse_wasm_extension_version;
use crate::ExtensionManifest;
use crate::{extension_manifest::ExtensionLibraryKind, GrammarManifestEntry};
use anyhow::{anyhow, bail, Context as _, Result};
@ -73,9 +74,11 @@ impl ExtensionBuilder {
pub async fn compile_extension(
&self,
extension_dir: &Path,
extension_manifest: &ExtensionManifest,
extension_manifest: &mut ExtensionManifest,
options: CompileExtensionOptions,
) -> Result<()> {
populate_defaults(extension_manifest, &extension_dir)?;
if extension_dir.is_relative() {
bail!(
"extension dir {} is not an absolute path",
@ -85,12 +88,9 @@ impl ExtensionBuilder {
fs::create_dir_all(&self.cache_dir).context("failed to create cache dir")?;
let cargo_toml_path = extension_dir.join("Cargo.toml");
if extension_manifest.lib.kind == Some(ExtensionLibraryKind::Rust)
|| fs::metadata(&cargo_toml_path).map_or(false, |stat| stat.is_file())
{
if extension_manifest.lib.kind == Some(ExtensionLibraryKind::Rust) {
log::info!("compiling Rust extension {}", extension_dir.display());
self.compile_rust_extension(extension_dir, options)
self.compile_rust_extension(extension_dir, extension_manifest, options)
.await
.context("failed to compile Rust extension")?;
}
@ -108,6 +108,7 @@ impl ExtensionBuilder {
async fn compile_rust_extension(
&self,
extension_dir: &Path,
manifest: &mut ExtensionManifest,
options: CompileExtensionOptions,
) -> Result<(), anyhow::Error> {
self.install_rust_wasm_target_if_needed()?;
@ -162,6 +163,11 @@ impl ExtensionBuilder {
.strip_custom_sections(&component_bytes)
.context("failed to strip debug sections from wasm component")?;
let wasm_extension_api_version =
parse_wasm_extension_version(&manifest.id, &component_bytes)
.context("compiled wasm did not contain a valid zed extension api version")?;
manifest.lib.version = Some(wasm_extension_api_version);
fs::write(extension_dir.join("extension.wasm"), &component_bytes)
.context("failed to write extension.wasm")?;
@ -469,3 +475,86 @@ impl ExtensionBuilder {
Ok(output)
}
}
fn populate_defaults(manifest: &mut ExtensionManifest, extension_path: &Path) -> Result<()> {
// For legacy extensions on the v0 schema (aka, using `extension.json`), clear out any existing
// contents of the computed fields, since we don't care what the existing values are.
if manifest.schema_version == 0 {
manifest.languages.clear();
manifest.grammars.clear();
manifest.themes.clear();
}
let cargo_toml_path = extension_path.join("Cargo.toml");
if cargo_toml_path.exists() {
manifest.lib.kind = Some(ExtensionLibraryKind::Rust);
}
let languages_dir = extension_path.join("languages");
if languages_dir.exists() {
for entry in fs::read_dir(&languages_dir).context("failed to list languages dir")? {
let entry = entry?;
let language_dir = entry.path();
let config_path = language_dir.join("config.toml");
if config_path.exists() {
let relative_language_dir =
language_dir.strip_prefix(extension_path)?.to_path_buf();
if !manifest.languages.contains(&relative_language_dir) {
manifest.languages.push(relative_language_dir);
}
}
}
}
let themes_dir = extension_path.join("themes");
if themes_dir.exists() {
for entry in fs::read_dir(&themes_dir).context("failed to list themes dir")? {
let entry = entry?;
let theme_path = entry.path();
if theme_path.extension() == Some("json".as_ref()) {
let relative_theme_path = theme_path.strip_prefix(extension_path)?.to_path_buf();
if !manifest.themes.contains(&relative_theme_path) {
manifest.themes.push(relative_theme_path);
}
}
}
}
// For legacy extensions on the v0 schema (aka, using `extension.json`), we want to populate the grammars in
// the manifest using the contents of the `grammars` directory.
if manifest.schema_version == 0 {
let grammars_dir = extension_path.join("grammars");
if grammars_dir.exists() {
for entry in fs::read_dir(&grammars_dir).context("failed to list grammars dir")? {
let entry = entry?;
let grammar_path = entry.path();
if grammar_path.extension() == Some("toml".as_ref()) {
#[derive(Deserialize)]
struct GrammarConfigToml {
pub repository: String,
pub commit: String,
}
let grammar_config = fs::read_to_string(&grammar_path)?;
let grammar_config: GrammarConfigToml = toml::from_str(&grammar_config)?;
let grammar_name = grammar_path
.file_stem()
.and_then(|stem| stem.to_str())
.ok_or_else(|| anyhow!("no grammar name"))?;
if !manifest.grammars.contains_key(grammar_name) {
manifest.grammars.insert(
grammar_name.into(),
GrammarManifestEntry {
repository: grammar_config.repository,
rev: grammar_config.commit,
},
);
}
}
}
}
}
Ok(())
}

75
crates/extension/src/extension_manifest.rs
View File

@ -1,7 +1,14 @@
use anyhow::{anyhow, Context, Result};
use collections::BTreeMap;
use fs::Fs;
use language::LanguageServerName;
use serde::{Deserialize, Serialize};
use std::{path::PathBuf, sync::Arc};
use std::{
ffi::OsStr,
path::{Path, PathBuf},
sync::Arc,
};
use util::SemanticVersion;
/// This is the old version of the extension manifest, from when it was `extension.json`.
#[derive(Clone, PartialEq, Eq, Debug, Deserialize, Serialize)]
@ -53,6 +60,7 @@ pub struct ExtensionManifest {
#[derive(Clone, Default, PartialEq, Eq, Debug, Deserialize, Serialize)]
pub struct LibManifestEntry {
pub kind: Option<ExtensionLibraryKind>,
pub version: Option<SemanticVersion>,
}
#[derive(Clone, PartialEq, Eq, Debug, Deserialize, Serialize)]
@ -71,3 +79,68 @@ pub struct GrammarManifestEntry {
pub struct LanguageServerManifestEntry {
pub language: Arc<str>,
}
impl ExtensionManifest {
pub async fn load(fs: Arc<dyn Fs>, extension_dir: &Path) -> Result<Self> {
let extension_name = extension_dir
.file_name()
.and_then(OsStr::to_str)
.ok_or_else(|| anyhow!("invalid extension name"))?;
let mut extension_manifest_path = extension_dir.join("extension.json");
if fs.is_file(&extension_manifest_path).await {
let manifest_content = fs
.load(&extension_manifest_path)
.await
.with_context(|| format!("failed to load {extension_name} extension.json"))?;
let manifest_json = serde_json::from_str::<OldExtensionManifest>(&manifest_content)
.with_context(|| {
format!("invalid extension.json for extension {extension_name}")
})?;
Ok(manifest_from_old_manifest(manifest_json, extension_name))
} else {
extension_manifest_path.set_extension("toml");
let manifest_content = fs
.load(&extension_manifest_path)
.await
.with_context(|| format!("failed to load {extension_name} extension.toml"))?;
toml::from_str(&manifest_content)
.with_context(|| format!("invalid extension.json for extension {extension_name}"))
}
}
}
fn manifest_from_old_manifest(
manifest_json: OldExtensionManifest,
extension_id: &str,
) -> ExtensionManifest {
ExtensionManifest {
id: extension_id.into(),
name: manifest_json.name,
version: manifest_json.version,
description: manifest_json.description,
repository: manifest_json.repository,
authors: manifest_json.authors,
schema_version: 0,
lib: Default::default(),
themes: {
let mut themes = manifest_json.themes.into_values().collect::<Vec<_>>();
themes.sort();
themes.dedup();
themes
},
languages: {
let mut languages = manifest_json.languages.into_values().collect::<Vec<_>>();
languages.sort();
languages.dedup();
languages
},
grammars: manifest_json
.grammars
.into_keys()
.map(|grammar_name| (grammar_name, Default::default()))
.collect(),
language_servers: Default::default(),
}
}

108
crates/extension/src/extension_store.rs
View File

@ -10,6 +10,7 @@ use crate::{extension_lsp_adapter::ExtensionLspAdapter, wasm_host::wit};
use anyhow::{anyhow, bail, Context as _, Result};
use async_compression::futures::bufread::GzipDecoder;
use async_tar::Archive;
use client::{telemetry::Telemetry, Client};
use collections::{hash_map, BTreeMap, HashMap, HashSet};
use extension_builder::{CompileExtensionOptions, ExtensionBuilder};
use fs::{Fs, RemoveOptions};
@ -30,7 +31,6 @@ use node_runtime::NodeRuntime;
use serde::{Deserialize, Serialize};
use std::{
cmp::Ordering,
ffi::OsStr,
path::{self, Path, PathBuf},
sync::Arc,
time::{Duration, Instant},
@ -75,6 +75,7 @@ pub struct ExtensionStore {
extension_index: ExtensionIndex,
fs: Arc<dyn Fs>,
http_client: Arc<HttpClientWithUrl>,
telemetry: Option<Arc<Telemetry>>,
reload_tx: UnboundedSender<Option<Arc<str>>>,
reload_complete_senders: Vec<oneshot::Sender<()>>,
installed_dir: PathBuf,
@ -149,7 +150,7 @@ actions!(zed, [ReloadExtensions]);
pub fn init(
fs: Arc<fs::RealFs>,
http_client: Arc<HttpClientWithUrl>,
client: Arc<Client>,
node_runtime: Arc<dyn NodeRuntime>,
language_registry: Arc<LanguageRegistry>,
theme_registry: Arc<ThemeRegistry>,
@ -160,7 +161,8 @@ pub fn init(
EXTENSIONS_DIR.clone(),
None,
fs,
http_client,
client.http_client().clone(),
Some(client.telemetry().clone()),
node_runtime,
language_registry,
theme_registry,
@ -187,6 +189,7 @@ impl ExtensionStore {
build_dir: Option<PathBuf>,
fs: Arc<dyn Fs>,
http_client: Arc<HttpClientWithUrl>,
telemetry: Option<Arc<Telemetry>>,
node_runtime: Arc<dyn NodeRuntime>,
language_registry: Arc<LanguageRegistry>,
theme_registry: Arc<ThemeRegistry>,
@ -216,6 +219,7 @@ impl ExtensionStore {
wasm_extensions: Vec::new(),
fs,
http_client,
telemetry,
language_registry,
theme_registry,
reload_tx,
@ -587,8 +591,8 @@ impl ExtensionStore {
let builder = self.builder.clone();
cx.spawn(move |this, mut cx| async move {
let extension_manifest =
Self::load_extension_manifest(fs.clone(), &extension_source_path).await?;
let mut extension_manifest =
ExtensionManifest::load(fs.clone(), &extension_source_path).await?;
let extension_id = extension_manifest.id.clone();
if !this.update(&mut cx, |this, cx| {
@ -622,7 +626,7 @@ impl ExtensionStore {
builder
.compile_extension(
&extension_source_path,
&extension_manifest,
&mut extension_manifest,
CompileExtensionOptions { release: false },
)
.await
@ -667,9 +671,13 @@ impl ExtensionStore {
cx.notify();
let compile = cx.background_executor().spawn(async move {
let manifest = Self::load_extension_manifest(fs, &path).await?;
let mut manifest = ExtensionManifest::load(fs, &path).await?;
builder
.compile_extension(&path, &manifest, CompileExtensionOptions { release: true })
.compile_extension(
&path,
&mut manifest,
CompileExtensionOptions { release: true },
)
.await
});
@ -759,6 +767,17 @@ impl ExtensionStore {
extensions_to_unload.len() - reload_count
);
if let Some(telemetry) = &self.telemetry {
for extension_id in &extensions_to_load {
if let Some(extension) = self.extension_index.extensions.get(extension_id) {
telemetry.report_extension_event(
extension_id.clone(),
extension.manifest.version.clone(),
);
}
}
}
let themes_to_remove = old_index
.themes
.iter()
@ -908,7 +927,9 @@ impl ExtensionStore {
cx.background_executor().clone(),
)
.await
.context("failed to load wasm extension")
.with_context(|| {
format!("failed to load wasm extension {}", extension.manifest.id)
})
})
.await;
@ -989,8 +1010,7 @@ impl ExtensionStore {
extension_dir: PathBuf,
index: &mut ExtensionIndex,
) -> Result<()> {
let mut extension_manifest =
Self::load_extension_manifest(fs.clone(), &extension_dir).await?;
let mut extension_manifest = ExtensionManifest::load(fs.clone(), &extension_dir).await?;
let extension_id = extension_manifest.id.clone();
// TODO: distinguish dev extensions more explicitly, by the absence
@ -1082,72 +1102,6 @@ impl ExtensionStore {
Ok(())
}
pub async fn load_extension_manifest(
fs: Arc<dyn Fs>,
extension_dir: &Path,
) -> Result<ExtensionManifest> {
let extension_name = extension_dir
.file_name()
.and_then(OsStr::to_str)
.ok_or_else(|| anyhow!("invalid extension name"))?;
let mut extension_manifest_path = extension_dir.join("extension.json");
if fs.is_file(&extension_manifest_path).await {
let manifest_content = fs
.load(&extension_manifest_path)
.await
.with_context(|| format!("failed to load {extension_name} extension.json"))?;
let manifest_json = serde_json::from_str::<OldExtensionManifest>(&manifest_content)
.with_context(|| {
format!("invalid extension.json for extension {extension_name}")
})?;
Ok(manifest_from_old_manifest(manifest_json, extension_name))
} else {
extension_manifest_path.set_extension("toml");
let manifest_content = fs
.load(&extension_manifest_path)
.await
.with_context(|| format!("failed to load {extension_name} extension.toml"))?;
toml::from_str(&manifest_content)
.with_context(|| format!("invalid extension.json for extension {extension_name}"))
}
}
}
fn manifest_from_old_manifest(
manifest_json: OldExtensionManifest,
extension_id: &str,
) -> ExtensionManifest {
ExtensionManifest {
id: extension_id.into(),
name: manifest_json.name,
version: manifest_json.version,
description: manifest_json.description,
repository: manifest_json.repository,
authors: manifest_json.authors,
schema_version: 0,
lib: Default::default(),
themes: {
let mut themes = manifest_json.themes.into_values().collect::<Vec<_>>();
themes.sort();
themes.dedup();
themes
},
languages: {
let mut languages = manifest_json.languages.into_values().collect::<Vec<_>>();
languages.sort();
languages.dedup();
languages
},
grammars: manifest_json
.grammars
.into_keys()
.map(|grammar_name| (grammar_name, Default::default()))
.collect(),
language_servers: Default::default(),
}
}
fn load_plugin_queries(root_path: &Path) -> LanguageQueries {

3
crates/extension/src/extension_store_test.rs
View File

@ -262,6 +262,7 @@ async fn test_extension_store(cx: &mut TestAppContext) {
None,
fs.clone(),
http_client.clone(),
None,
node_runtime.clone(),
language_registry.clone(),
theme_registry.clone(),
@ -381,6 +382,7 @@ async fn test_extension_store(cx: &mut TestAppContext) {
None,
fs.clone(),
http_client.clone(),
None,
node_runtime.clone(),
language_registry.clone(),
theme_registry.clone(),
@ -538,6 +540,7 @@ async fn test_extension_store_with_gleam_extension(cx: &mut TestAppContext) {
Some(cache_dir),
fs.clone(),
http_client.clone(),
None,
node_runtime,
language_registry.clone(),
theme_registry.clone(),

49
crates/extension/src/wasm_host.rs
View File

@ -40,7 +40,7 @@ pub struct WasmExtension {
tx: UnboundedSender<ExtensionCall>,
pub(crate) manifest: Arc<ExtensionManifest>,
#[allow(unused)]
zed_api_version: SemanticVersion,
pub zed_api_version: SemanticVersion,
}
pub(crate) struct WasmState {
@ -93,29 +93,11 @@ impl WasmHost {
) -> impl 'static + Future<Output = Result<WasmExtension>> {
let this = self.clone();
async move {
let zed_api_version = parse_wasm_extension_version(&manifest.id, &wasm_bytes)?;
let component = Component::from_binary(&this.engine, &wasm_bytes)
.context("failed to compile wasm component")?;
let mut zed_api_version = None;
for part in wasmparser::Parser::new(0).parse_all(&wasm_bytes) {
if let wasmparser::Payload::CustomSection(s) = part? {
if s.name() == "zed:api-version" {
zed_api_version = parse_extension_version(s.data());
if zed_api_version.is_none() {
bail!(
"extension {} has invalid zed:api-version section: {:?}",
manifest.id,
s.data()
);
}
}
}
}
let Some(zed_api_version) = zed_api_version else {
bail!("extension {} has no zed:api-version section", manifest.id);
};
let mut store = wasmtime::Store::new(
&this.engine,
WasmState {
@ -196,7 +178,30 @@ impl WasmHost {
}
}
fn parse_extension_version(data: &[u8]) -> Option<SemanticVersion> {
pub fn parse_wasm_extension_version(
extension_id: &str,
wasm_bytes: &[u8],
) -> Result<SemanticVersion> {
for part in wasmparser::Parser::new(0).parse_all(wasm_bytes) {
if let wasmparser::Payload::CustomSection(s) = part? {
if s.name() == "zed:api-version" {
let version = parse_wasm_extension_version_custom_section(s.data());
if let Some(version) = version {
return Ok(version);
} else {
bail!(
"extension {} has invalid zed:api-version section: {:?}",
extension_id,
s.data()
);
}
}
}
}
bail!("extension {} has no zed:api-version section", extension_id)
}
fn parse_wasm_extension_version_custom_section(data: &[u8]) -> Option<SemanticVersion> {
if data.len() == 6 {
Some(SemanticVersion {
major: u16::from_be_bytes([data[0], data[1]]) as _,

92
crates/extension_cli/src/main.rs
View File

@ -11,10 +11,9 @@ use anyhow::{anyhow, bail, Context, Result};
use clap::Parser;
use extension::{
extension_builder::{CompileExtensionOptions, ExtensionBuilder},
ExtensionLibraryKind, ExtensionManifest, ExtensionStore, GrammarManifestEntry,
ExtensionManifest,
};
use language::LanguageConfig;
use serde::Deserialize;
use theme::ThemeRegistry;
use tree_sitter::{Language, Query, WasmStore};
@ -56,15 +55,14 @@ async fn main() -> Result<()> {
};
log::info!("loading extension manifest");
let mut manifest = ExtensionStore::load_extension_manifest(fs.clone(), &extension_path).await?;
populate_default_paths(&mut manifest, &extension_path)?;
let mut manifest = ExtensionManifest::load(fs.clone(), &extension_path).await?;
log::info!("compiling extension");
let builder = ExtensionBuilder::new(scratch_dir);
builder
.compile_extension(
&extension_path,
&manifest,
&mut manifest,
CompileExtensionOptions { release: true },
)
.await
@ -101,6 +99,7 @@ async fn main() -> Result<()> {
repository: manifest
.repository
.ok_or_else(|| anyhow!("missing repository in extension manifest"))?,
wasm_api_version: manifest.lib.version.map(|version| version.to_string()),
})?;
fs::remove_dir_all(&archive_dir)?;
fs::write(output_dir.join("manifest.json"), manifest_json.as_bytes())?;
@ -108,89 +107,6 @@ async fn main() -> Result<()> {
Ok(())
}
fn populate_default_paths(manifest: &mut ExtensionManifest, extension_path: &Path) -> Result<()> {
// For legacy extensions on the v0 schema (aka, using `extension.json`), clear out any existing
// contents of the computed fields, since we don't care what the existing values are.
if manifest.schema_version == 0 {
manifest.languages.clear();
manifest.grammars.clear();
manifest.themes.clear();
}
let cargo_toml_path = extension_path.join("Cargo.toml");
if cargo_toml_path.exists() {
manifest.lib.kind = Some(ExtensionLibraryKind::Rust);
}
let languages_dir = extension_path.join("languages");
if languages_dir.exists() {
for entry in fs::read_dir(&languages_dir).context("failed to list languages dir")? {
let entry = entry?;
let language_dir = entry.path();
let config_path = language_dir.join("config.toml");
if config_path.exists() {
let relative_language_dir =
language_dir.strip_prefix(extension_path)?.to_path_buf();
if !manifest.languages.contains(&relative_language_dir) {
manifest.languages.push(relative_language_dir);
}
}
}
}
let themes_dir = extension_path.join("themes");
if themes_dir.exists() {
for entry in fs::read_dir(&themes_dir).context("failed to list themes dir")? {
let entry = entry?;
let theme_path = entry.path();
if theme_path.extension() == Some("json".as_ref()) {
let relative_theme_path = theme_path.strip_prefix(extension_path)?.to_path_buf();
if !manifest.themes.contains(&relative_theme_path) {
manifest.themes.push(relative_theme_path);
}
}
}
}
// For legacy extensions on the v0 schema (aka, using `extension.json`), we want to populate the grammars in
// the manifest using the contents of the `grammars` directory.
if manifest.schema_version == 0 {
let grammars_dir = extension_path.join("grammars");
if grammars_dir.exists() {
for entry in fs::read_dir(&grammars_dir).context("failed to list grammars dir")? {
let entry = entry?;
let grammar_path = entry.path();
if grammar_path.extension() == Some("toml".as_ref()) {
#[derive(Deserialize)]
struct GrammarConfigToml {
pub repository: String,
pub commit: String,
}
let grammar_config = fs::read_to_string(&grammar_path)?;
let grammar_config: GrammarConfigToml = toml::from_str(&grammar_config)?;
let grammar_name = grammar_path
.file_stem()
.and_then(|stem| stem.to_str())
.ok_or_else(|| anyhow!("no grammar name"))?;
if !manifest.grammars.contains_key(grammar_name) {
manifest.grammars.insert(
grammar_name.into(),
GrammarManifestEntry {
repository: grammar_config.repository,
rev: grammar_config.commit,
},
);
}
}
}
}
}
Ok(())
}
async fn copy_extension_resources(
manifest: &ExtensionManifest,
extension_path: &Path,

1
crates/rpc/Cargo.toml
View File

@ -20,6 +20,7 @@ test-support = ["collections/test-support", "gpui/test-support"]
anyhow.workspace = true
async-tungstenite = "0.16"
base64.workspace = true
chrono.workspace = true
collections.workspace = true
futures.workspace = true
gpui = { workspace = true, optional = true }

13
crates/rpc/src/extension.rs
View File

@ -1,6 +1,7 @@
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
#[derive(Serialize, Deserialize)]
#[derive(Serialize, Deserialize, Debug, PartialEq)]
pub struct ExtensionApiManifest {
pub name: String,
pub version: String,
@ -8,4 +9,14 @@ pub struct ExtensionApiManifest {
pub authors: Vec<String>,
pub repository: String,
pub schema_version: Option<i32>,
pub wasm_api_version: Option<String>,
}
#[derive(Debug, Serialize, PartialEq)]
pub struct ExtensionMetadata {
pub id: String,
#[serde(flatten)]
pub manifest: ExtensionApiManifest,
pub published_at: DateTime<Utc>,
pub download_count: u64,
}

10
crates/telemetry_events/src/telemetry_events.rs
View File

@ -1,6 +1,5 @@
use std::fmt::Display;
use serde::{Deserialize, Serialize};
use std::{fmt::Display, sync::Arc};
use util::SemanticVersion;
#[derive(Serialize, Deserialize, Debug)]
@ -61,6 +60,7 @@ pub enum Event {
Memory(MemoryEvent),
App(AppEvent),
Setting(SettingEvent),
Extension(ExtensionEvent),
Edit(EditEvent),
Action(ActionEvent),
}
@ -125,6 +125,12 @@ pub struct SettingEvent {
pub value: String,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct ExtensionEvent {
pub extension_id: Arc<str>,
pub version: Arc<str>,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct AppEvent {
pub operation: String,

24
crates/util/src/semantic_version.rs
View File

@ -4,10 +4,10 @@ use std::{
};
use anyhow::{anyhow, Result};
use serde::Serialize;
use serde::{de::Error, Deserialize, Serialize};
/// A datastructure representing a semantic version number
#[derive(Clone, Copy, Debug, Default, Eq, Ord, PartialEq, PartialOrd, Serialize)]
#[derive(Clone, Copy, Debug, Default, Eq, Ord, PartialEq, PartialOrd)]
pub struct SemanticVersion {
pub major: usize,
pub minor: usize,
@ -61,3 +61,23 @@ impl Display for SemanticVersion {
write!(f, "{}.{}.{}", self.major, self.minor, self.patch)
}
}
impl Serialize for SemanticVersion {
fn serialize<S>(&self, serializer: S) -> std::prelude::v1::Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
serializer.serialize_str(&self.to_string())
}
}
impl<'de> Deserialize<'de> for SemanticVersion {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
let string = String::deserialize(deserializer)?;
Self::from_str(&string)
.map_err(|_| Error::custom(format!("Invalid version string \"{string}\"")))
}
}

2
crates/zed/src/main.rs
View File

@ -179,7 +179,7 @@ fn main() {
extension::init(
fs.clone(),
http.clone(),
client.clone(),
node_runtime.clone(),
languages.clone(),
ThemeRegistry::global(cx),