From 5adc51f113c6646306e74fc22f4c1d1292b5daa5 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 25 Mar 2024 14:30:48 -0700 Subject: [PATCH] Add telemetry events for loading extensions (#9793) * Store extensions versions' wasm API version in the database * Share a common struct for extension API responses between collab and client * Add wasm API version and schema version to extension API responses Release Notes: - N/A Co-authored-by: Marshall --- Cargo.lock | 2 + crates/client/src/telemetry.rs | 10 +- .../20221109000000_test_schema.sql | 1 + ...5123500_add_extension_wasm_api_version.sql | 1 + crates/collab/src/api/events.rs | 99 +++++++++++++- crates/collab/src/api/extensions.rs | 10 +- crates/collab/src/db.rs | 16 +-- crates/collab/src/db/queries/extensions.rs | 92 ++++++++----- .../collab/src/db/tables/extension_version.rs | 1 + crates/collab/src/db/tests/extension_tests.rs | 126 +++++++++++------- crates/extension/Cargo.toml | 1 + crates/extension/src/extension_builder.rs | 101 +++++++++++++- crates/extension/src/extension_manifest.rs | 75 ++++++++++- crates/extension/src/extension_store.rs | 108 +++++---------- crates/extension/src/extension_store_test.rs | 3 + crates/extension/src/wasm_host.rs | 49 ++++--- crates/extension_cli/src/main.rs | 92 +------------ crates/rpc/Cargo.toml | 1 + crates/rpc/src/extension.rs | 13 +- .../telemetry_events/src/telemetry_events.rs | 10 +- crates/util/src/semantic_version.rs | 24 +++- crates/zed/src/main.rs | 2 +- 22 files changed, 531 insertions(+), 306 deletions(-) create mode 100644 crates/collab/migrations/20240335123500_add_extension_wasm_api_version.sql diff --git a/Cargo.lock b/Cargo.lock index 6fd86c4ce3..3528cc0d7e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3472,6 +3472,7 @@ dependencies = [ "async-tar", "async-trait", "cap-std", + "client", "collections", "ctor", "env_logger", @@ -7799,6 +7800,7 @@ dependencies = [ "anyhow", "async-tungstenite", "base64 0.13.1", + "chrono", "collections", "env_logger", "futures 0.3.28", diff --git a/crates/client/src/telemetry.rs b/crates/client/src/telemetry.rs index 7811fa4e14..6b21940fb8 100644 --- a/crates/client/src/telemetry.rs +++ b/crates/client/src/telemetry.rs @@ -15,7 +15,8 @@ use std::{env, mem, path::PathBuf, sync::Arc, time::Duration}; use sysinfo::{CpuRefreshKind, MemoryRefreshKind, Pid, ProcessRefreshKind, RefreshKind, System}; use telemetry_events::{ ActionEvent, AppEvent, AssistantEvent, AssistantKind, CallEvent, CopilotEvent, CpuEvent, - EditEvent, EditorEvent, Event, EventRequestBody, EventWrapper, MemoryEvent, SettingEvent, + EditEvent, EditorEvent, Event, EventRequestBody, EventWrapper, ExtensionEvent, MemoryEvent, + SettingEvent, }; use tempfile::NamedTempFile; use util::http::{self, HttpClient, HttpClientWithUrl, Method}; @@ -326,6 +327,13 @@ impl Telemetry { self.report_event(event) } + pub fn report_extension_event(self: &Arc, extension_id: Arc, version: Arc) { + self.report_event(Event::Extension(ExtensionEvent { + extension_id, + version, + })) + } + pub fn log_edit_event(self: &Arc, environment: &'static str) { let mut state = self.state.lock(); let period_data = state.event_coalescer.log_event(environment); diff --git a/crates/collab/migrations.sqlite/20221109000000_test_schema.sql b/crates/collab/migrations.sqlite/20221109000000_test_schema.sql index d82ef75813..9ad045e56d 100644 --- a/crates/collab/migrations.sqlite/20221109000000_test_schema.sql +++ b/crates/collab/migrations.sqlite/20221109000000_test_schema.sql @@ -374,6 +374,7 @@ CREATE TABLE extension_versions ( repository TEXT NOT NULL, description TEXT NOT NULL, schema_version INTEGER NOT NULL DEFAULT 0, + wasm_api_version TEXT, download_count INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (extension_id, version) ); diff --git a/crates/collab/migrations/20240335123500_add_extension_wasm_api_version.sql b/crates/collab/migrations/20240335123500_add_extension_wasm_api_version.sql new file mode 100644 index 0000000000..3b95323d26 --- /dev/null +++ b/crates/collab/migrations/20240335123500_add_extension_wasm_api_version.sql @@ -0,0 +1 @@ +ALTER TABLE extension_versions ADD COLUMN wasm_api_version TEXT; diff --git a/crates/collab/src/api/events.rs b/crates/collab/src/api/events.rs index 102609f2d9..26f811e5ff 100644 --- a/crates/collab/src/api/events.rs +++ b/crates/collab/src/api/events.rs @@ -1,5 +1,5 @@ -use std::sync::{Arc, OnceLock}; - +use super::ips_file::IpsFile; +use crate::{api::slack, AppState, Error, Result}; use anyhow::{anyhow, Context}; use aws_sdk_s3::primitives::ByteStream; use axum::{ @@ -9,18 +9,16 @@ use axum::{ routing::post, Extension, Router, TypedHeader, }; +use rpc::ExtensionMetadata; use serde::{Serialize, Serializer}; use sha2::{Digest, Sha256}; +use std::sync::{Arc, OnceLock}; use telemetry_events::{ ActionEvent, AppEvent, AssistantEvent, CallEvent, CopilotEvent, CpuEvent, EditEvent, - EditorEvent, Event, EventRequestBody, EventWrapper, MemoryEvent, SettingEvent, + EditorEvent, Event, EventRequestBody, EventWrapper, ExtensionEvent, MemoryEvent, SettingEvent, }; use util::SemanticVersion; -use crate::{api::slack, AppState, Error, Result}; - -use super::ips_file::IpsFile; - pub fn router() -> Router { Router::new() .route("/telemetry/events", post(post_events)) @@ -331,6 +329,21 @@ pub async fn post_events( &request_body, first_event_at, )), + Event::Extension(event) => { + let metadata = app + .db + .get_extension_version(&event.extension_id, &event.version) + .await?; + to_upload + .extension_events + .push(ExtensionEventRow::from_event( + event.clone(), + &wrapper, + &request_body, + metadata, + first_event_at, + )) + } } } @@ -352,6 +365,7 @@ struct ToUpload { memory_events: Vec, app_events: Vec, setting_events: Vec, + extension_events: Vec, edit_events: Vec, action_events: Vec, } @@ -410,6 +424,15 @@ impl ToUpload { .await .with_context(|| format!("failed to upload to table '{SETTING_EVENTS_TABLE}'"))?; + const EXTENSION_EVENTS_TABLE: &str = "extension_events"; + Self::upload_to_table( + EXTENSION_EVENTS_TABLE, + &self.extension_events, + clickhouse_client, + ) + .await + .with_context(|| format!("failed to upload to table '{EXTENSION_EVENTS_TABLE}'"))?; + const EDIT_EVENTS_TABLE: &str = "edit_events"; Self::upload_to_table(EDIT_EVENTS_TABLE, &self.edit_events, clickhouse_client) .await @@ -861,6 +884,68 @@ impl SettingEventRow { } } +#[derive(Serialize, Debug, clickhouse::Row)] +pub struct ExtensionEventRow { + // AppInfoBase + app_version: String, + major: Option, + minor: Option, + patch: Option, + release_channel: String, + + // ClientEventBase + installation_id: Option, + session_id: Option, + is_staff: Option, + time: i64, + + // ExtensionEventRow + extension_id: Arc, + extension_version: Arc, + dev: bool, + schema_version: Option, + wasm_api_version: Option, +} + +impl ExtensionEventRow { + fn from_event( + event: ExtensionEvent, + wrapper: &EventWrapper, + body: &EventRequestBody, + extension_metadata: Option, + first_event_at: chrono::DateTime, + ) -> Self { + let semver = body.semver(); + let time = + first_event_at + chrono::Duration::milliseconds(wrapper.milliseconds_since_first_event); + + Self { + app_version: body.app_version.clone(), + major: semver.map(|s| s.major as i32), + minor: semver.map(|s| s.minor as i32), + patch: semver.map(|s| s.patch as i32), + release_channel: body.release_channel.clone().unwrap_or_default(), + installation_id: body.installation_id.clone(), + session_id: body.session_id.clone(), + is_staff: body.is_staff, + time: time.timestamp_millis(), + extension_id: event.extension_id, + extension_version: event.version, + dev: extension_metadata.is_none(), + schema_version: extension_metadata + .as_ref() + .and_then(|metadata| metadata.manifest.schema_version), + wasm_api_version: extension_metadata.as_ref().and_then(|metadata| { + metadata + .manifest + .wasm_api_version + .as_ref() + .map(|version| version.to_string()) + }), + } + } +} + #[derive(Serialize, Debug, clickhouse::Row)] pub struct EditEventRow { // AppInfoBase diff --git a/crates/collab/src/api/extensions.rs b/crates/collab/src/api/extensions.rs index c586a23ccb..1c29270f88 100644 --- a/crates/collab/src/api/extensions.rs +++ b/crates/collab/src/api/extensions.rs @@ -1,7 +1,4 @@ -use crate::{ - db::{ExtensionMetadata, NewExtensionVersion}, - AppState, Error, Result, -}; +use crate::{db::NewExtensionVersion, AppState, Error, Result}; use anyhow::{anyhow, Context as _}; use aws_sdk_s3::presigning::PresigningConfig; use axum::{ @@ -12,7 +9,7 @@ use axum::{ Extension, Json, Router, }; use collections::HashMap; -use rpc::ExtensionApiManifest; +use rpc::{ExtensionApiManifest, ExtensionMetadata}; use serde::{Deserialize, Serialize}; use std::{sync::Arc, time::Duration}; use time::PrimitiveDateTime; @@ -78,7 +75,7 @@ async fn download_latest_extension( Extension(app), Path(DownloadExtensionParams { extension_id: params.extension_id, - version: extension.version, + version: extension.manifest.version, }), ) .await @@ -285,6 +282,7 @@ async fn fetch_extension_manifest( authors: manifest.authors, repository: manifest.repository, schema_version: manifest.schema_version.unwrap_or(0), + wasm_api_version: manifest.wasm_api_version, published_at, }) } diff --git a/crates/collab/src/db.rs b/crates/collab/src/db.rs index 71565e8444..637a8c31f5 100644 --- a/crates/collab/src/db.rs +++ b/crates/collab/src/db.rs @@ -12,7 +12,7 @@ use futures::StreamExt; use rand::{prelude::StdRng, Rng, SeedableRng}; use rpc::{ proto::{self}, - ConnectionId, + ConnectionId, ExtensionMetadata, }; use sea_orm::{ entity::prelude::*, @@ -726,22 +726,10 @@ pub struct NewExtensionVersion { pub authors: Vec, pub repository: String, pub schema_version: i32, + pub wasm_api_version: Option, pub published_at: PrimitiveDateTime, } -#[derive(Debug, Serialize, PartialEq)] -pub struct ExtensionMetadata { - pub id: String, - pub name: String, - pub version: String, - pub authors: Vec, - pub description: String, - pub repository: String, - #[serde(serialize_with = "serialize_iso8601")] - pub published_at: PrimitiveDateTime, - pub download_count: u64, -} - pub fn serialize_iso8601( datetime: &PrimitiveDateTime, serializer: S, diff --git a/crates/collab/src/db/queries/extensions.rs b/crates/collab/src/db/queries/extensions.rs index 065b5800ab..9304c3c60a 100644 --- a/crates/collab/src/db/queries/extensions.rs +++ b/crates/collab/src/db/queries/extensions.rs @@ -1,3 +1,5 @@ +use chrono::Utc; + use super::*; impl Database { @@ -31,22 +33,8 @@ impl Database { Ok(extensions .into_iter() - .filter_map(|(extension, latest_version)| { - let version = latest_version?; - Some(ExtensionMetadata { - id: extension.external_id, - name: extension.name, - version: version.version, - authors: version - .authors - .split(',') - .map(|author| author.trim().to_string()) - .collect::>(), - description: version.description, - repository: version.repository, - published_at: version.published_at, - download_count: extension.total_download_count as u64, - }) + .filter_map(|(extension, version)| { + Some(metadata_from_extension_and_version(extension, version?)) }) .collect()) }) @@ -67,22 +55,29 @@ impl Database { .one(&*tx) .await?; - Ok(extension.and_then(|(extension, latest_version)| { - let version = latest_version?; - Some(ExtensionMetadata { - id: extension.external_id, - name: extension.name, - version: version.version, - authors: version - .authors - .split(',') - .map(|author| author.trim().to_string()) - .collect::>(), - description: version.description, - repository: version.repository, - published_at: version.published_at, - download_count: extension.total_download_count as u64, - }) + Ok(extension.and_then(|(extension, version)| { + Some(metadata_from_extension_and_version(extension, version?)) + })) + }) + .await + } + + pub async fn get_extension_version( + &self, + extension_id: &str, + version: &str, + ) -> Result> { + self.transaction(|tx| async move { + let extension = extension::Entity::find() + .filter(extension::Column::ExternalId.eq(extension_id)) + .filter(extension_version::Column::Version.eq(version)) + .inner_join(extension_version::Entity) + .select_also(extension_version::Entity) + .one(&*tx) + .await?; + + Ok(extension.and_then(|(extension, version)| { + Some(metadata_from_extension_and_version(extension, version?)) })) }) .await @@ -172,6 +167,7 @@ impl Database { repository: ActiveValue::Set(version.repository.clone()), description: ActiveValue::Set(version.description.clone()), schema_version: ActiveValue::Set(version.schema_version), + wasm_api_version: ActiveValue::Set(version.wasm_api_version.clone()), download_count: ActiveValue::NotSet, } })) @@ -241,3 +237,35 @@ impl Database { .await } } + +fn metadata_from_extension_and_version( + extension: extension::Model, + version: extension_version::Model, +) -> ExtensionMetadata { + ExtensionMetadata { + id: extension.external_id, + manifest: rpc::ExtensionApiManifest { + name: extension.name, + version: version.version, + authors: version + .authors + .split(',') + .map(|author| author.trim().to_string()) + .collect::>(), + description: Some(version.description), + repository: version.repository, + schema_version: Some(version.schema_version), + wasm_api_version: version.wasm_api_version, + }, + + published_at: convert_time_to_chrono(version.published_at), + download_count: extension.total_download_count as u64, + } +} + +pub fn convert_time_to_chrono(time: time::PrimitiveDateTime) -> chrono::DateTime { + chrono::DateTime::from_naive_utc_and_offset( + chrono::NaiveDateTime::from_timestamp_opt(time.assume_utc().unix_timestamp(), 0).unwrap(), + Utc, + ) +} diff --git a/crates/collab/src/db/tables/extension_version.rs b/crates/collab/src/db/tables/extension_version.rs index b3f6565ac5..60e3e5c7da 100644 --- a/crates/collab/src/db/tables/extension_version.rs +++ b/crates/collab/src/db/tables/extension_version.rs @@ -14,6 +14,7 @@ pub struct Model { pub repository: String, pub description: String, pub schema_version: i32, + pub wasm_api_version: Option, pub download_count: i64, } diff --git a/crates/collab/src/db/tests/extension_tests.rs b/crates/collab/src/db/tests/extension_tests.rs index 4a8af2d652..49e94e24d5 100644 --- a/crates/collab/src/db/tests/extension_tests.rs +++ b/crates/collab/src/db/tests/extension_tests.rs @@ -1,10 +1,9 @@ use super::Database; use crate::{ - db::{ExtensionMetadata, NewExtensionVersion}, + db::{queries::extensions::convert_time_to_chrono, ExtensionMetadata, NewExtensionVersion}, test_both_dbs, }; use std::sync::Arc; -use time::{OffsetDateTime, PrimitiveDateTime}; test_both_dbs!( test_extensions, @@ -19,8 +18,10 @@ async fn test_extensions(db: &Arc) { let extensions = db.get_extensions(None, 1, 5).await.unwrap(); assert!(extensions.is_empty()); - let t0 = OffsetDateTime::from_unix_timestamp_nanos(0).unwrap(); - let t0 = PrimitiveDateTime::new(t0.date(), t0.time()); + let t0 = time::OffsetDateTime::from_unix_timestamp_nanos(0).unwrap(); + let t0 = time::PrimitiveDateTime::new(t0.date(), t0.time()); + + let t0_chrono = convert_time_to_chrono(t0); db.insert_extension_versions( &[ @@ -34,6 +35,7 @@ async fn test_extensions(db: &Arc) { authors: vec!["max".into()], repository: "ext1/repo".into(), schema_version: 1, + wasm_api_version: None, published_at: t0, }, NewExtensionVersion { @@ -43,6 +45,7 @@ async fn test_extensions(db: &Arc) { authors: vec!["max".into(), "marshall".into()], repository: "ext1/repo".into(), schema_version: 1, + wasm_api_version: None, published_at: t0, }, ], @@ -56,6 +59,7 @@ async fn test_extensions(db: &Arc) { authors: vec!["marshall".into()], repository: "ext2/repo".into(), schema_version: 0, + wasm_api_version: None, published_at: t0, }], ), @@ -84,22 +88,30 @@ async fn test_extensions(db: &Arc) { &[ ExtensionMetadata { id: "ext1".into(), - name: "Extension One".into(), - version: "0.0.2".into(), - authors: vec!["max".into(), "marshall".into()], - description: "a good extension".into(), - repository: "ext1/repo".into(), - published_at: t0, + manifest: rpc::ExtensionApiManifest { + name: "Extension One".into(), + version: "0.0.2".into(), + authors: vec!["max".into(), "marshall".into()], + description: Some("a good extension".into()), + repository: "ext1/repo".into(), + schema_version: Some(1), + wasm_api_version: None, + }, + published_at: t0_chrono, download_count: 0, }, ExtensionMetadata { id: "ext2".into(), - name: "Extension Two".into(), - version: "0.2.0".into(), - authors: vec!["marshall".into()], - description: "a great extension".into(), - repository: "ext2/repo".into(), - published_at: t0, + manifest: rpc::ExtensionApiManifest { + name: "Extension Two".into(), + version: "0.2.0".into(), + authors: vec!["marshall".into()], + description: Some("a great extension".into()), + repository: "ext2/repo".into(), + schema_version: Some(0), + wasm_api_version: None, + }, + published_at: t0_chrono, download_count: 0 }, ] @@ -111,12 +123,16 @@ async fn test_extensions(db: &Arc) { extensions, &[ExtensionMetadata { id: "ext2".into(), - name: "Extension Two".into(), - version: "0.2.0".into(), - authors: vec!["marshall".into()], - description: "a great extension".into(), - repository: "ext2/repo".into(), - published_at: t0, + manifest: rpc::ExtensionApiManifest { + name: "Extension Two".into(), + version: "0.2.0".into(), + authors: vec!["marshall".into()], + description: Some("a great extension".into()), + repository: "ext2/repo".into(), + schema_version: Some(0), + wasm_api_version: None, + }, + published_at: t0_chrono, download_count: 0 },] ); @@ -147,22 +163,30 @@ async fn test_extensions(db: &Arc) { &[ ExtensionMetadata { id: "ext2".into(), - name: "Extension Two".into(), - version: "0.2.0".into(), - authors: vec!["marshall".into()], - description: "a great extension".into(), - repository: "ext2/repo".into(), - published_at: t0, + manifest: rpc::ExtensionApiManifest { + name: "Extension Two".into(), + version: "0.2.0".into(), + authors: vec!["marshall".into()], + description: Some("a great extension".into()), + repository: "ext2/repo".into(), + schema_version: Some(0), + wasm_api_version: None, + }, + published_at: t0_chrono, download_count: 7 }, ExtensionMetadata { id: "ext1".into(), - name: "Extension One".into(), - version: "0.0.2".into(), - authors: vec!["max".into(), "marshall".into()], - description: "a good extension".into(), - repository: "ext1/repo".into(), - published_at: t0, + manifest: rpc::ExtensionApiManifest { + name: "Extension One".into(), + version: "0.0.2".into(), + authors: vec!["max".into(), "marshall".into()], + description: Some("a good extension".into()), + repository: "ext1/repo".into(), + schema_version: Some(1), + wasm_api_version: None, + }, + published_at: t0_chrono, download_count: 5, }, ] @@ -181,6 +205,7 @@ async fn test_extensions(db: &Arc) { authors: vec!["max".into(), "marshall".into()], repository: "ext1/repo".into(), schema_version: 1, + wasm_api_version: None, published_at: t0, }], ), @@ -193,6 +218,7 @@ async fn test_extensions(db: &Arc) { authors: vec!["marshall".into()], repository: "ext2/repo".into(), schema_version: 0, + wasm_api_version: None, published_at: t0, }], ), @@ -223,22 +249,30 @@ async fn test_extensions(db: &Arc) { &[ ExtensionMetadata { id: "ext2".into(), - name: "Extension Two".into(), - version: "0.2.0".into(), - authors: vec!["marshall".into()], - description: "a great extension".into(), - repository: "ext2/repo".into(), - published_at: t0, + manifest: rpc::ExtensionApiManifest { + name: "Extension Two".into(), + version: "0.2.0".into(), + authors: vec!["marshall".into()], + description: Some("a great extension".into()), + repository: "ext2/repo".into(), + schema_version: Some(0), + wasm_api_version: None, + }, + published_at: t0_chrono, download_count: 7 }, ExtensionMetadata { id: "ext1".into(), - name: "Extension One".into(), - version: "0.0.3".into(), - authors: vec!["max".into(), "marshall".into()], - description: "a real good extension".into(), - repository: "ext1/repo".into(), - published_at: t0, + manifest: rpc::ExtensionApiManifest { + name: "Extension One".into(), + version: "0.0.3".into(), + authors: vec!["max".into(), "marshall".into()], + description: Some("a real good extension".into()), + repository: "ext1/repo".into(), + schema_version: Some(1), + wasm_api_version: None, + }, + published_at: t0_chrono, download_count: 5, }, ] diff --git a/crates/extension/Cargo.toml b/crates/extension/Cargo.toml index 4c982ad0fe..fdd7bfb6ce 100644 --- a/crates/extension/Cargo.toml +++ b/crates/extension/Cargo.toml @@ -22,6 +22,7 @@ async-compression.workspace = true async-tar.workspace = true async-trait.workspace = true cap-std.workspace = true +client.workspace = true collections.workspace = true fs.workspace = true futures.workspace = true diff --git a/crates/extension/src/extension_builder.rs b/crates/extension/src/extension_builder.rs index e26de37d3a..b933ef5680 100644 --- a/crates/extension/src/extension_builder.rs +++ b/crates/extension/src/extension_builder.rs @@ -1,3 +1,4 @@ +use crate::wasm_host::parse_wasm_extension_version; use crate::ExtensionManifest; use crate::{extension_manifest::ExtensionLibraryKind, GrammarManifestEntry}; use anyhow::{anyhow, bail, Context as _, Result}; @@ -73,9 +74,11 @@ impl ExtensionBuilder { pub async fn compile_extension( &self, extension_dir: &Path, - extension_manifest: &ExtensionManifest, + extension_manifest: &mut ExtensionManifest, options: CompileExtensionOptions, ) -> Result<()> { + populate_defaults(extension_manifest, &extension_dir)?; + if extension_dir.is_relative() { bail!( "extension dir {} is not an absolute path", @@ -85,12 +88,9 @@ impl ExtensionBuilder { fs::create_dir_all(&self.cache_dir).context("failed to create cache dir")?; - let cargo_toml_path = extension_dir.join("Cargo.toml"); - if extension_manifest.lib.kind == Some(ExtensionLibraryKind::Rust) - || fs::metadata(&cargo_toml_path).map_or(false, |stat| stat.is_file()) - { + if extension_manifest.lib.kind == Some(ExtensionLibraryKind::Rust) { log::info!("compiling Rust extension {}", extension_dir.display()); - self.compile_rust_extension(extension_dir, options) + self.compile_rust_extension(extension_dir, extension_manifest, options) .await .context("failed to compile Rust extension")?; } @@ -108,6 +108,7 @@ impl ExtensionBuilder { async fn compile_rust_extension( &self, extension_dir: &Path, + manifest: &mut ExtensionManifest, options: CompileExtensionOptions, ) -> Result<(), anyhow::Error> { self.install_rust_wasm_target_if_needed()?; @@ -162,6 +163,11 @@ impl ExtensionBuilder { .strip_custom_sections(&component_bytes) .context("failed to strip debug sections from wasm component")?; + let wasm_extension_api_version = + parse_wasm_extension_version(&manifest.id, &component_bytes) + .context("compiled wasm did not contain a valid zed extension api version")?; + manifest.lib.version = Some(wasm_extension_api_version); + fs::write(extension_dir.join("extension.wasm"), &component_bytes) .context("failed to write extension.wasm")?; @@ -469,3 +475,86 @@ impl ExtensionBuilder { Ok(output) } } + +fn populate_defaults(manifest: &mut ExtensionManifest, extension_path: &Path) -> Result<()> { + // For legacy extensions on the v0 schema (aka, using `extension.json`), clear out any existing + // contents of the computed fields, since we don't care what the existing values are. + if manifest.schema_version == 0 { + manifest.languages.clear(); + manifest.grammars.clear(); + manifest.themes.clear(); + } + + let cargo_toml_path = extension_path.join("Cargo.toml"); + if cargo_toml_path.exists() { + manifest.lib.kind = Some(ExtensionLibraryKind::Rust); + } + + let languages_dir = extension_path.join("languages"); + if languages_dir.exists() { + for entry in fs::read_dir(&languages_dir).context("failed to list languages dir")? { + let entry = entry?; + let language_dir = entry.path(); + let config_path = language_dir.join("config.toml"); + if config_path.exists() { + let relative_language_dir = + language_dir.strip_prefix(extension_path)?.to_path_buf(); + if !manifest.languages.contains(&relative_language_dir) { + manifest.languages.push(relative_language_dir); + } + } + } + } + + let themes_dir = extension_path.join("themes"); + if themes_dir.exists() { + for entry in fs::read_dir(&themes_dir).context("failed to list themes dir")? { + let entry = entry?; + let theme_path = entry.path(); + if theme_path.extension() == Some("json".as_ref()) { + let relative_theme_path = theme_path.strip_prefix(extension_path)?.to_path_buf(); + if !manifest.themes.contains(&relative_theme_path) { + manifest.themes.push(relative_theme_path); + } + } + } + } + + // For legacy extensions on the v0 schema (aka, using `extension.json`), we want to populate the grammars in + // the manifest using the contents of the `grammars` directory. + if manifest.schema_version == 0 { + let grammars_dir = extension_path.join("grammars"); + if grammars_dir.exists() { + for entry in fs::read_dir(&grammars_dir).context("failed to list grammars dir")? { + let entry = entry?; + let grammar_path = entry.path(); + if grammar_path.extension() == Some("toml".as_ref()) { + #[derive(Deserialize)] + struct GrammarConfigToml { + pub repository: String, + pub commit: String, + } + + let grammar_config = fs::read_to_string(&grammar_path)?; + let grammar_config: GrammarConfigToml = toml::from_str(&grammar_config)?; + + let grammar_name = grammar_path + .file_stem() + .and_then(|stem| stem.to_str()) + .ok_or_else(|| anyhow!("no grammar name"))?; + if !manifest.grammars.contains_key(grammar_name) { + manifest.grammars.insert( + grammar_name.into(), + GrammarManifestEntry { + repository: grammar_config.repository, + rev: grammar_config.commit, + }, + ); + } + } + } + } + } + + Ok(()) +} diff --git a/crates/extension/src/extension_manifest.rs b/crates/extension/src/extension_manifest.rs index bf2bd45fb5..c542914058 100644 --- a/crates/extension/src/extension_manifest.rs +++ b/crates/extension/src/extension_manifest.rs @@ -1,7 +1,14 @@ +use anyhow::{anyhow, Context, Result}; use collections::BTreeMap; +use fs::Fs; use language::LanguageServerName; use serde::{Deserialize, Serialize}; -use std::{path::PathBuf, sync::Arc}; +use std::{ + ffi::OsStr, + path::{Path, PathBuf}, + sync::Arc, +}; +use util::SemanticVersion; /// This is the old version of the extension manifest, from when it was `extension.json`. #[derive(Clone, PartialEq, Eq, Debug, Deserialize, Serialize)] @@ -53,6 +60,7 @@ pub struct ExtensionManifest { #[derive(Clone, Default, PartialEq, Eq, Debug, Deserialize, Serialize)] pub struct LibManifestEntry { pub kind: Option, + pub version: Option, } #[derive(Clone, PartialEq, Eq, Debug, Deserialize, Serialize)] @@ -71,3 +79,68 @@ pub struct GrammarManifestEntry { pub struct LanguageServerManifestEntry { pub language: Arc, } + +impl ExtensionManifest { + pub async fn load(fs: Arc, extension_dir: &Path) -> Result { + let extension_name = extension_dir + .file_name() + .and_then(OsStr::to_str) + .ok_or_else(|| anyhow!("invalid extension name"))?; + + let mut extension_manifest_path = extension_dir.join("extension.json"); + if fs.is_file(&extension_manifest_path).await { + let manifest_content = fs + .load(&extension_manifest_path) + .await + .with_context(|| format!("failed to load {extension_name} extension.json"))?; + let manifest_json = serde_json::from_str::(&manifest_content) + .with_context(|| { + format!("invalid extension.json for extension {extension_name}") + })?; + + Ok(manifest_from_old_manifest(manifest_json, extension_name)) + } else { + extension_manifest_path.set_extension("toml"); + let manifest_content = fs + .load(&extension_manifest_path) + .await + .with_context(|| format!("failed to load {extension_name} extension.toml"))?; + toml::from_str(&manifest_content) + .with_context(|| format!("invalid extension.json for extension {extension_name}")) + } + } +} + +fn manifest_from_old_manifest( + manifest_json: OldExtensionManifest, + extension_id: &str, +) -> ExtensionManifest { + ExtensionManifest { + id: extension_id.into(), + name: manifest_json.name, + version: manifest_json.version, + description: manifest_json.description, + repository: manifest_json.repository, + authors: manifest_json.authors, + schema_version: 0, + lib: Default::default(), + themes: { + let mut themes = manifest_json.themes.into_values().collect::>(); + themes.sort(); + themes.dedup(); + themes + }, + languages: { + let mut languages = manifest_json.languages.into_values().collect::>(); + languages.sort(); + languages.dedup(); + languages + }, + grammars: manifest_json + .grammars + .into_keys() + .map(|grammar_name| (grammar_name, Default::default())) + .collect(), + language_servers: Default::default(), + } +} diff --git a/crates/extension/src/extension_store.rs b/crates/extension/src/extension_store.rs index 237555407d..fdb45ebf5c 100644 --- a/crates/extension/src/extension_store.rs +++ b/crates/extension/src/extension_store.rs @@ -10,6 +10,7 @@ use crate::{extension_lsp_adapter::ExtensionLspAdapter, wasm_host::wit}; use anyhow::{anyhow, bail, Context as _, Result}; use async_compression::futures::bufread::GzipDecoder; use async_tar::Archive; +use client::{telemetry::Telemetry, Client}; use collections::{hash_map, BTreeMap, HashMap, HashSet}; use extension_builder::{CompileExtensionOptions, ExtensionBuilder}; use fs::{Fs, RemoveOptions}; @@ -30,7 +31,6 @@ use node_runtime::NodeRuntime; use serde::{Deserialize, Serialize}; use std::{ cmp::Ordering, - ffi::OsStr, path::{self, Path, PathBuf}, sync::Arc, time::{Duration, Instant}, @@ -75,6 +75,7 @@ pub struct ExtensionStore { extension_index: ExtensionIndex, fs: Arc, http_client: Arc, + telemetry: Option>, reload_tx: UnboundedSender>>, reload_complete_senders: Vec>, installed_dir: PathBuf, @@ -149,7 +150,7 @@ actions!(zed, [ReloadExtensions]); pub fn init( fs: Arc, - http_client: Arc, + client: Arc, node_runtime: Arc, language_registry: Arc, theme_registry: Arc, @@ -160,7 +161,8 @@ pub fn init( EXTENSIONS_DIR.clone(), None, fs, - http_client, + client.http_client().clone(), + Some(client.telemetry().clone()), node_runtime, language_registry, theme_registry, @@ -187,6 +189,7 @@ impl ExtensionStore { build_dir: Option, fs: Arc, http_client: Arc, + telemetry: Option>, node_runtime: Arc, language_registry: Arc, theme_registry: Arc, @@ -216,6 +219,7 @@ impl ExtensionStore { wasm_extensions: Vec::new(), fs, http_client, + telemetry, language_registry, theme_registry, reload_tx, @@ -587,8 +591,8 @@ impl ExtensionStore { let builder = self.builder.clone(); cx.spawn(move |this, mut cx| async move { - let extension_manifest = - Self::load_extension_manifest(fs.clone(), &extension_source_path).await?; + let mut extension_manifest = + ExtensionManifest::load(fs.clone(), &extension_source_path).await?; let extension_id = extension_manifest.id.clone(); if !this.update(&mut cx, |this, cx| { @@ -622,7 +626,7 @@ impl ExtensionStore { builder .compile_extension( &extension_source_path, - &extension_manifest, + &mut extension_manifest, CompileExtensionOptions { release: false }, ) .await @@ -667,9 +671,13 @@ impl ExtensionStore { cx.notify(); let compile = cx.background_executor().spawn(async move { - let manifest = Self::load_extension_manifest(fs, &path).await?; + let mut manifest = ExtensionManifest::load(fs, &path).await?; builder - .compile_extension(&path, &manifest, CompileExtensionOptions { release: true }) + .compile_extension( + &path, + &mut manifest, + CompileExtensionOptions { release: true }, + ) .await }); @@ -759,6 +767,17 @@ impl ExtensionStore { extensions_to_unload.len() - reload_count ); + if let Some(telemetry) = &self.telemetry { + for extension_id in &extensions_to_load { + if let Some(extension) = self.extension_index.extensions.get(extension_id) { + telemetry.report_extension_event( + extension_id.clone(), + extension.manifest.version.clone(), + ); + } + } + } + let themes_to_remove = old_index .themes .iter() @@ -908,7 +927,9 @@ impl ExtensionStore { cx.background_executor().clone(), ) .await - .context("failed to load wasm extension") + .with_context(|| { + format!("failed to load wasm extension {}", extension.manifest.id) + }) }) .await; @@ -989,8 +1010,7 @@ impl ExtensionStore { extension_dir: PathBuf, index: &mut ExtensionIndex, ) -> Result<()> { - let mut extension_manifest = - Self::load_extension_manifest(fs.clone(), &extension_dir).await?; + let mut extension_manifest = ExtensionManifest::load(fs.clone(), &extension_dir).await?; let extension_id = extension_manifest.id.clone(); // TODO: distinguish dev extensions more explicitly, by the absence @@ -1082,72 +1102,6 @@ impl ExtensionStore { Ok(()) } - - pub async fn load_extension_manifest( - fs: Arc, - extension_dir: &Path, - ) -> Result { - let extension_name = extension_dir - .file_name() - .and_then(OsStr::to_str) - .ok_or_else(|| anyhow!("invalid extension name"))?; - - let mut extension_manifest_path = extension_dir.join("extension.json"); - if fs.is_file(&extension_manifest_path).await { - let manifest_content = fs - .load(&extension_manifest_path) - .await - .with_context(|| format!("failed to load {extension_name} extension.json"))?; - let manifest_json = serde_json::from_str::(&manifest_content) - .with_context(|| { - format!("invalid extension.json for extension {extension_name}") - })?; - - Ok(manifest_from_old_manifest(manifest_json, extension_name)) - } else { - extension_manifest_path.set_extension("toml"); - let manifest_content = fs - .load(&extension_manifest_path) - .await - .with_context(|| format!("failed to load {extension_name} extension.toml"))?; - toml::from_str(&manifest_content) - .with_context(|| format!("invalid extension.json for extension {extension_name}")) - } - } -} - -fn manifest_from_old_manifest( - manifest_json: OldExtensionManifest, - extension_id: &str, -) -> ExtensionManifest { - ExtensionManifest { - id: extension_id.into(), - name: manifest_json.name, - version: manifest_json.version, - description: manifest_json.description, - repository: manifest_json.repository, - authors: manifest_json.authors, - schema_version: 0, - lib: Default::default(), - themes: { - let mut themes = manifest_json.themes.into_values().collect::>(); - themes.sort(); - themes.dedup(); - themes - }, - languages: { - let mut languages = manifest_json.languages.into_values().collect::>(); - languages.sort(); - languages.dedup(); - languages - }, - grammars: manifest_json - .grammars - .into_keys() - .map(|grammar_name| (grammar_name, Default::default())) - .collect(), - language_servers: Default::default(), - } } fn load_plugin_queries(root_path: &Path) -> LanguageQueries { diff --git a/crates/extension/src/extension_store_test.rs b/crates/extension/src/extension_store_test.rs index b179329168..01742484a5 100644 --- a/crates/extension/src/extension_store_test.rs +++ b/crates/extension/src/extension_store_test.rs @@ -262,6 +262,7 @@ async fn test_extension_store(cx: &mut TestAppContext) { None, fs.clone(), http_client.clone(), + None, node_runtime.clone(), language_registry.clone(), theme_registry.clone(), @@ -381,6 +382,7 @@ async fn test_extension_store(cx: &mut TestAppContext) { None, fs.clone(), http_client.clone(), + None, node_runtime.clone(), language_registry.clone(), theme_registry.clone(), @@ -538,6 +540,7 @@ async fn test_extension_store_with_gleam_extension(cx: &mut TestAppContext) { Some(cache_dir), fs.clone(), http_client.clone(), + None, node_runtime, language_registry.clone(), theme_registry.clone(), diff --git a/crates/extension/src/wasm_host.rs b/crates/extension/src/wasm_host.rs index 23801304da..b610db205e 100644 --- a/crates/extension/src/wasm_host.rs +++ b/crates/extension/src/wasm_host.rs @@ -40,7 +40,7 @@ pub struct WasmExtension { tx: UnboundedSender, pub(crate) manifest: Arc, #[allow(unused)] - zed_api_version: SemanticVersion, + pub zed_api_version: SemanticVersion, } pub(crate) struct WasmState { @@ -93,29 +93,11 @@ impl WasmHost { ) -> impl 'static + Future> { let this = self.clone(); async move { + let zed_api_version = parse_wasm_extension_version(&manifest.id, &wasm_bytes)?; + let component = Component::from_binary(&this.engine, &wasm_bytes) .context("failed to compile wasm component")?; - let mut zed_api_version = None; - for part in wasmparser::Parser::new(0).parse_all(&wasm_bytes) { - if let wasmparser::Payload::CustomSection(s) = part? { - if s.name() == "zed:api-version" { - zed_api_version = parse_extension_version(s.data()); - if zed_api_version.is_none() { - bail!( - "extension {} has invalid zed:api-version section: {:?}", - manifest.id, - s.data() - ); - } - } - } - } - - let Some(zed_api_version) = zed_api_version else { - bail!("extension {} has no zed:api-version section", manifest.id); - }; - let mut store = wasmtime::Store::new( &this.engine, WasmState { @@ -196,7 +178,30 @@ impl WasmHost { } } -fn parse_extension_version(data: &[u8]) -> Option { +pub fn parse_wasm_extension_version( + extension_id: &str, + wasm_bytes: &[u8], +) -> Result { + for part in wasmparser::Parser::new(0).parse_all(wasm_bytes) { + if let wasmparser::Payload::CustomSection(s) = part? { + if s.name() == "zed:api-version" { + let version = parse_wasm_extension_version_custom_section(s.data()); + if let Some(version) = version { + return Ok(version); + } else { + bail!( + "extension {} has invalid zed:api-version section: {:?}", + extension_id, + s.data() + ); + } + } + } + } + bail!("extension {} has no zed:api-version section", extension_id) +} + +fn parse_wasm_extension_version_custom_section(data: &[u8]) -> Option { if data.len() == 6 { Some(SemanticVersion { major: u16::from_be_bytes([data[0], data[1]]) as _, diff --git a/crates/extension_cli/src/main.rs b/crates/extension_cli/src/main.rs index 30ca4edcc9..ee252dc732 100644 --- a/crates/extension_cli/src/main.rs +++ b/crates/extension_cli/src/main.rs @@ -11,10 +11,9 @@ use anyhow::{anyhow, bail, Context, Result}; use clap::Parser; use extension::{ extension_builder::{CompileExtensionOptions, ExtensionBuilder}, - ExtensionLibraryKind, ExtensionManifest, ExtensionStore, GrammarManifestEntry, + ExtensionManifest, }; use language::LanguageConfig; -use serde::Deserialize; use theme::ThemeRegistry; use tree_sitter::{Language, Query, WasmStore}; @@ -56,15 +55,14 @@ async fn main() -> Result<()> { }; log::info!("loading extension manifest"); - let mut manifest = ExtensionStore::load_extension_manifest(fs.clone(), &extension_path).await?; - populate_default_paths(&mut manifest, &extension_path)?; + let mut manifest = ExtensionManifest::load(fs.clone(), &extension_path).await?; log::info!("compiling extension"); let builder = ExtensionBuilder::new(scratch_dir); builder .compile_extension( &extension_path, - &manifest, + &mut manifest, CompileExtensionOptions { release: true }, ) .await @@ -101,6 +99,7 @@ async fn main() -> Result<()> { repository: manifest .repository .ok_or_else(|| anyhow!("missing repository in extension manifest"))?, + wasm_api_version: manifest.lib.version.map(|version| version.to_string()), })?; fs::remove_dir_all(&archive_dir)?; fs::write(output_dir.join("manifest.json"), manifest_json.as_bytes())?; @@ -108,89 +107,6 @@ async fn main() -> Result<()> { Ok(()) } -fn populate_default_paths(manifest: &mut ExtensionManifest, extension_path: &Path) -> Result<()> { - // For legacy extensions on the v0 schema (aka, using `extension.json`), clear out any existing - // contents of the computed fields, since we don't care what the existing values are. - if manifest.schema_version == 0 { - manifest.languages.clear(); - manifest.grammars.clear(); - manifest.themes.clear(); - } - - let cargo_toml_path = extension_path.join("Cargo.toml"); - if cargo_toml_path.exists() { - manifest.lib.kind = Some(ExtensionLibraryKind::Rust); - } - - let languages_dir = extension_path.join("languages"); - if languages_dir.exists() { - for entry in fs::read_dir(&languages_dir).context("failed to list languages dir")? { - let entry = entry?; - let language_dir = entry.path(); - let config_path = language_dir.join("config.toml"); - if config_path.exists() { - let relative_language_dir = - language_dir.strip_prefix(extension_path)?.to_path_buf(); - if !manifest.languages.contains(&relative_language_dir) { - manifest.languages.push(relative_language_dir); - } - } - } - } - - let themes_dir = extension_path.join("themes"); - if themes_dir.exists() { - for entry in fs::read_dir(&themes_dir).context("failed to list themes dir")? { - let entry = entry?; - let theme_path = entry.path(); - if theme_path.extension() == Some("json".as_ref()) { - let relative_theme_path = theme_path.strip_prefix(extension_path)?.to_path_buf(); - if !manifest.themes.contains(&relative_theme_path) { - manifest.themes.push(relative_theme_path); - } - } - } - } - - // For legacy extensions on the v0 schema (aka, using `extension.json`), we want to populate the grammars in - // the manifest using the contents of the `grammars` directory. - if manifest.schema_version == 0 { - let grammars_dir = extension_path.join("grammars"); - if grammars_dir.exists() { - for entry in fs::read_dir(&grammars_dir).context("failed to list grammars dir")? { - let entry = entry?; - let grammar_path = entry.path(); - if grammar_path.extension() == Some("toml".as_ref()) { - #[derive(Deserialize)] - struct GrammarConfigToml { - pub repository: String, - pub commit: String, - } - - let grammar_config = fs::read_to_string(&grammar_path)?; - let grammar_config: GrammarConfigToml = toml::from_str(&grammar_config)?; - - let grammar_name = grammar_path - .file_stem() - .and_then(|stem| stem.to_str()) - .ok_or_else(|| anyhow!("no grammar name"))?; - if !manifest.grammars.contains_key(grammar_name) { - manifest.grammars.insert( - grammar_name.into(), - GrammarManifestEntry { - repository: grammar_config.repository, - rev: grammar_config.commit, - }, - ); - } - } - } - } - } - - Ok(()) -} - async fn copy_extension_resources( manifest: &ExtensionManifest, extension_path: &Path, diff --git a/crates/rpc/Cargo.toml b/crates/rpc/Cargo.toml index f41be65de1..b197073b7a 100644 --- a/crates/rpc/Cargo.toml +++ b/crates/rpc/Cargo.toml @@ -20,6 +20,7 @@ test-support = ["collections/test-support", "gpui/test-support"] anyhow.workspace = true async-tungstenite = "0.16" base64.workspace = true +chrono.workspace = true collections.workspace = true futures.workspace = true gpui = { workspace = true, optional = true } diff --git a/crates/rpc/src/extension.rs b/crates/rpc/src/extension.rs index 01915225b2..7313cbc7e5 100644 --- a/crates/rpc/src/extension.rs +++ b/crates/rpc/src/extension.rs @@ -1,6 +1,7 @@ +use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Debug, PartialEq)] pub struct ExtensionApiManifest { pub name: String, pub version: String, @@ -8,4 +9,14 @@ pub struct ExtensionApiManifest { pub authors: Vec, pub repository: String, pub schema_version: Option, + pub wasm_api_version: Option, +} + +#[derive(Debug, Serialize, PartialEq)] +pub struct ExtensionMetadata { + pub id: String, + #[serde(flatten)] + pub manifest: ExtensionApiManifest, + pub published_at: DateTime, + pub download_count: u64, } diff --git a/crates/telemetry_events/src/telemetry_events.rs b/crates/telemetry_events/src/telemetry_events.rs index d2ea0610db..f27e9a49e0 100644 --- a/crates/telemetry_events/src/telemetry_events.rs +++ b/crates/telemetry_events/src/telemetry_events.rs @@ -1,6 +1,5 @@ -use std::fmt::Display; - use serde::{Deserialize, Serialize}; +use std::{fmt::Display, sync::Arc}; use util::SemanticVersion; #[derive(Serialize, Deserialize, Debug)] @@ -61,6 +60,7 @@ pub enum Event { Memory(MemoryEvent), App(AppEvent), Setting(SettingEvent), + Extension(ExtensionEvent), Edit(EditEvent), Action(ActionEvent), } @@ -125,6 +125,12 @@ pub struct SettingEvent { pub value: String, } +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct ExtensionEvent { + pub extension_id: Arc, + pub version: Arc, +} + #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub struct AppEvent { pub operation: String, diff --git a/crates/util/src/semantic_version.rs b/crates/util/src/semantic_version.rs index 2d149f1d54..77c1a88ed2 100644 --- a/crates/util/src/semantic_version.rs +++ b/crates/util/src/semantic_version.rs @@ -4,10 +4,10 @@ use std::{ }; use anyhow::{anyhow, Result}; -use serde::Serialize; +use serde::{de::Error, Deserialize, Serialize}; /// A datastructure representing a semantic version number -#[derive(Clone, Copy, Debug, Default, Eq, Ord, PartialEq, PartialOrd, Serialize)] +#[derive(Clone, Copy, Debug, Default, Eq, Ord, PartialEq, PartialOrd)] pub struct SemanticVersion { pub major: usize, pub minor: usize, @@ -61,3 +61,23 @@ impl Display for SemanticVersion { write!(f, "{}.{}.{}", self.major, self.minor, self.patch) } } + +impl Serialize for SemanticVersion { + fn serialize(&self, serializer: S) -> std::prelude::v1::Result + where + S: serde::Serializer, + { + serializer.serialize_str(&self.to_string()) + } +} + +impl<'de> Deserialize<'de> for SemanticVersion { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let string = String::deserialize(deserializer)?; + Self::from_str(&string) + .map_err(|_| Error::custom(format!("Invalid version string \"{string}\""))) + } +} diff --git a/crates/zed/src/main.rs b/crates/zed/src/main.rs index 008fe9f66f..d72001baa2 100644 --- a/crates/zed/src/main.rs +++ b/crates/zed/src/main.rs @@ -179,7 +179,7 @@ fn main() { extension::init( fs.clone(), - http.clone(), + client.clone(), node_runtime.clone(), languages.clone(), ThemeRegistry::global(cx),