mirror of
https://github.com/zed-industries/zed.git
synced 2024-09-18 18:08:07 +03:00
Hang diagnostics (#11190)
Release Notes: - Added diagnostics for main-thread hangs on macOS. These are only enabled if you've opted into diagnostics. --------- Co-authored-by: Mikayla <mikayla@zed.dev>
This commit is contained in:
parent
04cd8dd0f2
commit
0697b417a0
2
Cargo.lock
generated
2
Cargo.lock
generated
@ -12701,6 +12701,7 @@ dependencies = [
|
|||||||
"markdown_preview",
|
"markdown_preview",
|
||||||
"menu",
|
"menu",
|
||||||
"mimalloc",
|
"mimalloc",
|
||||||
|
"nix 0.28.0",
|
||||||
"node_runtime",
|
"node_runtime",
|
||||||
"notifications",
|
"notifications",
|
||||||
"outline",
|
"outline",
|
||||||
@ -12723,6 +12724,7 @@ dependencies = [
|
|||||||
"tab_switcher",
|
"tab_switcher",
|
||||||
"task",
|
"task",
|
||||||
"tasks_ui",
|
"tasks_ui",
|
||||||
|
"telemetry_events",
|
||||||
"terminal_view",
|
"terminal_view",
|
||||||
"theme",
|
"theme",
|
||||||
"theme_selector",
|
"theme_selector",
|
||||||
|
@ -284,6 +284,7 @@ lazy_static = "1.4.0"
|
|||||||
linkify = "0.10.0"
|
linkify = "0.10.0"
|
||||||
log = { version = "0.4.16", features = ["kv_unstable_serde"] }
|
log = { version = "0.4.16", features = ["kv_unstable_serde"] }
|
||||||
nanoid = "0.4"
|
nanoid = "0.4"
|
||||||
|
nix = "0.28"
|
||||||
ordered-float = "2.1.1"
|
ordered-float = "2.1.1"
|
||||||
palette = { version = "0.7.5", default-features = false, features = ["std"] }
|
palette = { version = "0.7.5", default-features = false, features = ["std"] }
|
||||||
parking_lot = "0.12.1"
|
parking_lot = "0.12.1"
|
||||||
|
@ -421,7 +421,7 @@ impl Telemetry {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
let Some(checksum_seed) = &*ZED_CLIENT_CHECKSUM_SEED else {
|
if ZED_CLIENT_CHECKSUM_SEED.is_none() {
|
||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -466,15 +466,9 @@ impl Telemetry {
|
|||||||
serde_json::to_writer(&mut json_bytes, &request_body)?;
|
serde_json::to_writer(&mut json_bytes, &request_body)?;
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut summer = Sha256::new();
|
let Some(checksum) = calculate_json_checksum(&json_bytes) else {
|
||||||
summer.update(checksum_seed);
|
return Ok(());
|
||||||
summer.update(&json_bytes);
|
};
|
||||||
summer.update(checksum_seed);
|
|
||||||
let mut checksum = String::new();
|
|
||||||
for byte in summer.finalize().as_slice() {
|
|
||||||
use std::fmt::Write;
|
|
||||||
write!(&mut checksum, "{:02x}", byte).unwrap();
|
|
||||||
}
|
|
||||||
|
|
||||||
let request = http::Request::builder()
|
let request = http::Request::builder()
|
||||||
.method(Method::POST)
|
.method(Method::POST)
|
||||||
@ -657,3 +651,21 @@ mod tests {
|
|||||||
&& telemetry.state.lock().first_event_date_time.is_none()
|
&& telemetry.state.lock().first_event_date_time.is_none()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn calculate_json_checksum(json: &impl AsRef<[u8]>) -> Option<String> {
|
||||||
|
let Some(checksum_seed) = &*ZED_CLIENT_CHECKSUM_SEED else {
|
||||||
|
return None;
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut summer = Sha256::new();
|
||||||
|
summer.update(checksum_seed);
|
||||||
|
summer.update(&json);
|
||||||
|
summer.update(checksum_seed);
|
||||||
|
let mut checksum = String::new();
|
||||||
|
for byte in summer.finalize().as_slice() {
|
||||||
|
use std::fmt::Write;
|
||||||
|
write!(&mut checksum, "{:02x}", byte).unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
Some(checksum)
|
||||||
|
}
|
||||||
|
@ -18,11 +18,15 @@ use telemetry_events::{
|
|||||||
ActionEvent, AppEvent, AssistantEvent, CallEvent, CopilotEvent, CpuEvent, EditEvent,
|
ActionEvent, AppEvent, AssistantEvent, CallEvent, CopilotEvent, CpuEvent, EditEvent,
|
||||||
EditorEvent, Event, EventRequestBody, EventWrapper, ExtensionEvent, MemoryEvent, SettingEvent,
|
EditorEvent, Event, EventRequestBody, EventWrapper, ExtensionEvent, MemoryEvent, SettingEvent,
|
||||||
};
|
};
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
static CRASH_REPORTS_BUCKET: &str = "zed-crash-reports";
|
||||||
|
|
||||||
pub fn router() -> Router {
|
pub fn router() -> Router {
|
||||||
Router::new()
|
Router::new()
|
||||||
.route("/telemetry/events", post(post_events))
|
.route("/telemetry/events", post(post_events))
|
||||||
.route("/telemetry/crashes", post(post_crash))
|
.route("/telemetry/crashes", post(post_crash))
|
||||||
|
.route("/telemetry/hangs", post(post_hang))
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct ZedChecksumHeader(Vec<u8>);
|
pub struct ZedChecksumHeader(Vec<u8>);
|
||||||
@ -85,8 +89,6 @@ pub async fn post_crash(
|
|||||||
headers: HeaderMap,
|
headers: HeaderMap,
|
||||||
body: Bytes,
|
body: Bytes,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
static CRASH_REPORTS_BUCKET: &str = "zed-crash-reports";
|
|
||||||
|
|
||||||
let report = IpsFile::parse(&body)?;
|
let report = IpsFile::parse(&body)?;
|
||||||
let version_threshold = SemanticVersion::new(0, 123, 0);
|
let version_threshold = SemanticVersion::new(0, 123, 0);
|
||||||
|
|
||||||
@ -222,6 +224,107 @@ pub async fn post_crash(
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub async fn post_hang(
|
||||||
|
Extension(app): Extension<Arc<AppState>>,
|
||||||
|
TypedHeader(ZedChecksumHeader(checksum)): TypedHeader<ZedChecksumHeader>,
|
||||||
|
body: Bytes,
|
||||||
|
) -> Result<()> {
|
||||||
|
let Some(expected) = calculate_json_checksum(app.clone(), &body) else {
|
||||||
|
return Err(Error::Http(
|
||||||
|
StatusCode::INTERNAL_SERVER_ERROR,
|
||||||
|
"events not enabled".into(),
|
||||||
|
))?;
|
||||||
|
};
|
||||||
|
|
||||||
|
if checksum != expected {
|
||||||
|
return Err(Error::Http(
|
||||||
|
StatusCode::BAD_REQUEST,
|
||||||
|
"invalid checksum".into(),
|
||||||
|
))?;
|
||||||
|
}
|
||||||
|
|
||||||
|
let incident_id = Uuid::new_v4().to_string();
|
||||||
|
|
||||||
|
// dump JSON into S3 so we can get frame offsets if we need to.
|
||||||
|
if let Some(blob_store_client) = app.blob_store_client.as_ref() {
|
||||||
|
blob_store_client
|
||||||
|
.put_object()
|
||||||
|
.bucket(CRASH_REPORTS_BUCKET)
|
||||||
|
.key(incident_id.clone() + ".hang.json")
|
||||||
|
.acl(aws_sdk_s3::types::ObjectCannedAcl::PublicRead)
|
||||||
|
.body(ByteStream::from(body.to_vec()))
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.map_err(|e| log::error!("Failed to upload crash: {}", e))
|
||||||
|
.ok();
|
||||||
|
}
|
||||||
|
|
||||||
|
let report: telemetry_events::HangReport = serde_json::from_slice(&body).map_err(|err| {
|
||||||
|
log::error!("can't parse report json: {err}");
|
||||||
|
Error::Internal(anyhow!(err))
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let mut backtrace = "Possible hang detected on main threadL".to_string();
|
||||||
|
let unknown = "<unknown>".to_string();
|
||||||
|
for frame in report.backtrace.iter() {
|
||||||
|
backtrace.push_str(&format!("\n{}", frame.symbols.first().unwrap_or(&unknown)));
|
||||||
|
}
|
||||||
|
|
||||||
|
tracing::error!(
|
||||||
|
service = "client",
|
||||||
|
version = %report.app_version.unwrap_or_default().to_string(),
|
||||||
|
os_name = %report.os_name,
|
||||||
|
os_version = report.os_version.unwrap_or_default().to_string(),
|
||||||
|
incident_id = %incident_id,
|
||||||
|
installation_id = %report.installation_id.unwrap_or_default(),
|
||||||
|
backtrace = %backtrace,
|
||||||
|
"hang report");
|
||||||
|
|
||||||
|
if let Some(slack_panics_webhook) = app.config.slack_panics_webhook.clone() {
|
||||||
|
let payload = slack::WebhookBody::new(|w| {
|
||||||
|
w.add_section(|s| s.text(slack::Text::markdown("Possible Hang".to_string())))
|
||||||
|
.add_section(|s| {
|
||||||
|
s.add_field(slack::Text::markdown(format!(
|
||||||
|
"*Version:*\n {} ",
|
||||||
|
report.app_version.unwrap_or_default()
|
||||||
|
)))
|
||||||
|
.add_field({
|
||||||
|
let hostname = app.config.blob_store_url.clone().unwrap_or_default();
|
||||||
|
let hostname = hostname.strip_prefix("https://").unwrap_or_else(|| {
|
||||||
|
hostname.strip_prefix("http://").unwrap_or_default()
|
||||||
|
});
|
||||||
|
|
||||||
|
slack::Text::markdown(format!(
|
||||||
|
"*Incident:*\n<https://{}.{}/{}.hang.json|{}…>",
|
||||||
|
CRASH_REPORTS_BUCKET,
|
||||||
|
hostname,
|
||||||
|
incident_id,
|
||||||
|
incident_id.chars().take(8).collect::<String>(),
|
||||||
|
))
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.add_rich_text(|r| r.add_preformatted(|p| p.add_text(backtrace)))
|
||||||
|
});
|
||||||
|
let payload_json = serde_json::to_string(&payload).map_err(|err| {
|
||||||
|
log::error!("Failed to serialize payload to JSON: {err}");
|
||||||
|
Error::Internal(anyhow!(err))
|
||||||
|
})?;
|
||||||
|
|
||||||
|
reqwest::Client::new()
|
||||||
|
.post(slack_panics_webhook)
|
||||||
|
.header("Content-Type", "application/json")
|
||||||
|
.body(payload_json)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.map_err(|err| {
|
||||||
|
log::error!("Failed to send payload to Slack: {err}");
|
||||||
|
Error::Internal(anyhow!(err))
|
||||||
|
})?;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn post_events(
|
pub async fn post_events(
|
||||||
Extension(app): Extension<Arc<AppState>>,
|
Extension(app): Extension<Arc<AppState>>,
|
||||||
TypedHeader(ZedChecksumHeader(checksum)): TypedHeader<ZedChecksumHeader>,
|
TypedHeader(ZedChecksumHeader(checksum)): TypedHeader<ZedChecksumHeader>,
|
||||||
@ -235,19 +338,14 @@ pub async fn post_events(
|
|||||||
))?
|
))?
|
||||||
};
|
};
|
||||||
|
|
||||||
let Some(checksum_seed) = app.config.zed_client_checksum_seed.as_ref() else {
|
let Some(expected) = calculate_json_checksum(app.clone(), &body) else {
|
||||||
return Err(Error::Http(
|
return Err(Error::Http(
|
||||||
StatusCode::INTERNAL_SERVER_ERROR,
|
StatusCode::INTERNAL_SERVER_ERROR,
|
||||||
"events not enabled".into(),
|
"events not enabled".into(),
|
||||||
))?;
|
))?;
|
||||||
};
|
};
|
||||||
|
|
||||||
let mut summer = Sha256::new();
|
if checksum != expected {
|
||||||
summer.update(checksum_seed);
|
|
||||||
summer.update(&body);
|
|
||||||
summer.update(checksum_seed);
|
|
||||||
|
|
||||||
if &checksum != &summer.finalize()[..] {
|
|
||||||
return Err(Error::Http(
|
return Err(Error::Http(
|
||||||
StatusCode::BAD_REQUEST,
|
StatusCode::BAD_REQUEST,
|
||||||
"invalid checksum".into(),
|
"invalid checksum".into(),
|
||||||
@ -1061,3 +1159,15 @@ impl ActionEventRow {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn calculate_json_checksum(app: Arc<AppState>, json: &impl AsRef<[u8]>) -> Option<Vec<u8>> {
|
||||||
|
let Some(checksum_seed) = app.config.zed_client_checksum_seed.as_ref() else {
|
||||||
|
return None;
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut summer = Sha256::new();
|
||||||
|
summer.update(checksum_seed);
|
||||||
|
summer.update(&json);
|
||||||
|
summer.update(checksum_seed);
|
||||||
|
Some(summer.finalize().into_iter().collect())
|
||||||
|
}
|
||||||
|
@ -135,3 +135,21 @@ pub struct ExtensionEvent {
|
|||||||
pub struct AppEvent {
|
pub struct AppEvent {
|
||||||
pub operation: String,
|
pub operation: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||||
|
pub struct BacktraceFrame {
|
||||||
|
pub ip: usize,
|
||||||
|
pub symbol_addr: usize,
|
||||||
|
pub base: Option<usize>,
|
||||||
|
pub symbols: Vec<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||||
|
pub struct HangReport {
|
||||||
|
pub backtrace: Vec<BacktraceFrame>,
|
||||||
|
pub app_version: Option<SemanticVersion>,
|
||||||
|
pub os_name: String,
|
||||||
|
pub os_version: Option<SemanticVersion>,
|
||||||
|
pub architecture: String,
|
||||||
|
pub installation_id: Option<String>,
|
||||||
|
}
|
||||||
|
@ -62,6 +62,7 @@ log.workspace = true
|
|||||||
markdown_preview.workspace = true
|
markdown_preview.workspace = true
|
||||||
menu.workspace = true
|
menu.workspace = true
|
||||||
mimalloc = "0.1"
|
mimalloc = "0.1"
|
||||||
|
nix = {workspace = true, features = ["pthread"] }
|
||||||
node_runtime.workspace = true
|
node_runtime.workspace = true
|
||||||
notifications.workspace = true
|
notifications.workspace = true
|
||||||
outline.workspace = true
|
outline.workspace = true
|
||||||
@ -84,6 +85,7 @@ smol.workspace = true
|
|||||||
tab_switcher.workspace = true
|
tab_switcher.workspace = true
|
||||||
task.workspace = true
|
task.workspace = true
|
||||||
tasks_ui.workspace = true
|
tasks_ui.workspace = true
|
||||||
|
telemetry_events.workspace = true
|
||||||
terminal_view.workspace = true
|
terminal_view.workspace = true
|
||||||
theme.workspace = true
|
theme.workspace = true
|
||||||
theme_selector.workspace = true
|
theme_selector.workspace = true
|
||||||
|
@ -3,11 +3,10 @@
|
|||||||
// Disable command line from opening on release mode
|
// Disable command line from opening on release mode
|
||||||
#![cfg_attr(not(debug_assertions), windows_subsystem = "windows")]
|
#![cfg_attr(not(debug_assertions), windows_subsystem = "windows")]
|
||||||
|
|
||||||
|
mod reliability;
|
||||||
mod zed;
|
mod zed;
|
||||||
|
|
||||||
use anyhow::{anyhow, Context as _, Result};
|
use anyhow::{anyhow, Context as _, Result};
|
||||||
use backtrace::Backtrace;
|
|
||||||
use chrono::Utc;
|
|
||||||
use clap::{command, Parser};
|
use clap::{command, Parser};
|
||||||
use cli::FORCE_CLI_MODE_ENV_VAR_NAME;
|
use cli::FORCE_CLI_MODE_ENV_VAR_NAME;
|
||||||
use client::{parse_zed_link, telemetry::Telemetry, Client, DevServerToken, UserStore};
|
use client::{parse_zed_link, telemetry::Telemetry, Client, DevServerToken, UserStore};
|
||||||
@ -19,11 +18,8 @@ use editor::{Editor, EditorMode};
|
|||||||
use env_logger::Builder;
|
use env_logger::Builder;
|
||||||
use fs::RealFs;
|
use fs::RealFs;
|
||||||
use futures::{future, StreamExt};
|
use futures::{future, StreamExt};
|
||||||
use gpui::{
|
use gpui::{App, AppContext, AsyncAppContext, Context, Task, ViewContext, VisualContext};
|
||||||
App, AppContext, AsyncAppContext, Context, SemanticVersion, Task, ViewContext, VisualContext,
|
|
||||||
};
|
|
||||||
use image_viewer;
|
use image_viewer;
|
||||||
use isahc::{prelude::Configurable, Request};
|
|
||||||
use language::LanguageRegistry;
|
use language::LanguageRegistry;
|
||||||
use log::LevelFilter;
|
use log::LevelFilter;
|
||||||
|
|
||||||
@ -31,8 +27,7 @@ use assets::Assets;
|
|||||||
use mimalloc::MiMalloc;
|
use mimalloc::MiMalloc;
|
||||||
use node_runtime::RealNodeRuntime;
|
use node_runtime::RealNodeRuntime;
|
||||||
use parking_lot::Mutex;
|
use parking_lot::Mutex;
|
||||||
use release_channel::{AppCommitSha, ReleaseChannel, RELEASE_CHANNEL};
|
use release_channel::AppCommitSha;
|
||||||
use serde::{Deserialize, Serialize};
|
|
||||||
use settings::{
|
use settings::{
|
||||||
default_settings, handle_settings_file_changes, watch_config_file, Settings, SettingsStore,
|
default_settings, handle_settings_file_changes, watch_config_file, Settings, SettingsStore,
|
||||||
};
|
};
|
||||||
@ -40,22 +35,16 @@ use simplelog::ConfigBuilder;
|
|||||||
use smol::process::Command;
|
use smol::process::Command;
|
||||||
use std::{
|
use std::{
|
||||||
env,
|
env,
|
||||||
ffi::OsStr,
|
|
||||||
fs::OpenOptions,
|
fs::OpenOptions,
|
||||||
io::{IsTerminal, Write},
|
io::{IsTerminal, Write},
|
||||||
panic,
|
|
||||||
path::Path,
|
path::Path,
|
||||||
sync::{
|
sync::Arc,
|
||||||
atomic::{AtomicU32, Ordering},
|
|
||||||
Arc,
|
|
||||||
},
|
|
||||||
thread,
|
|
||||||
};
|
};
|
||||||
use theme::{ActiveTheme, SystemAppearance, ThemeRegistry, ThemeSettings};
|
use theme::{ActiveTheme, SystemAppearance, ThemeRegistry, ThemeSettings};
|
||||||
use util::{
|
use util::{
|
||||||
http::{HttpClient, HttpClientWithUrl},
|
http::HttpClientWithUrl,
|
||||||
maybe, parse_env_output,
|
maybe, parse_env_output,
|
||||||
paths::{self, CRASHES_DIR, CRASHES_RETIRED_DIR},
|
paths::{self},
|
||||||
ResultExt, TryFutureExt,
|
ResultExt, TryFutureExt,
|
||||||
};
|
};
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
@ -93,7 +82,18 @@ fn init_headless(dev_server_token: DevServerToken) {
|
|||||||
}
|
}
|
||||||
init_logger();
|
init_logger();
|
||||||
|
|
||||||
App::new().run(|cx| {
|
let app = App::new();
|
||||||
|
|
||||||
|
let session_id = Uuid::new_v4().to_string();
|
||||||
|
let (installation_id, _) = app
|
||||||
|
.background_executor()
|
||||||
|
.block(installation_id())
|
||||||
|
.ok()
|
||||||
|
.unzip();
|
||||||
|
|
||||||
|
reliability::init_panic_hook(&app, installation_id.clone(), session_id.clone());
|
||||||
|
|
||||||
|
app.run(|cx| {
|
||||||
release_channel::init(env!("CARGO_PKG_VERSION"), cx);
|
release_channel::init(env!("CARGO_PKG_VERSION"), cx);
|
||||||
if let Some(build_sha) = option_env!("ZED_COMMIT_SHA") {
|
if let Some(build_sha) = option_env!("ZED_COMMIT_SHA") {
|
||||||
AppCommitSha::set_global(AppCommitSha(build_sha.into()), cx);
|
AppCommitSha::set_global(AppCommitSha(build_sha.into()), cx);
|
||||||
@ -145,12 +145,7 @@ fn init_headless(dev_server_token: DevServerToken) {
|
|||||||
);
|
);
|
||||||
handle_settings_file_changes(user_settings_file_rx, cx);
|
handle_settings_file_changes(user_settings_file_rx, cx);
|
||||||
|
|
||||||
let (installation_id, _) = cx
|
reliability::init(client.http_client(), installation_id, cx);
|
||||||
.background_executor()
|
|
||||||
.block(installation_id())
|
|
||||||
.ok()
|
|
||||||
.unzip();
|
|
||||||
upload_panics_and_crashes(client.http_client(), installation_id, cx);
|
|
||||||
|
|
||||||
headless::init(
|
headless::init(
|
||||||
client.clone(),
|
client.clone(),
|
||||||
@ -189,7 +184,7 @@ fn init_ui(args: Args) {
|
|||||||
.ok()
|
.ok()
|
||||||
.unzip();
|
.unzip();
|
||||||
let session_id = Uuid::new_v4().to_string();
|
let session_id = Uuid::new_v4().to_string();
|
||||||
init_panic_hook(&app, installation_id.clone(), session_id.clone());
|
reliability::init_panic_hook(&app, installation_id.clone(), session_id.clone());
|
||||||
|
|
||||||
let git_binary_path = if option_env!("ZED_BUNDLE").as_deref() == Some("true") {
|
let git_binary_path = if option_env!("ZED_BUNDLE").as_deref() == Some("true") {
|
||||||
app.path_for_auxiliary_executable("git")
|
app.path_for_auxiliary_executable("git")
|
||||||
@ -386,7 +381,7 @@ fn init_ui(args: Args) {
|
|||||||
cx.set_menus(app_menus());
|
cx.set_menus(app_menus());
|
||||||
initialize_workspace(app_state.clone(), cx);
|
initialize_workspace(app_state.clone(), cx);
|
||||||
|
|
||||||
upload_panics_and_crashes(client.http_client(), installation_id, cx);
|
reliability::init(client.http_client(), installation_id, cx);
|
||||||
|
|
||||||
cx.activate(true);
|
cx.activate(true);
|
||||||
|
|
||||||
@ -688,317 +683,6 @@ fn init_stdout_logger() {
|
|||||||
})
|
})
|
||||||
.init();
|
.init();
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Serialize, Deserialize)]
|
|
||||||
struct LocationData {
|
|
||||||
file: String,
|
|
||||||
line: u32,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Serialize, Deserialize)]
|
|
||||||
struct Panic {
|
|
||||||
thread: String,
|
|
||||||
payload: String,
|
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
|
||||||
location_data: Option<LocationData>,
|
|
||||||
backtrace: Vec<String>,
|
|
||||||
app_version: String,
|
|
||||||
release_channel: String,
|
|
||||||
os_name: String,
|
|
||||||
os_version: Option<String>,
|
|
||||||
architecture: String,
|
|
||||||
panicked_on: i64,
|
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
|
||||||
installation_id: Option<String>,
|
|
||||||
session_id: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Serialize)]
|
|
||||||
struct PanicRequest {
|
|
||||||
panic: Panic,
|
|
||||||
}
|
|
||||||
|
|
||||||
static PANIC_COUNT: AtomicU32 = AtomicU32::new(0);
|
|
||||||
|
|
||||||
fn init_panic_hook(app: &App, installation_id: Option<String>, session_id: String) {
|
|
||||||
let is_pty = stdout_is_a_pty();
|
|
||||||
let app_metadata = app.metadata();
|
|
||||||
|
|
||||||
panic::set_hook(Box::new(move |info| {
|
|
||||||
let prior_panic_count = PANIC_COUNT.fetch_add(1, Ordering::SeqCst);
|
|
||||||
if prior_panic_count > 0 {
|
|
||||||
// Give the panic-ing thread time to write the panic file
|
|
||||||
loop {
|
|
||||||
std::thread::yield_now();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let thread = thread::current();
|
|
||||||
let thread_name = thread.name().unwrap_or("<unnamed>");
|
|
||||||
|
|
||||||
let payload = info
|
|
||||||
.payload()
|
|
||||||
.downcast_ref::<&str>()
|
|
||||||
.map(|s| s.to_string())
|
|
||||||
.or_else(|| info.payload().downcast_ref::<String>().map(|s| s.clone()))
|
|
||||||
.unwrap_or_else(|| "Box<Any>".to_string());
|
|
||||||
|
|
||||||
if *release_channel::RELEASE_CHANNEL == ReleaseChannel::Dev {
|
|
||||||
let location = info.location().unwrap();
|
|
||||||
let backtrace = Backtrace::new();
|
|
||||||
eprintln!(
|
|
||||||
"Thread {:?} panicked with {:?} at {}:{}:{}\n{:?}",
|
|
||||||
thread_name,
|
|
||||||
payload,
|
|
||||||
location.file(),
|
|
||||||
location.line(),
|
|
||||||
location.column(),
|
|
||||||
backtrace,
|
|
||||||
);
|
|
||||||
std::process::exit(-1);
|
|
||||||
}
|
|
||||||
|
|
||||||
let app_version = if let Some(version) = app_metadata.app_version {
|
|
||||||
version.to_string()
|
|
||||||
} else {
|
|
||||||
option_env!("CARGO_PKG_VERSION")
|
|
||||||
.unwrap_or("dev")
|
|
||||||
.to_string()
|
|
||||||
};
|
|
||||||
|
|
||||||
let backtrace = Backtrace::new();
|
|
||||||
let mut backtrace = backtrace
|
|
||||||
.frames()
|
|
||||||
.iter()
|
|
||||||
.flat_map(|frame| {
|
|
||||||
frame
|
|
||||||
.symbols()
|
|
||||||
.iter()
|
|
||||||
.filter_map(|frame| Some(format!("{:#}", frame.name()?)))
|
|
||||||
})
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
|
|
||||||
// Strip out leading stack frames for rust panic-handling.
|
|
||||||
if let Some(ix) = backtrace
|
|
||||||
.iter()
|
|
||||||
.position(|name| name == "rust_begin_unwind")
|
|
||||||
{
|
|
||||||
backtrace.drain(0..=ix);
|
|
||||||
}
|
|
||||||
|
|
||||||
let panic_data = Panic {
|
|
||||||
thread: thread_name.into(),
|
|
||||||
payload,
|
|
||||||
location_data: info.location().map(|location| LocationData {
|
|
||||||
file: location.file().into(),
|
|
||||||
line: location.line(),
|
|
||||||
}),
|
|
||||||
app_version: app_version.to_string(),
|
|
||||||
release_channel: RELEASE_CHANNEL.display_name().into(),
|
|
||||||
os_name: app_metadata.os_name.into(),
|
|
||||||
os_version: app_metadata
|
|
||||||
.os_version
|
|
||||||
.as_ref()
|
|
||||||
.map(SemanticVersion::to_string),
|
|
||||||
architecture: env::consts::ARCH.into(),
|
|
||||||
panicked_on: Utc::now().timestamp_millis(),
|
|
||||||
backtrace,
|
|
||||||
installation_id: installation_id.clone(),
|
|
||||||
session_id: session_id.clone(),
|
|
||||||
};
|
|
||||||
|
|
||||||
if let Some(panic_data_json) = serde_json::to_string_pretty(&panic_data).log_err() {
|
|
||||||
log::error!("{}", panic_data_json);
|
|
||||||
}
|
|
||||||
|
|
||||||
if !is_pty {
|
|
||||||
if let Some(panic_data_json) = serde_json::to_string(&panic_data).log_err() {
|
|
||||||
let timestamp = chrono::Utc::now().format("%Y_%m_%d %H_%M_%S").to_string();
|
|
||||||
let panic_file_path = paths::LOGS_DIR.join(format!("zed-{}.panic", timestamp));
|
|
||||||
let panic_file = std::fs::OpenOptions::new()
|
|
||||||
.append(true)
|
|
||||||
.create(true)
|
|
||||||
.open(&panic_file_path)
|
|
||||||
.log_err();
|
|
||||||
if let Some(mut panic_file) = panic_file {
|
|
||||||
writeln!(&mut panic_file, "{}", panic_data_json).log_err();
|
|
||||||
panic_file.flush().log_err();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
std::process::abort();
|
|
||||||
}));
|
|
||||||
}
|
|
||||||
|
|
||||||
fn upload_panics_and_crashes(
|
|
||||||
http: Arc<HttpClientWithUrl>,
|
|
||||||
installation_id: Option<String>,
|
|
||||||
cx: &mut AppContext,
|
|
||||||
) {
|
|
||||||
let telemetry_settings = *client::TelemetrySettings::get_global(cx);
|
|
||||||
cx.background_executor()
|
|
||||||
.spawn(async move {
|
|
||||||
let most_recent_panic = upload_previous_panics(http.clone(), telemetry_settings)
|
|
||||||
.await
|
|
||||||
.log_err()
|
|
||||||
.flatten();
|
|
||||||
upload_previous_crashes(http, most_recent_panic, installation_id, telemetry_settings)
|
|
||||||
.await
|
|
||||||
.log_err()
|
|
||||||
})
|
|
||||||
.detach()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Uploads panics via `zed.dev`.
|
|
||||||
async fn upload_previous_panics(
|
|
||||||
http: Arc<HttpClientWithUrl>,
|
|
||||||
telemetry_settings: client::TelemetrySettings,
|
|
||||||
) -> Result<Option<(i64, String)>> {
|
|
||||||
let panic_report_url = http.build_url("/api/panic");
|
|
||||||
let mut children = smol::fs::read_dir(&*paths::LOGS_DIR).await?;
|
|
||||||
|
|
||||||
let mut most_recent_panic = None;
|
|
||||||
|
|
||||||
while let Some(child) = children.next().await {
|
|
||||||
let child = child?;
|
|
||||||
let child_path = child.path();
|
|
||||||
|
|
||||||
if child_path.extension() != Some(OsStr::new("panic")) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
let filename = if let Some(filename) = child_path.file_name() {
|
|
||||||
filename.to_string_lossy()
|
|
||||||
} else {
|
|
||||||
continue;
|
|
||||||
};
|
|
||||||
|
|
||||||
if !filename.starts_with("zed") {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
if telemetry_settings.diagnostics {
|
|
||||||
let panic_file_content = smol::fs::read_to_string(&child_path)
|
|
||||||
.await
|
|
||||||
.context("error reading panic file")?;
|
|
||||||
|
|
||||||
let panic: Option<Panic> = serde_json::from_str(&panic_file_content)
|
|
||||||
.ok()
|
|
||||||
.or_else(|| {
|
|
||||||
panic_file_content
|
|
||||||
.lines()
|
|
||||||
.next()
|
|
||||||
.and_then(|line| serde_json::from_str(line).ok())
|
|
||||||
})
|
|
||||||
.unwrap_or_else(|| {
|
|
||||||
log::error!("failed to deserialize panic file {:?}", panic_file_content);
|
|
||||||
None
|
|
||||||
});
|
|
||||||
|
|
||||||
if let Some(panic) = panic {
|
|
||||||
most_recent_panic = Some((panic.panicked_on, panic.payload.clone()));
|
|
||||||
|
|
||||||
let body = serde_json::to_string(&PanicRequest { panic }).unwrap();
|
|
||||||
|
|
||||||
let request = Request::post(&panic_report_url)
|
|
||||||
.redirect_policy(isahc::config::RedirectPolicy::Follow)
|
|
||||||
.header("Content-Type", "application/json")
|
|
||||||
.body(body.into())?;
|
|
||||||
let response = http.send(request).await.context("error sending panic")?;
|
|
||||||
if !response.status().is_success() {
|
|
||||||
log::error!("Error uploading panic to server: {}", response.status());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// We've done what we can, delete the file
|
|
||||||
std::fs::remove_file(child_path)
|
|
||||||
.context("error removing panic")
|
|
||||||
.log_err();
|
|
||||||
}
|
|
||||||
Ok::<_, anyhow::Error>(most_recent_panic)
|
|
||||||
}
|
|
||||||
|
|
||||||
static LAST_CRASH_UPLOADED: &'static str = "LAST_CRASH_UPLOADED";
|
|
||||||
|
|
||||||
/// upload crashes from apple's diagnostic reports to our server.
|
|
||||||
/// (only if telemetry is enabled)
|
|
||||||
async fn upload_previous_crashes(
|
|
||||||
http: Arc<HttpClientWithUrl>,
|
|
||||||
most_recent_panic: Option<(i64, String)>,
|
|
||||||
installation_id: Option<String>,
|
|
||||||
telemetry_settings: client::TelemetrySettings,
|
|
||||||
) -> Result<()> {
|
|
||||||
if !telemetry_settings.diagnostics {
|
|
||||||
return Ok(());
|
|
||||||
}
|
|
||||||
let last_uploaded = KEY_VALUE_STORE
|
|
||||||
.read_kvp(LAST_CRASH_UPLOADED)?
|
|
||||||
.unwrap_or("zed-2024-01-17-221900.ips".to_string()); // don't upload old crash reports from before we had this.
|
|
||||||
let mut uploaded = last_uploaded.clone();
|
|
||||||
|
|
||||||
let crash_report_url = http.build_zed_api_url("/telemetry/crashes", &[])?;
|
|
||||||
|
|
||||||
// crash directories are only set on MacOS
|
|
||||||
for dir in [&*CRASHES_DIR, &*CRASHES_RETIRED_DIR]
|
|
||||||
.iter()
|
|
||||||
.filter_map(|d| d.as_deref())
|
|
||||||
{
|
|
||||||
let mut children = smol::fs::read_dir(&dir).await?;
|
|
||||||
while let Some(child) = children.next().await {
|
|
||||||
let child = child?;
|
|
||||||
let Some(filename) = child
|
|
||||||
.path()
|
|
||||||
.file_name()
|
|
||||||
.map(|f| f.to_string_lossy().to_lowercase())
|
|
||||||
else {
|
|
||||||
continue;
|
|
||||||
};
|
|
||||||
|
|
||||||
if !filename.starts_with("zed-") || !filename.ends_with(".ips") {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
if filename <= last_uploaded {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
let body = smol::fs::read_to_string(&child.path())
|
|
||||||
.await
|
|
||||||
.context("error reading crash file")?;
|
|
||||||
|
|
||||||
let mut request = Request::post(&crash_report_url.to_string())
|
|
||||||
.redirect_policy(isahc::config::RedirectPolicy::Follow)
|
|
||||||
.header("Content-Type", "text/plain");
|
|
||||||
|
|
||||||
if let Some((panicked_on, payload)) = most_recent_panic.as_ref() {
|
|
||||||
request = request
|
|
||||||
.header("x-zed-panicked-on", format!("{}", panicked_on))
|
|
||||||
.header("x-zed-panic", payload)
|
|
||||||
}
|
|
||||||
if let Some(installation_id) = installation_id.as_ref() {
|
|
||||||
request = request.header("x-zed-installation-id", installation_id);
|
|
||||||
}
|
|
||||||
|
|
||||||
let request = request.body(body.into())?;
|
|
||||||
|
|
||||||
let response = http.send(request).await.context("error sending crash")?;
|
|
||||||
if !response.status().is_success() {
|
|
||||||
log::error!("Error uploading crash to server: {}", response.status());
|
|
||||||
}
|
|
||||||
|
|
||||||
if uploaded < filename {
|
|
||||||
uploaded = filename.clone();
|
|
||||||
KEY_VALUE_STORE
|
|
||||||
.write_kvp(LAST_CRASH_UPLOADED.to_string(), filename)
|
|
||||||
.await?;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn load_login_shell_environment() -> Result<()> {
|
async fn load_login_shell_environment() -> Result<()> {
|
||||||
let marker = "ZED_LOGIN_SHELL_START";
|
let marker = "ZED_LOGIN_SHELL_START";
|
||||||
let shell = env::var("SHELL").context(
|
let shell = env::var("SHELL").context(
|
||||||
|
536
crates/zed/src/reliability.rs
Normal file
536
crates/zed/src/reliability.rs
Normal file
@ -0,0 +1,536 @@
|
|||||||
|
use anyhow::{Context, Result};
|
||||||
|
use backtrace::{self, Backtrace};
|
||||||
|
use chrono::Utc;
|
||||||
|
use db::kvp::KEY_VALUE_STORE;
|
||||||
|
use gpui::{App, AppContext, SemanticVersion};
|
||||||
|
use isahc::config::Configurable;
|
||||||
|
|
||||||
|
use paths::{CRASHES_DIR, CRASHES_RETIRED_DIR};
|
||||||
|
use release_channel::ReleaseChannel;
|
||||||
|
use release_channel::RELEASE_CHANNEL;
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
use settings::Settings;
|
||||||
|
use smol::stream::StreamExt;
|
||||||
|
use std::{
|
||||||
|
env,
|
||||||
|
ffi::OsStr,
|
||||||
|
sync::{atomic::Ordering, Arc},
|
||||||
|
};
|
||||||
|
use std::{io::Write, panic, sync::atomic::AtomicU32, thread};
|
||||||
|
use util::{
|
||||||
|
http::{self, HttpClient, HttpClientWithUrl},
|
||||||
|
paths, ResultExt,
|
||||||
|
};
|
||||||
|
|
||||||
|
use crate::stdout_is_a_pty;
|
||||||
|
|
||||||
|
#[derive(Serialize, Deserialize)]
|
||||||
|
struct LocationData {
|
||||||
|
file: String,
|
||||||
|
line: u32,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize, Deserialize)]
|
||||||
|
struct Panic {
|
||||||
|
thread: String,
|
||||||
|
payload: String,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
location_data: Option<LocationData>,
|
||||||
|
backtrace: Vec<String>,
|
||||||
|
app_version: String,
|
||||||
|
release_channel: String,
|
||||||
|
os_name: String,
|
||||||
|
os_version: Option<String>,
|
||||||
|
architecture: String,
|
||||||
|
panicked_on: i64,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
installation_id: Option<String>,
|
||||||
|
session_id: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
struct PanicRequest {
|
||||||
|
panic: Panic,
|
||||||
|
}
|
||||||
|
|
||||||
|
static PANIC_COUNT: AtomicU32 = AtomicU32::new(0);
|
||||||
|
|
||||||
|
pub fn init_panic_hook(app: &App, installation_id: Option<String>, session_id: String) {
|
||||||
|
let is_pty = stdout_is_a_pty();
|
||||||
|
let app_metadata = app.metadata();
|
||||||
|
|
||||||
|
panic::set_hook(Box::new(move |info| {
|
||||||
|
let prior_panic_count = PANIC_COUNT.fetch_add(1, Ordering::SeqCst);
|
||||||
|
if prior_panic_count > 0 {
|
||||||
|
// Give the panic-ing thread time to write the panic file
|
||||||
|
loop {
|
||||||
|
std::thread::yield_now();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let thread = thread::current();
|
||||||
|
let thread_name = thread.name().unwrap_or("<unnamed>");
|
||||||
|
|
||||||
|
let payload = info
|
||||||
|
.payload()
|
||||||
|
.downcast_ref::<&str>()
|
||||||
|
.map(|s| s.to_string())
|
||||||
|
.or_else(|| info.payload().downcast_ref::<String>().map(|s| s.clone()))
|
||||||
|
.unwrap_or_else(|| "Box<Any>".to_string());
|
||||||
|
|
||||||
|
if *release_channel::RELEASE_CHANNEL == ReleaseChannel::Dev {
|
||||||
|
let location = info.location().unwrap();
|
||||||
|
let backtrace = Backtrace::new();
|
||||||
|
eprintln!(
|
||||||
|
"Thread {:?} panicked with {:?} at {}:{}:{}\n{:?}",
|
||||||
|
thread_name,
|
||||||
|
payload,
|
||||||
|
location.file(),
|
||||||
|
location.line(),
|
||||||
|
location.column(),
|
||||||
|
backtrace,
|
||||||
|
);
|
||||||
|
std::process::exit(-1);
|
||||||
|
}
|
||||||
|
|
||||||
|
let app_version = if let Some(version) = app_metadata.app_version {
|
||||||
|
version.to_string()
|
||||||
|
} else {
|
||||||
|
option_env!("CARGO_PKG_VERSION")
|
||||||
|
.unwrap_or("dev")
|
||||||
|
.to_string()
|
||||||
|
};
|
||||||
|
|
||||||
|
let backtrace = Backtrace::new();
|
||||||
|
let mut backtrace = backtrace
|
||||||
|
.frames()
|
||||||
|
.iter()
|
||||||
|
.flat_map(|frame| {
|
||||||
|
frame
|
||||||
|
.symbols()
|
||||||
|
.iter()
|
||||||
|
.filter_map(|frame| Some(format!("{:#}", frame.name()?)))
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
|
||||||
|
// Strip out leading stack frames for rust panic-handling.
|
||||||
|
if let Some(ix) = backtrace
|
||||||
|
.iter()
|
||||||
|
.position(|name| name == "rust_begin_unwind")
|
||||||
|
{
|
||||||
|
backtrace.drain(0..=ix);
|
||||||
|
}
|
||||||
|
|
||||||
|
let panic_data = Panic {
|
||||||
|
thread: thread_name.into(),
|
||||||
|
payload,
|
||||||
|
location_data: info.location().map(|location| LocationData {
|
||||||
|
file: location.file().into(),
|
||||||
|
line: location.line(),
|
||||||
|
}),
|
||||||
|
app_version: app_version.to_string(),
|
||||||
|
release_channel: RELEASE_CHANNEL.display_name().into(),
|
||||||
|
os_name: app_metadata.os_name.into(),
|
||||||
|
os_version: app_metadata
|
||||||
|
.os_version
|
||||||
|
.as_ref()
|
||||||
|
.map(SemanticVersion::to_string),
|
||||||
|
architecture: env::consts::ARCH.into(),
|
||||||
|
panicked_on: Utc::now().timestamp_millis(),
|
||||||
|
backtrace,
|
||||||
|
installation_id: installation_id.clone(),
|
||||||
|
session_id: session_id.clone(),
|
||||||
|
};
|
||||||
|
|
||||||
|
if let Some(panic_data_json) = serde_json::to_string_pretty(&panic_data).log_err() {
|
||||||
|
log::error!("{}", panic_data_json);
|
||||||
|
}
|
||||||
|
|
||||||
|
if !is_pty {
|
||||||
|
if let Some(panic_data_json) = serde_json::to_string(&panic_data).log_err() {
|
||||||
|
let timestamp = chrono::Utc::now().format("%Y_%m_%d %H_%M_%S").to_string();
|
||||||
|
let panic_file_path = paths::LOGS_DIR.join(format!("zed-{}.panic", timestamp));
|
||||||
|
let panic_file = std::fs::OpenOptions::new()
|
||||||
|
.append(true)
|
||||||
|
.create(true)
|
||||||
|
.open(&panic_file_path)
|
||||||
|
.log_err();
|
||||||
|
if let Some(mut panic_file) = panic_file {
|
||||||
|
writeln!(&mut panic_file, "{}", panic_data_json).log_err();
|
||||||
|
panic_file.flush().log_err();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
std::process::abort();
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn init(
|
||||||
|
http_client: Arc<HttpClientWithUrl>,
|
||||||
|
installation_id: Option<String>,
|
||||||
|
cx: &mut AppContext,
|
||||||
|
) {
|
||||||
|
#[cfg(target_os = "macos")]
|
||||||
|
monitor_main_thread_hangs(http_client.clone(), installation_id.clone(), cx);
|
||||||
|
|
||||||
|
upload_panics_and_crashes(http_client, installation_id, cx)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(target_os = "macos")]
|
||||||
|
pub fn monitor_main_thread_hangs(
|
||||||
|
http_client: Arc<HttpClientWithUrl>,
|
||||||
|
installation_id: Option<String>,
|
||||||
|
cx: &AppContext,
|
||||||
|
) {
|
||||||
|
use nix::sys::signal::{
|
||||||
|
sigaction, SaFlags, SigAction, SigHandler, SigSet,
|
||||||
|
Signal::{self, SIGUSR2},
|
||||||
|
};
|
||||||
|
|
||||||
|
use parking_lot::Mutex;
|
||||||
|
|
||||||
|
use std::{
|
||||||
|
ffi::c_int,
|
||||||
|
sync::{mpsc, OnceLock},
|
||||||
|
time::Duration,
|
||||||
|
};
|
||||||
|
use telemetry_events::{BacktraceFrame, HangReport};
|
||||||
|
use util::http::Method;
|
||||||
|
|
||||||
|
use nix::sys::pthread;
|
||||||
|
|
||||||
|
let foreground_executor = cx.foreground_executor();
|
||||||
|
let background_executor = cx.background_executor();
|
||||||
|
let telemetry_settings = *client::TelemetrySettings::get_global(cx);
|
||||||
|
let metadata = cx.app_metadata();
|
||||||
|
|
||||||
|
// Initialize SIGUSR2 handler to send a backrace to a channel.
|
||||||
|
let (backtrace_tx, backtrace_rx) = mpsc::channel();
|
||||||
|
static BACKTRACE: Mutex<Vec<backtrace::Frame>> = Mutex::new(Vec::new());
|
||||||
|
static BACKTRACE_SENDER: OnceLock<mpsc::Sender<()>> = OnceLock::new();
|
||||||
|
BACKTRACE_SENDER.get_or_init(|| backtrace_tx);
|
||||||
|
BACKTRACE.lock().reserve(100);
|
||||||
|
|
||||||
|
fn handle_backtrace_signal() {
|
||||||
|
unsafe {
|
||||||
|
extern "C" fn handle_sigusr2(_i: c_int) {
|
||||||
|
unsafe {
|
||||||
|
// ASYNC SIGNAL SAFETY: This lock is only accessed one other time,
|
||||||
|
// which can only be triggered by This signal handler. In addition,
|
||||||
|
// this signal handler is immediately removed by SA_RESETHAND, and this
|
||||||
|
// signal handler cannot be re-entrant due to to the SIGUSR2 mask defined
|
||||||
|
// below
|
||||||
|
let mut bt = BACKTRACE.lock();
|
||||||
|
bt.clear();
|
||||||
|
backtrace::trace_unsynchronized(|frame| {
|
||||||
|
if bt.len() < bt.capacity() {
|
||||||
|
bt.push(frame.clone());
|
||||||
|
true
|
||||||
|
} else {
|
||||||
|
false
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
BACKTRACE_SENDER.get().unwrap().send(()).ok();
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut mask = SigSet::empty();
|
||||||
|
mask.add(SIGUSR2);
|
||||||
|
sigaction(
|
||||||
|
Signal::SIGUSR2,
|
||||||
|
&SigAction::new(
|
||||||
|
SigHandler::Handler(handle_sigusr2),
|
||||||
|
SaFlags::SA_RESTART | SaFlags::SA_RESETHAND,
|
||||||
|
mask,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.log_err();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
handle_backtrace_signal();
|
||||||
|
let main_thread = pthread::pthread_self();
|
||||||
|
|
||||||
|
let (mut tx, mut rx) = futures::channel::mpsc::channel(3);
|
||||||
|
foreground_executor
|
||||||
|
.spawn(async move { while let Some(_) = rx.next().await {} })
|
||||||
|
.detach();
|
||||||
|
|
||||||
|
background_executor
|
||||||
|
.spawn({
|
||||||
|
let background_executor = background_executor.clone();
|
||||||
|
async move {
|
||||||
|
loop {
|
||||||
|
background_executor.timer(Duration::from_secs(1)).await;
|
||||||
|
match tx.try_send(()) {
|
||||||
|
Ok(_) => continue,
|
||||||
|
Err(e) => {
|
||||||
|
if e.into_send_error().is_full() {
|
||||||
|
pthread::pthread_kill(main_thread, SIGUSR2).log_err();
|
||||||
|
}
|
||||||
|
// Only detect the first hang
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.detach();
|
||||||
|
|
||||||
|
background_executor
|
||||||
|
.clone()
|
||||||
|
.spawn(async move {
|
||||||
|
loop {
|
||||||
|
while let Some(_) = backtrace_rx.recv().ok() {
|
||||||
|
if !telemetry_settings.diagnostics {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ASYNC SIGNAL SAFETY: This lock is only accessed _after_
|
||||||
|
// the backtrace transmitter has fired, which itself is only done
|
||||||
|
// by the signal handler. And due to SA_RESETHAND the signal handler
|
||||||
|
// will not run again until `handle_backtrace_signal` is called.
|
||||||
|
let raw_backtrace = BACKTRACE.lock().drain(..).collect::<Vec<_>>();
|
||||||
|
let backtrace: Vec<_> = raw_backtrace
|
||||||
|
.into_iter()
|
||||||
|
.map(|frame| {
|
||||||
|
let mut btf = BacktraceFrame {
|
||||||
|
ip: frame.ip() as usize,
|
||||||
|
symbol_addr: frame.symbol_address() as usize,
|
||||||
|
base: frame.module_base_address().map(|addr| addr as usize),
|
||||||
|
symbols: vec![],
|
||||||
|
};
|
||||||
|
|
||||||
|
backtrace::resolve_frame(&frame, |symbol| {
|
||||||
|
if let Some(name) = symbol.name() {
|
||||||
|
btf.symbols.push(name.to_string());
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
btf
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
// IMPORTANT: Don't move this to before `BACKTRACE.lock()`
|
||||||
|
handle_backtrace_signal();
|
||||||
|
|
||||||
|
log::error!(
|
||||||
|
"Suspected hang on main thread:\n{}",
|
||||||
|
backtrace
|
||||||
|
.iter()
|
||||||
|
.flat_map(|bt| bt.symbols.first().as_ref().map(|s| s.as_str()))
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join("\n")
|
||||||
|
);
|
||||||
|
|
||||||
|
let report = HangReport {
|
||||||
|
backtrace,
|
||||||
|
app_version: metadata.app_version,
|
||||||
|
os_name: metadata.os_name.to_owned(),
|
||||||
|
os_version: metadata.os_version,
|
||||||
|
architecture: env::consts::ARCH.into(),
|
||||||
|
installation_id: installation_id.clone(),
|
||||||
|
};
|
||||||
|
|
||||||
|
let Some(json_bytes) = serde_json::to_vec(&report).log_err() else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
|
||||||
|
let Some(checksum) = client::telemetry::calculate_json_checksum(&json_bytes)
|
||||||
|
else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
|
||||||
|
let Ok(url) = http_client.build_zed_api_url("/telemetry/hangs", &[]) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
|
||||||
|
let Ok(request) = http::Request::builder()
|
||||||
|
.method(Method::POST)
|
||||||
|
.uri(url.as_ref())
|
||||||
|
.header("x-zed-checksum", checksum)
|
||||||
|
.body(json_bytes.into())
|
||||||
|
else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
|
||||||
|
if let Some(response) = http_client.send(request).await.log_err() {
|
||||||
|
if response.status() != 200 {
|
||||||
|
log::error!("Failed to send hang report: HTTP {:?}", response.status());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.detach()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn upload_panics_and_crashes(
|
||||||
|
http: Arc<HttpClientWithUrl>,
|
||||||
|
installation_id: Option<String>,
|
||||||
|
cx: &mut AppContext,
|
||||||
|
) {
|
||||||
|
let telemetry_settings = *client::TelemetrySettings::get_global(cx);
|
||||||
|
cx.background_executor()
|
||||||
|
.spawn(async move {
|
||||||
|
let most_recent_panic = upload_previous_panics(http.clone(), telemetry_settings)
|
||||||
|
.await
|
||||||
|
.log_err()
|
||||||
|
.flatten();
|
||||||
|
upload_previous_crashes(http, most_recent_panic, installation_id, telemetry_settings)
|
||||||
|
.await
|
||||||
|
.log_err()
|
||||||
|
})
|
||||||
|
.detach()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Uploads panics via `zed.dev`.
|
||||||
|
async fn upload_previous_panics(
|
||||||
|
http: Arc<HttpClientWithUrl>,
|
||||||
|
telemetry_settings: client::TelemetrySettings,
|
||||||
|
) -> Result<Option<(i64, String)>> {
|
||||||
|
let panic_report_url = http.build_url("/api/panic");
|
||||||
|
let mut children = smol::fs::read_dir(&*paths::LOGS_DIR).await?;
|
||||||
|
|
||||||
|
let mut most_recent_panic = None;
|
||||||
|
|
||||||
|
while let Some(child) = children.next().await {
|
||||||
|
let child = child?;
|
||||||
|
let child_path = child.path();
|
||||||
|
|
||||||
|
if child_path.extension() != Some(OsStr::new("panic")) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let filename = if let Some(filename) = child_path.file_name() {
|
||||||
|
filename.to_string_lossy()
|
||||||
|
} else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
|
||||||
|
if !filename.starts_with("zed") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if telemetry_settings.diagnostics {
|
||||||
|
let panic_file_content = smol::fs::read_to_string(&child_path)
|
||||||
|
.await
|
||||||
|
.context("error reading panic file")?;
|
||||||
|
|
||||||
|
let panic: Option<Panic> = serde_json::from_str(&panic_file_content)
|
||||||
|
.ok()
|
||||||
|
.or_else(|| {
|
||||||
|
panic_file_content
|
||||||
|
.lines()
|
||||||
|
.next()
|
||||||
|
.and_then(|line| serde_json::from_str(line).ok())
|
||||||
|
})
|
||||||
|
.unwrap_or_else(|| {
|
||||||
|
log::error!("failed to deserialize panic file {:?}", panic_file_content);
|
||||||
|
None
|
||||||
|
});
|
||||||
|
|
||||||
|
if let Some(panic) = panic {
|
||||||
|
most_recent_panic = Some((panic.panicked_on, panic.payload.clone()));
|
||||||
|
|
||||||
|
let body = serde_json::to_string(&PanicRequest { panic }).unwrap();
|
||||||
|
|
||||||
|
let request = http::Request::post(&panic_report_url)
|
||||||
|
.redirect_policy(isahc::config::RedirectPolicy::Follow)
|
||||||
|
.header("Content-Type", "application/json")
|
||||||
|
.body(body.into())?;
|
||||||
|
let response = http.send(request).await.context("error sending panic")?;
|
||||||
|
if !response.status().is_success() {
|
||||||
|
log::error!("Error uploading panic to server: {}", response.status());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// We've done what we can, delete the file
|
||||||
|
std::fs::remove_file(child_path)
|
||||||
|
.context("error removing panic")
|
||||||
|
.log_err();
|
||||||
|
}
|
||||||
|
Ok::<_, anyhow::Error>(most_recent_panic)
|
||||||
|
}
|
||||||
|
|
||||||
|
static LAST_CRASH_UPLOADED: &'static str = "LAST_CRASH_UPLOADED";
|
||||||
|
|
||||||
|
/// upload crashes from apple's diagnostic reports to our server.
|
||||||
|
/// (only if telemetry is enabled)
|
||||||
|
async fn upload_previous_crashes(
|
||||||
|
http: Arc<HttpClientWithUrl>,
|
||||||
|
most_recent_panic: Option<(i64, String)>,
|
||||||
|
installation_id: Option<String>,
|
||||||
|
telemetry_settings: client::TelemetrySettings,
|
||||||
|
) -> Result<()> {
|
||||||
|
if !telemetry_settings.diagnostics {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
let last_uploaded = KEY_VALUE_STORE
|
||||||
|
.read_kvp(LAST_CRASH_UPLOADED)?
|
||||||
|
.unwrap_or("zed-2024-01-17-221900.ips".to_string()); // don't upload old crash reports from before we had this.
|
||||||
|
let mut uploaded = last_uploaded.clone();
|
||||||
|
|
||||||
|
let crash_report_url = http.build_zed_api_url("/telemetry/crashes", &[])?;
|
||||||
|
|
||||||
|
// crash directories are only set on MacOS
|
||||||
|
for dir in [&*CRASHES_DIR, &*CRASHES_RETIRED_DIR]
|
||||||
|
.iter()
|
||||||
|
.filter_map(|d| d.as_deref())
|
||||||
|
{
|
||||||
|
let mut children = smol::fs::read_dir(&dir).await?;
|
||||||
|
while let Some(child) = children.next().await {
|
||||||
|
let child = child?;
|
||||||
|
let Some(filename) = child
|
||||||
|
.path()
|
||||||
|
.file_name()
|
||||||
|
.map(|f| f.to_string_lossy().to_lowercase())
|
||||||
|
else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
|
||||||
|
if !filename.starts_with("zed-") || !filename.ends_with(".ips") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if filename <= last_uploaded {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let body = smol::fs::read_to_string(&child.path())
|
||||||
|
.await
|
||||||
|
.context("error reading crash file")?;
|
||||||
|
|
||||||
|
let mut request = http::Request::post(&crash_report_url.to_string())
|
||||||
|
.redirect_policy(isahc::config::RedirectPolicy::Follow)
|
||||||
|
.header("Content-Type", "text/plain");
|
||||||
|
|
||||||
|
if let Some((panicked_on, payload)) = most_recent_panic.as_ref() {
|
||||||
|
request = request
|
||||||
|
.header("x-zed-panicked-on", format!("{}", panicked_on))
|
||||||
|
.header("x-zed-panic", payload)
|
||||||
|
}
|
||||||
|
if let Some(installation_id) = installation_id.as_ref() {
|
||||||
|
request = request.header("x-zed-installation-id", installation_id);
|
||||||
|
}
|
||||||
|
|
||||||
|
let request = request.body(body.into())?;
|
||||||
|
|
||||||
|
let response = http.send(request).await.context("error sending crash")?;
|
||||||
|
if !response.status().is_success() {
|
||||||
|
log::error!("Error uploading crash to server: {}", response.status());
|
||||||
|
}
|
||||||
|
|
||||||
|
if uploaded < filename {
|
||||||
|
uploaded = filename.clone();
|
||||||
|
KEY_VALUE_STORE
|
||||||
|
.write_kvp(LAST_CRASH_UPLOADED.to_string(), filename)
|
||||||
|
.await?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
Loading…
Reference in New Issue
Block a user