rustdoc: Automatically index crates (#13014)

This PR removes the need to use `/rustdoc --index <CRATE_NAME>` and
instead indexes the crates once they are referenced.

As soon as the first `:` is added after the crate name, the indexing
will kick off in the background and update the index as it goes.

Release Notes:

- N/A
This commit is contained in:
Marshall Bowers 2024-06-13 18:30:15 -04:00 committed by GitHub
parent e0c1ab650e
commit 86167138a9
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 70 additions and 102 deletions

1
Cargo.lock generated
View File

@ -8723,6 +8723,7 @@ dependencies = [
"http 0.1.0", "http 0.1.0",
"indexmap 1.9.3", "indexmap 1.9.3",
"indoc", "indoc",
"parking_lot",
"pretty_assertions", "pretty_assertions",
"serde", "serde",
"strum", "strum",

View File

@ -13,6 +13,7 @@ use project::{Project, ProjectPath};
use rustdoc::LocalProvider; use rustdoc::LocalProvider;
use rustdoc::{convert_rustdoc_to_markdown, RustdocStore}; use rustdoc::{convert_rustdoc_to_markdown, RustdocStore};
use ui::{prelude::*, ButtonLike, ElevationIndex}; use ui::{prelude::*, ButtonLike, ElevationIndex};
use util::{maybe, ResultExt};
use workspace::Workspace; use workspace::Workspace;
#[derive(Debug, Clone, Copy)] #[derive(Debug, Clone, Copy)]
@ -118,11 +119,36 @@ impl SlashCommand for RustdocSlashCommand {
&self, &self,
query: String, query: String,
_cancel: Arc<AtomicBool>, _cancel: Arc<AtomicBool>,
_workspace: Option<WeakView<Workspace>>, workspace: Option<WeakView<Workspace>>,
cx: &mut AppContext, cx: &mut AppContext,
) -> Task<Result<Vec<String>>> { ) -> Task<Result<Vec<String>>> {
let index_provider_deps = maybe!({
let workspace = workspace.ok_or_else(|| anyhow!("no workspace"))?;
let workspace = workspace
.upgrade()
.ok_or_else(|| anyhow!("workspace was dropped"))?;
let project = workspace.read(cx).project().clone();
let fs = project.read(cx).fs().clone();
let cargo_workspace_root = Self::path_to_cargo_toml(project, cx)
.and_then(|path| path.parent().map(|path| path.to_path_buf()))
.ok_or_else(|| anyhow!("no Cargo workspace root found"))?;
anyhow::Ok((fs, cargo_workspace_root))
});
let store = RustdocStore::global(cx); let store = RustdocStore::global(cx);
cx.background_executor().spawn(async move { cx.background_executor().spawn(async move {
if let Some((crate_name, rest)) = query.split_once(':') {
if rest.is_empty() {
if let Some((fs, cargo_workspace_root)) = index_provider_deps.log_err() {
let provider = Box::new(LocalProvider::new(fs, cargo_workspace_root));
// We don't need to hold onto this task, as the `RustdocStore` will hold it
// until it completes.
let _ = store.clone().index(crate_name.to_string(), provider);
}
}
}
let items = store.search(query).await; let items = store.search(query).await;
Ok(items) Ok(items)
}) })
@ -147,65 +173,7 @@ impl SlashCommand for RustdocSlashCommand {
let http_client = workspace.read(cx).client().http_client(); let http_client = workspace.read(cx).client().http_client();
let path_to_cargo_toml = Self::path_to_cargo_toml(project, cx); let path_to_cargo_toml = Self::path_to_cargo_toml(project, cx);
let mut item_path = String::new(); let mut path_components = argument.split("::");
let mut crate_name_to_index = None;
let mut args = argument.split(' ').map(|word| word.trim());
while let Some(arg) = args.next() {
if arg == "--index" {
let Some(crate_name) = args.next() else {
return Task::ready(Err(anyhow!("no crate name provided to --index")));
};
crate_name_to_index = Some(crate_name.to_string());
continue;
}
item_path.push_str(arg);
}
if let Some(crate_name_to_index) = crate_name_to_index {
let index_task = cx.background_executor().spawn({
let rustdoc_store = RustdocStore::global(cx);
let fs = fs.clone();
let crate_name_to_index = crate_name_to_index.clone();
async move {
let cargo_workspace_root = path_to_cargo_toml
.and_then(|path| path.parent().map(|path| path.to_path_buf()))
.ok_or_else(|| anyhow!("no Cargo workspace root found"))?;
let provider = Box::new(LocalProvider::new(fs, cargo_workspace_root));
rustdoc_store
.index(crate_name_to_index.clone(), provider)
.await?;
anyhow::Ok(format!("Indexed {crate_name_to_index}"))
}
});
return cx.foreground_executor().spawn(async move {
let text = index_task.await?;
let range = 0..text.len();
Ok(SlashCommandOutput {
text,
sections: vec![SlashCommandOutputSection {
range,
render_placeholder: Arc::new(move |id, unfold, _cx| {
RustdocIndexPlaceholder {
id,
unfold,
source: RustdocSource::Local,
crate_name: SharedString::from(crate_name_to_index.clone()),
}
.into_any_element()
}),
}],
run_commands_in_text: false,
})
});
}
let mut path_components = item_path.split("::");
let crate_name = match path_components let crate_name = match path_components
.next() .next()
.ok_or_else(|| anyhow!("missing crate name")) .ok_or_else(|| anyhow!("missing crate name"))
@ -301,31 +269,3 @@ impl RenderOnce for RustdocPlaceholder {
.on_click(move |_, cx| unfold(cx)) .on_click(move |_, cx| unfold(cx))
} }
} }
#[derive(IntoElement)]
struct RustdocIndexPlaceholder {
pub id: ElementId,
pub unfold: Arc<dyn Fn(&mut WindowContext)>,
pub source: RustdocSource,
pub crate_name: SharedString,
}
impl RenderOnce for RustdocIndexPlaceholder {
fn render(self, _cx: &mut WindowContext) -> impl IntoElement {
let unfold = self.unfold;
ButtonLike::new(self.id)
.style(ButtonStyle::Filled)
.layer(ElevationIndex::ElevatedSurface)
.child(Icon::new(IconName::FileRust))
.child(Label::new(format!(
"rustdoc index ({source}): {crate_name}",
crate_name = self.crate_name,
source = match self.source {
RustdocSource::Local => "local",
RustdocSource::DocsDotRs => "docs.rs",
}
)))
.on_click(move |_, cx| unfold(cx))
}
}

View File

@ -23,6 +23,7 @@ heed.workspace = true
html_to_markdown.workspace = true html_to_markdown.workspace = true
http.workspace = true http.workspace = true
indexmap.workspace = true indexmap.workspace = true
parking_lot.workspace = true
serde.workspace = true serde.workspace = true
strum.workspace = true strum.workspace = true
util.workspace = true util.workspace = true

View File

@ -56,8 +56,6 @@ impl RustdocProvider for LocalProvider {
local_cargo_doc_path.push("index.html"); local_cargo_doc_path.push("index.html");
} }
println!("Fetching {}", local_cargo_doc_path.display());
let Ok(contents) = self.fs.load(&local_cargo_doc_path).await else { let Ok(contents) = self.fs.load(&local_cargo_doc_path).await else {
return Ok(None); return Ok(None);
}; };
@ -91,8 +89,6 @@ impl RustdocProvider for DocsDotRsProvider {
.unwrap_or_default() .unwrap_or_default()
); );
println!("Fetching {}", &format!("https://docs.rs/{path}"));
let mut response = self let mut response = self
.http_client .http_client
.get( .get(
@ -165,8 +161,6 @@ impl RustdocIndexer {
while let Some(item_with_history) = items_to_visit.pop_front() { while let Some(item_with_history) = items_to_visit.pop_front() {
let item = &item_with_history.item; let item = &item_with_history.item;
println!("Visiting {:?} {:?} {}", &item.kind, &item.path, &item.name);
let Some(result) = self let Some(result) = self
.provider .provider
.fetch_page(&crate_name, Some(&item)) .fetch_page(&crate_name, Some(&item))

View File

@ -3,12 +3,14 @@ use std::sync::atomic::AtomicBool;
use std::sync::Arc; use std::sync::Arc;
use anyhow::{anyhow, Result}; use anyhow::{anyhow, Result};
use collections::HashMap;
use futures::future::{self, BoxFuture, Shared}; use futures::future::{self, BoxFuture, Shared};
use futures::FutureExt; use futures::FutureExt;
use fuzzy::StringMatchCandidate; use fuzzy::StringMatchCandidate;
use gpui::{AppContext, BackgroundExecutor, Global, ReadGlobal, Task, UpdateGlobal}; use gpui::{AppContext, BackgroundExecutor, Global, ReadGlobal, Task, UpdateGlobal};
use heed::types::SerdeBincode; use heed::types::SerdeBincode;
use heed::Database; use heed::Database;
use parking_lot::RwLock;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use util::paths::SUPPORT_DIR; use util::paths::SUPPORT_DIR;
use util::ResultExt; use util::ResultExt;
@ -23,6 +25,7 @@ impl Global for GlobalRustdocStore {}
pub struct RustdocStore { pub struct RustdocStore {
executor: BackgroundExecutor, executor: BackgroundExecutor,
database_future: Shared<BoxFuture<'static, Result<Arc<RustdocDatabase>, Arc<anyhow::Error>>>>, database_future: Shared<BoxFuture<'static, Result<Arc<RustdocDatabase>, Arc<anyhow::Error>>>>,
indexing_tasks_by_crate: RwLock<HashMap<String, Shared<Task<Result<(), Arc<anyhow::Error>>>>>>,
} }
impl RustdocStore { impl RustdocStore {
@ -52,6 +55,7 @@ impl RustdocStore {
Self { Self {
executor, executor,
database_future, database_future,
indexing_tasks_by_crate: RwLock::new(HashMap::default()),
} }
} }
@ -69,17 +73,45 @@ impl RustdocStore {
} }
pub fn index( pub fn index(
&self, self: Arc<Self>,
crate_name: String, crate_name: String,
provider: Box<dyn RustdocProvider + Send + Sync + 'static>, provider: Box<dyn RustdocProvider + Send + Sync + 'static>,
) -> Task<Result<()>> { ) -> Shared<Task<Result<(), Arc<anyhow::Error>>>> {
let database_future = self.database_future.clone(); let indexing_task = self
self.executor.spawn(async move { .executor
let database = database_future.await.map_err(|err| anyhow!(err))?; .spawn({
let this = self.clone();
let crate_name = crate_name.clone();
async move {
let _finally = util::defer({
let this = this.clone();
let crate_name = crate_name.clone();
move || {
this.indexing_tasks_by_crate.write().remove(&crate_name);
}
});
let index_task = async {
let database = this
.database_future
.clone()
.await
.map_err(|err| anyhow!(err))?;
let indexer = RustdocIndexer::new(database, provider); let indexer = RustdocIndexer::new(database, provider);
indexer.index(crate_name.clone()).await indexer.index(crate_name.clone()).await
};
index_task.await.map_err(Arc::new)
}
}) })
.shared();
self.indexing_tasks_by_crate
.write()
.insert(crate_name, indexing_task.clone());
indexing_task
} }
pub fn search(&self, query: String) -> Task<Vec<String>> { pub fn search(&self, query: String) -> Task<Vec<String>> {