rustdoc: Automatically index crates (#13014)

This PR removes the need to use `/rustdoc --index <CRATE_NAME>` and
instead indexes the crates once they are referenced.

As soon as the first `:` is added after the crate name, the indexing
will kick off in the background and update the index as it goes.

Release Notes:

- N/A

Cargo.lock

@@ -8723,6 +8723,7 @@ dependencies = [
  "http 0.1.0",
  "indexmap 1.9.3",
  "indoc",
+ "parking_lot",
  "pretty_assertions",
  "serde",
  "strum",

crates/assistant/src/slash_command/rustdoc_command.rs

@@ -13,6 +13,7 @@ use project::{Project, ProjectPath};
 use rustdoc::LocalProvider;
 use rustdoc::{convert_rustdoc_to_markdown, RustdocStore};
 use ui::{prelude::*, ButtonLike, ElevationIndex};
+use util::{maybe, ResultExt};
 use workspace::Workspace;
 
 #[derive(Debug, Clone, Copy)]
@@ -118,11 +119,36 @@ impl SlashCommand for RustdocSlashCommand {
         &self,
         query: String,
         _cancel: Arc<AtomicBool>,
-        _workspace: Option<WeakView<Workspace>>,
+        workspace: Option<WeakView<Workspace>>,
         cx: &mut AppContext,
     ) -> Task<Result<Vec<String>>> {
+        let index_provider_deps = maybe!({
+            let workspace = workspace.ok_or_else(|| anyhow!("no workspace"))?;
+            let workspace = workspace
+                .upgrade()
+                .ok_or_else(|| anyhow!("workspace was dropped"))?;
+            let project = workspace.read(cx).project().clone();
+            let fs = project.read(cx).fs().clone();
+            let cargo_workspace_root = Self::path_to_cargo_toml(project, cx)
+                .and_then(|path| path.parent().map(|path| path.to_path_buf()))
+                .ok_or_else(|| anyhow!("no Cargo workspace root found"))?;
+
+            anyhow::Ok((fs, cargo_workspace_root))
+        });
+
         let store = RustdocStore::global(cx);
         cx.background_executor().spawn(async move {
+            if let Some((crate_name, rest)) = query.split_once(':') {
+                if rest.is_empty() {
+                    if let Some((fs, cargo_workspace_root)) = index_provider_deps.log_err() {
+                        let provider = Box::new(LocalProvider::new(fs, cargo_workspace_root));
+                        // We don't need to hold onto this task, as the `RustdocStore` will hold it
+                        // until it completes.
+                        let _ = store.clone().index(crate_name.to_string(), provider);
+                    }
+                }
+            }
+
             let items = store.search(query).await;
             Ok(items)
         })
@@ -147,65 +173,7 @@ impl SlashCommand for RustdocSlashCommand {
         let http_client = workspace.read(cx).client().http_client();
         let path_to_cargo_toml = Self::path_to_cargo_toml(project, cx);
 
-        let mut item_path = String::new();
+        let mut path_components = argument.split("::");
-        let mut crate_name_to_index = None;
-
-        let mut args = argument.split(' ').map(|word| word.trim());
-        while let Some(arg) = args.next() {
-            if arg == "--index" {
-                let Some(crate_name) = args.next() else {
-                    return Task::ready(Err(anyhow!("no crate name provided to --index")));
-                };
-                crate_name_to_index = Some(crate_name.to_string());
-                continue;
-            }
-
-            item_path.push_str(arg);
-        }
-
-        if let Some(crate_name_to_index) = crate_name_to_index {
-            let index_task = cx.background_executor().spawn({
-                let rustdoc_store = RustdocStore::global(cx);
-                let fs = fs.clone();
-                let crate_name_to_index = crate_name_to_index.clone();
-                async move {
-                    let cargo_workspace_root = path_to_cargo_toml
-                        .and_then(|path| path.parent().map(|path| path.to_path_buf()))
-                        .ok_or_else(|| anyhow!("no Cargo workspace root found"))?;
-
-                    let provider = Box::new(LocalProvider::new(fs, cargo_workspace_root));
-
-                    rustdoc_store
-                        .index(crate_name_to_index.clone(), provider)
-                        .await?;
-
-                    anyhow::Ok(format!("Indexed {crate_name_to_index}"))
-                }
-            });
-
-            return cx.foreground_executor().spawn(async move {
-                let text = index_task.await?;
-                let range = 0..text.len();
-                Ok(SlashCommandOutput {
-                    text,
-                    sections: vec![SlashCommandOutputSection {
-                        range,
-                        render_placeholder: Arc::new(move |id, unfold, _cx| {
-                            RustdocIndexPlaceholder {
-                                id,
-                                unfold,
-                                source: RustdocSource::Local,
-                                crate_name: SharedString::from(crate_name_to_index.clone()),
-                            }
-                            .into_any_element()
-                        }),
-                    }],
-                    run_commands_in_text: false,
-                })
-            });
-        }
-
-        let mut path_components = item_path.split("::");
         let crate_name = match path_components
             .next()
             .ok_or_else(|| anyhow!("missing crate name"))
@@ -301,31 +269,3 @@ impl RenderOnce for RustdocPlaceholder {
             .on_click(move |_, cx| unfold(cx))
     }
 }
-
-#[derive(IntoElement)]
-struct RustdocIndexPlaceholder {
-    pub id: ElementId,
-    pub unfold: Arc<dyn Fn(&mut WindowContext)>,
-    pub source: RustdocSource,
-    pub crate_name: SharedString,
-}
-
-impl RenderOnce for RustdocIndexPlaceholder {
-    fn render(self, _cx: &mut WindowContext) -> impl IntoElement {
-        let unfold = self.unfold;
-
-        ButtonLike::new(self.id)
-            .style(ButtonStyle::Filled)
-            .layer(ElevationIndex::ElevatedSurface)
-            .child(Icon::new(IconName::FileRust))
-            .child(Label::new(format!(
-                "rustdoc index ({source}): {crate_name}",
-                crate_name = self.crate_name,
-                source = match self.source {
-                    RustdocSource::Local => "local",
-                    RustdocSource::DocsDotRs => "docs.rs",
-                }
-            )))
-            .on_click(move |_, cx| unfold(cx))
-    }
-}

crates/rustdoc/Cargo.toml

@@ -23,6 +23,7 @@ heed.workspace = true
 html_to_markdown.workspace = true
 http.workspace = true
 indexmap.workspace = true
+parking_lot.workspace = true
 serde.workspace = true
 strum.workspace = true
 util.workspace = true

crates/rustdoc/src/indexer.rs

@@ -56,8 +56,6 @@ impl RustdocProvider for LocalProvider {
             local_cargo_doc_path.push("index.html");
         }
 
-        println!("Fetching {}", local_cargo_doc_path.display());
-
         let Ok(contents) = self.fs.load(&local_cargo_doc_path).await else {
             return Ok(None);
         };
@@ -91,8 +89,6 @@ impl RustdocProvider for DocsDotRsProvider {
                 .unwrap_or_default()
         );
 
-        println!("Fetching {}", &format!("https://docs.rs/{path}"));
-
         let mut response = self
             .http_client
             .get(
@@ -165,8 +161,6 @@ impl RustdocIndexer {
         while let Some(item_with_history) = items_to_visit.pop_front() {
             let item = &item_with_history.item;
 
-            println!("Visiting {:?} {:?} {}", &item.kind, &item.path, &item.name);
-
             let Some(result) = self
                 .provider
                 .fetch_page(&crate_name, Some(&item))

crates/rustdoc/src/store.rs

@@ -3,12 +3,14 @@ use std::sync::atomic::AtomicBool;
 use std::sync::Arc;
 
 use anyhow::{anyhow, Result};
+use collections::HashMap;
 use futures::future::{self, BoxFuture, Shared};
 use futures::FutureExt;
 use fuzzy::StringMatchCandidate;
 use gpui::{AppContext, BackgroundExecutor, Global, ReadGlobal, Task, UpdateGlobal};
 use heed::types::SerdeBincode;
 use heed::Database;
+use parking_lot::RwLock;
 use serde::{Deserialize, Serialize};
 use util::paths::SUPPORT_DIR;
 use util::ResultExt;
@@ -23,6 +25,7 @@ impl Global for GlobalRustdocStore {}
 pub struct RustdocStore {
     executor: BackgroundExecutor,
     database_future: Shared<BoxFuture<'static, Result<Arc<RustdocDatabase>, Arc<anyhow::Error>>>>,
+    indexing_tasks_by_crate: RwLock<HashMap<String, Shared<Task<Result<(), Arc<anyhow::Error>>>>>>,
 }
 
 impl RustdocStore {
@@ -52,6 +55,7 @@ impl RustdocStore {
         Self {
             executor,
             database_future,
+            indexing_tasks_by_crate: RwLock::new(HashMap::default()),
         }
     }
 
@@ -69,17 +73,45 @@ impl RustdocStore {
     }
 
     pub fn index(
-        &self,
+        self: Arc<Self>,
         crate_name: String,
         provider: Box<dyn RustdocProvider + Send + Sync + 'static>,
-    ) -> Task<Result<()>> {
+    ) -> Shared<Task<Result<(), Arc<anyhow::Error>>>> {
-        let database_future = self.database_future.clone();
+        let indexing_task = self
-        self.executor.spawn(async move {
+            .executor
-            let database = database_future.await.map_err(|err| anyhow!(err))?;
+            .spawn({
+                let this = self.clone();
+                let crate_name = crate_name.clone();
+                async move {
+                    let _finally = util::defer({
+                        let this = this.clone();
+                        let crate_name = crate_name.clone();
+                        move || {
+                            this.indexing_tasks_by_crate.write().remove(&crate_name);
+                        }
+                    });
+
+                    let index_task = async {
+                        let database = this
+                            .database_future
+                            .clone()
+                            .await
+                            .map_err(|err| anyhow!(err))?;
                         let indexer = RustdocIndexer::new(database, provider);
 
                         indexer.index(crate_name.clone()).await
+                    };
+
+                    index_task.await.map_err(Arc::new)
+                }
             })
+            .shared();
+
+        self.indexing_tasks_by_crate
+            .write()
+            .insert(crate_name, indexing_task.clone());
+
+        indexing_task
     }
 
     pub fn search(&self, query: String) -> Task<Vec<String>> {