assistant: Add /fetch slash command (#12645)

This PR adds a new `/fetch` slash command to the Assistant for fetching
the content of an arbitrary URL as Markdown.

Currently it's just using the same HTML to Markdown conversion that
`/rustdoc` uses, but I'll be working to refine the output to be more
widely useful.

Release Notes:

- N/A

crates/assistant/src/assistant.rs

@@ -25,8 +25,8 @@ use semantic_index::{CloudEmbeddingProvider, SemanticIndex};
 use serde::{Deserialize, Serialize};
 use settings::{Settings, SettingsStore};
 use slash_command::{
-    active_command, file_command, project_command, prompt_command, rustdoc_command, search_command,
-    tabs_command,
+    active_command, fetch_command, file_command, project_command, prompt_command, rustdoc_command,
+    search_command, tabs_command,
 };
 use std::{
     fmt::{self, Display},
@@ -304,6 +304,7 @@ fn register_slash_commands(cx: &mut AppContext) {
     slash_command_registry.register_command(project_command::ProjectSlashCommand, true);
     slash_command_registry.register_command(search_command::SearchSlashCommand, true);
     slash_command_registry.register_command(rustdoc_command::RustdocSlashCommand, false);
+    slash_command_registry.register_command(fetch_command::FetchSlashCommand, false);
 
     let store = PromptStore::global(cx);
     cx.background_executor()

crates/assistant/src/slash_command.rs

@@ -17,6 +17,7 @@ use std::{
 use workspace::Workspace;
 
 pub mod active_command;
+pub mod fetch_command;
 pub mod file_command;
 pub mod project_command;
 pub mod prompt_command;

crates/assistant/src/slash_command/fetch_command.rs

@@ -0,0 +1,133 @@
+use std::sync::atomic::AtomicBool;
+use std::sync::Arc;
+
+use anyhow::{anyhow, bail, Context, Result};
+use assistant_slash_command::{SlashCommand, SlashCommandOutput, SlashCommandOutputSection};
+use futures::AsyncReadExt;
+use gpui::{AppContext, Task, WeakView};
+use http::{AsyncBody, HttpClient, HttpClientWithUrl};
+use language::LspAdapterDelegate;
+use rustdoc_to_markdown::convert_html_to_markdown;
+use ui::{prelude::*, ButtonLike, ElevationIndex};
+use workspace::Workspace;
+
+pub(crate) struct FetchSlashCommand;
+
+impl FetchSlashCommand {
+    async fn build_message(http_client: Arc<HttpClientWithUrl>, url: &str) -> Result<String> {
+        let mut url = url.to_owned();
+        if !url.starts_with("https://") {
+            url = format!("https://{url}");
+        }
+
+        let mut response = http_client.get(&url, AsyncBody::default(), true).await?;
+
+        let mut body = Vec::new();
+        response
+            .body_mut()
+            .read_to_end(&mut body)
+            .await
+            .context("error reading response body")?;
+
+        if response.status().is_client_error() {
+            let text = String::from_utf8_lossy(body.as_slice());
+            bail!(
+                "status error {}, response: {text:?}",
+                response.status().as_u16()
+            );
+        }
+
+        convert_html_to_markdown(&body[..])
+    }
+}
+
+impl SlashCommand for FetchSlashCommand {
+    fn name(&self) -> String {
+        "fetch".into()
+    }
+
+    fn description(&self) -> String {
+        "insert URL contents".into()
+    }
+
+    fn menu_text(&self) -> String {
+        "Insert fetched URL contents".into()
+    }
+
+    fn requires_argument(&self) -> bool {
+        true
+    }
+
+    fn complete_argument(
+        &self,
+        _query: String,
+        _cancel: Arc<AtomicBool>,
+        _workspace: WeakView<Workspace>,
+        _cx: &mut AppContext,
+    ) -> Task<Result<Vec<String>>> {
+        Task::ready(Ok(Vec::new()))
+    }
+
+    fn run(
+        self: Arc<Self>,
+        argument: Option<&str>,
+        workspace: WeakView<Workspace>,
+        _delegate: Arc<dyn LspAdapterDelegate>,
+        cx: &mut WindowContext,
+    ) -> Task<Result<SlashCommandOutput>> {
+        let Some(argument) = argument else {
+            return Task::ready(Err(anyhow!("missing URL")));
+        };
+        let Some(workspace) = workspace.upgrade() else {
+            return Task::ready(Err(anyhow!("workspace was dropped")));
+        };
+
+        let http_client = workspace.read(cx).client().http_client();
+        let url = argument.to_string();
+
+        let text = cx.background_executor().spawn({
+            let url = url.clone();
+            async move { Self::build_message(http_client, &url).await }
+        });
+
+        let url = SharedString::from(url);
+        cx.foreground_executor().spawn(async move {
+            let text = text.await?;
+            let range = 0..text.len();
+            Ok(SlashCommandOutput {
+                text,
+                sections: vec![SlashCommandOutputSection {
+                    range,
+                    render_placeholder: Arc::new(move |id, unfold, _cx| {
+                        FetchPlaceholder {
+                            id,
+                            unfold,
+                            url: url.clone(),
+                        }
+                        .into_any_element()
+                    }),
+                }],
+            })
+        })
+    }
+}
+
+#[derive(IntoElement)]
+struct FetchPlaceholder {
+    pub id: ElementId,
+    pub unfold: Arc<dyn Fn(&mut WindowContext)>,
+    pub url: SharedString,
+}
+
+impl RenderOnce for FetchPlaceholder {
+    fn render(self, _cx: &mut WindowContext) -> impl IntoElement {
+        let unfold = self.unfold;
+
+        ButtonLike::new(self.id)
+            .style(ButtonStyle::Filled)
+            .layer(ElevationIndex::ElevatedSurface)
+            .child(Icon::new(IconName::AtSign))
+            .child(Label::new(format!("fetch {url}", url = self.url)))
+            .on_click(move |_, cx| unfold(cx))
+    }
+}

crates/rustdoc_to_markdown/src/rustdoc_to_markdown.rs

@@ -16,8 +16,31 @@ use markup5ever_rcdom::RcDom;
 
 use crate::markdown_writer::MarkdownWriter;
 
+/// Converts the provided HTML to Markdown.
+pub fn convert_html_to_markdown(html: impl Read) -> Result<String> {
+    let dom = parse_html(html).context("failed to parse HTML")?;
+
+    let markdown_writer = MarkdownWriter::new();
+    let markdown = markdown_writer
+        .run(&dom.document)
+        .context("failed to convert HTML to Markdown")?;
+
+    Ok(markdown)
+}
+
 /// Converts the provided rustdoc HTML to Markdown.
-pub fn convert_rustdoc_to_markdown(mut html: impl Read) -> Result<String> {
+pub fn convert_rustdoc_to_markdown(html: impl Read) -> Result<String> {
+    let dom = parse_html(html).context("failed to parse rustdoc HTML")?;
+
+    let markdown_writer = MarkdownWriter::new();
+    let markdown = markdown_writer
+        .run(&dom.document)
+        .context("failed to convert rustdoc HTML to Markdown")?;
+
+    Ok(markdown)
+}
+
+fn parse_html(mut html: impl Read) -> Result<RcDom> {
     let parse_options = ParseOpts {
         tree_builder: TreeBuilderOpts {
             drop_doctype: true,
@@ -28,14 +51,9 @@ pub fn convert_rustdoc_to_markdown(mut html: impl Read) -> Result<String> {
     let dom = parse_document(RcDom::default(), parse_options)
         .from_utf8()
         .read_from(&mut html)
-        .context("failed to parse rustdoc HTML")?;
+        .context("failed to parse HTML document")?;
 
-    let markdown_writer = MarkdownWriter::new();
-    let markdown = markdown_writer
-        .run(&dom.document)
-        .context("failed to convert rustdoc to HTML")?;
-
-    Ok(markdown)
+    Ok(dom)
 }
 
 #[cfg(test)]