use std::rc::Rc; use std::sync::Arc; use std::{borrow::Cow, cell::RefCell}; use agent_client_protocol::schema::v1 as acp; use anyhow::{Context as _, Result, bail}; use futures::{AsyncReadExt as _, FutureExt as _}; use gpui::{App, AppContext as _, Task}; use html_to_markdown::{TagHandler, convert_html_to_markdown, markdown}; use http_client::{AsyncBody, HttpClientWithUrl}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use ui::SharedString; use util::markdown::{MarkdownEscaped, MarkdownInlineCode}; use crate::sandboxing::{NetworkRequest, SandboxRequest}; use crate::{AgentTool, ToolCallEventStream, ToolInput}; #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Clone, Copy)] enum ContentType { Html, Plaintext, Json, } /// The maximum number of HTTP redirects the fetch tool will follow. Each hop is /// re-authorized against the shared network grants before being followed. const MAX_REDIRECTS: usize = 20; /// The outcome of a single (non-redirect-following) HTTP request. enum FetchStep { /// The server responded with a redirect to this absolute URL. Its host must /// be authorized before the redirect is followed. Redirect(String), /// A terminal response was received and converted to Markdown. Complete(String), } /// Prepends `https://` when the URL has no explicit HTTP(S) scheme, matching the /// behavior the fetch tool has always had for user/model-supplied URLs. fn normalize_url(url: &str) -> Cow<'_, str> { if !url.starts_with("https://") && !url.starts_with("http://") { Cow::Owned(format!("https://{url}")) } else { Cow::Borrowed(url) } } /// Fetches a URL and returns the content as Markdown. /// /// This tool is not run inside the terminal OS sandbox, but it still refuses to /// reach any host that hasn't been granted network access. It shares the same /// per-host grants as the `terminal` tool: approving a host for one authorizes /// it for the other, whether the grant is for this thread or saved permanently. /// HTTP redirects are followed one hop at a time, and each hop's host must be /// granted the same way, so a granted host can't redirect the request to a host /// that hasn't been approved. /// When unsandboxed access has been granted, these restrictions are lifted /// entirely, matching the terminal, which is also how loopback and IP-literal /// hosts (which can't be granted individually) become reachable. #[derive(Debug, Serialize, Deserialize, JsonSchema)] pub struct FetchToolInput { /// The URL to fetch. url: String, } pub struct FetchTool { http_client: Arc, } impl FetchTool { pub fn new(http_client: Arc) -> Self { Self { http_client } } /// Performs a single HTTP GET *without* following redirects, so the tool can /// re-authorize each hop against the shared network grants before following /// it. Returns the redirect target when the server responds with a 3xx, or /// the final content converted to Markdown otherwise. async fn fetch_step(http_client: Arc, url: &str) -> Result { let normalized = normalize_url(url); let mut response = http_client .get(&normalized, AsyncBody::default(), false) .await?; let status = response.status(); if status.is_redirection() { let location = response .headers() .get("location") .context("redirect response is missing a Location header")? .to_str() .context("redirect response has an invalid Location header")?; let target = url::Url::parse(&normalized) .with_context(|| format!("could not parse URL {normalized:?}"))? .join(location) .with_context(|| format!("invalid redirect target {location:?}"))?; anyhow::ensure!( matches!(target.scheme(), "http" | "https"), "refusing to follow redirect to non-HTTP(S) URL {target}" ); return Ok(FetchStep::Redirect(target.to_string())); } let mut body = Vec::new(); response .body_mut() .read_to_end(&mut body) .await .context("error reading response body")?; if status.is_client_error() { let text = String::from_utf8_lossy(body.as_slice()); bail!("status error {}, response: {text:?}", status.as_u16()); } let Some(content_type) = response.headers().get("content-type") else { bail!("missing Content-Type header"); }; let content_type = content_type .to_str() .context("invalid Content-Type header")?; let content_type = if content_type.starts_with("text/plain") { ContentType::Plaintext } else if content_type.starts_with("application/json") { ContentType::Json } else { ContentType::Html }; let text = match content_type { ContentType::Html => { let mut handlers: Vec = vec![ Rc::new(RefCell::new(markdown::WebpageChromeRemover)), Rc::new(RefCell::new(markdown::ParagraphHandler)), Rc::new(RefCell::new(markdown::HeadingHandler)), Rc::new(RefCell::new(markdown::ListHandler)), Rc::new(RefCell::new(markdown::TableHandler::new())), Rc::new(RefCell::new(markdown::StyledTextHandler)), ]; if normalized.contains("wikipedia.org") { use html_to_markdown::structure::wikipedia; handlers.push(Rc::new(RefCell::new(wikipedia::WikipediaChromeRemover))); handlers.push(Rc::new(RefCell::new(wikipedia::WikipediaInfoboxHandler))); handlers.push(Rc::new( RefCell::new(wikipedia::WikipediaCodeHandler::new()), )); } else { handlers.push(Rc::new(RefCell::new(markdown::CodeHandler))); } convert_html_to_markdown(&body[..], &mut handlers)? } ContentType::Plaintext => std::str::from_utf8(&body)?.to_owned(), ContentType::Json => { let json: serde_json::Value = serde_json::from_slice(&body)?; format!("```json\n{}\n```", serde_json::to_string_pretty(&json)?) } }; Ok(FetchStep::Complete(text)) } } /// Resolve the host of `url` and confirm it doesn't point into loopback / /// private / link-local space, applying the same forbidden-IP policy the /// terminal sandbox's proxy uses. Returns an error (including "resolves only to /// forbidden addresses") that aborts the fetch. /// /// DNS resolution blocks, so callers should run this off the foreground thread. /// See the caller for why this is a gate rather than a full resolve-to-connect /// pin. fn verify_host_not_forbidden(url: &str) -> Result<()> { let normalized = normalize_url(url); let parsed = url::Url::parse(&normalized).with_context(|| format!("could not parse URL {url:?}"))?; let host = parsed .host_str() .with_context(|| format!("URL {url:?} has no host to reach"))?; // Default to the scheme's port when the URL omits one; resolution needs a // port but the value doesn't affect which IPs a host resolves to. let port = parsed .port_or_known_default() .unwrap_or(if parsed.scheme() == "http" { 80 } else { 443 }); http_proxy::PinnedHost::resolve(host, port).map(|_pinned| ())?; Ok(()) } /// Extracts the host from a fetch URL as a [`http_proxy::HostPattern`] so it can /// be matched against the shared network grants. Mirrors the scheme handling in /// [`normalize_url`] (defaulting to `https://` when none is given). fn host_pattern_for_url(url: &str) -> Result { let normalized = normalize_url(url); let parsed = url::Url::parse(&normalized).with_context(|| format!("could not parse URL {url:?}"))?; let host = parsed .host_str() .with_context(|| format!("URL {url:?} has no host to authorize network access for"))?; http_proxy::HostPattern::parse(host).map_err(|error| match error { http_proxy::HostPatternError::IpLiteral(_) => anyhow::anyhow!( "cannot fetch {host:?}: loopback and IP-literal hosts can't be granted network \ access individually. They are only reachable once unsandboxed access has been \ granted (for example, via a terminal command that requests it)." ), error => anyhow::anyhow!("cannot authorize network access to {host:?}: {error}"), }) } impl AgentTool for FetchTool { type Input = FetchToolInput; type Output = String; const NAME: &'static str = "fetch"; fn kind() -> acp::ToolKind { acp::ToolKind::Fetch } fn allow_in_restricted_mode() -> bool { false } fn initial_title( &self, input: Result, _cx: &mut App, ) -> SharedString { match input { Ok(input) => format!("Fetch {}", MarkdownEscaped(&input.url)).into(), Err(_) => "Fetch URL".into(), } } fn run( self: Arc, input: ToolInput, event_stream: ToolCallEventStream, cx: &mut App, ) -> Task> { let http_client = self.http_client.clone(); cx.spawn(async move |cx| { let input: FetchToolInput = input.recv().await.map_err(|e| e.to_string())?; // First, the standard tool-permission gate (honors the fetch tool's // allow/deny/confirm rules). let authorize = cx.update(|cx| { let context = crate::ToolPermissionContext::new(Self::NAME, vec![input.url.clone()]); event_stream.authorize( format!("Fetch {}", MarkdownInlineCode(&input.url)), context, cx, ) }); futures::select! { result = authorize.fuse() => result.map_err(|e| e.to_string())?, _ = event_stream.cancelled_by_user().fuse() => { return Err("Fetch cancelled by user".to_string()); } }; // Then, unless unsandboxed access is already in effect, the per-host // network grant shared with the terminal tool. If the host isn't // already granted (for this thread or in saved settings) the user is // shown the same escalation prompt the terminal uses; a denial // aborts the fetch. This tool never runs inside the OS sandbox, so // the grant is only consulted to decide whether the request may // proceed. When unsandboxed access has been granted the terminal // already runs without isolation, so we drop fetch's restrictions // too — including reaching hosts that can't be granted individually // (loopback and IP literals). // // Crucially, this authorization is applied to every redirect hop as // well as the initial URL, so a granted host can't 30x-redirect the // fetch to a host the user never approved. We disable the HTTP // client's own redirect following and re-run the grant for each hop // before requesting it. let unsandboxed = cx.update(|cx| event_stream.unsandboxed_access_granted(cx)); let mut current_url = input.url.clone(); let mut redirects = 0; let text = loop { if !unsandboxed { let host = host_pattern_for_url(¤t_url).map_err(|e| e.to_string())?; let authorize_host = cx.update(|cx| { let request = SandboxRequest { network: NetworkRequest::Hosts(vec![host]), ..Default::default() }; event_stream.authorize_sandbox(request, String::new(), cx) }); futures::select! { result = authorize_host.fuse() => result.map_err(|e| e.to_string())?, _ = event_stream.cancelled_by_user().fuse() => { return Err("Fetch cancelled by user".to_string()); } }; // Authorizing the *hostname* is not enough: a granted host // (or a redirect to one) whose DNS points into loopback / // private / link-local space would otherwise let the model // reach the local machine or LAN (SSRF / DNS rebinding). // Resolve and vet the host now, applying the same // forbidden-IP policy the terminal sandbox's proxy enforces. // // NOTE: this is a gate, not a full pin. `HttpClientWithUrl` // resolves the hostname again when it connects, so a DNS // answer that flips between this check and that connect could // still slip through. Closing that residual window would // require the HTTP client to connect to a pre-vetted IP // (`PinnedHost::socket_addrs`) rather than re-resolving; // until then this blocks the realistic case of a stably // resolving host that points at forbidden space. let verify_task = cx.background_spawn({ let url = current_url.clone(); async move { verify_host_not_forbidden(&url) } }); futures::select! { result = verify_task.fuse() => result.map_err(|e| e.to_string())?, _ = event_stream.cancelled_by_user().fuse() => { return Err("Fetch cancelled by user".to_string()); } }; } let fetch_task = cx.background_spawn({ let http_client = http_client.clone(); let url = current_url.clone(); async move { Self::fetch_step(http_client, &url).await } }); let step = futures::select! { result = fetch_task.fuse() => result.map_err(|e| e.to_string())?, _ = event_stream.cancelled_by_user().fuse() => { return Err("Fetch cancelled by user".to_string()); } }; match step { FetchStep::Complete(text) => break text, FetchStep::Redirect(target) => { redirects += 1; if redirects > MAX_REDIRECTS { return Err(format!( "exceeded the maximum of {MAX_REDIRECTS} redirects" )); } current_url = target; } } }; if text.trim().is_empty() { return Err("no textual content found".to_string()); } Ok(text) }) } } #[cfg(test)] mod tests { use super::*; // These use IP-literal URLs, which "resolve" to themselves, so the SSRF gate // is exercised without depending on real DNS. IP literals can't be *granted* // network access (that's a separate, earlier check), but they can be the // target a granted hostname redirects to or resolves into — which is exactly // the case this gate defends. #[test] fn verify_host_rejects_loopback_literal() { let error = verify_host_not_forbidden("http://127.0.0.1/internal") .expect_err("loopback must be refused"); assert!( error.to_string().contains("loopback"), "error should explain the forbidden range, got: {error}" ); } #[test] fn verify_host_rejects_private_and_metadata_literals() { for url in [ "http://10.0.0.5/", "https://192.168.1.1/", "http://169.254.169.254/latest/meta-data/", // cloud metadata "http://[::1]/", ] { assert!( verify_host_not_forbidden(url).is_err(), "expected {url} to be refused as a forbidden destination" ); } } #[test] fn verify_host_allows_public_literal() { verify_host_not_forbidden("https://93.184.215.14/") .expect("a public address must be allowed through the gate"); } }