tenant.openagents/omega

1use std::rc::Rc;
2use std::sync::Arc;
3use std::{borrow::Cow, cell::RefCell};
4
5use agent_client_protocol::schema::v1 as acp;
6use anyhow::{Context as _, Result, bail};
7use futures::{AsyncReadExt as _, FutureExt as _};
8use gpui::{App, AppContext as _, Task};
9use html_to_markdown::{TagHandler, convert_html_to_markdown, markdown};
10use http_client::{AsyncBody, HttpClientWithUrl};
11use schemars::JsonSchema;
12use serde::{Deserialize, Serialize};
13use ui::SharedString;
14use util::markdown::{MarkdownEscaped, MarkdownInlineCode};
15
16use crate::sandboxing::{NetworkRequest, SandboxRequest};
17use crate::{AgentTool, ToolCallEventStream, ToolInput};
18
19#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Clone, Copy)]
20enum ContentType {
21    Html,
22    Plaintext,
23    Json,
24}
25
26/// The maximum number of HTTP redirects the fetch tool will follow. Each hop is
27/// re-authorized against the shared network grants before being followed.
28const MAX_REDIRECTS: usize = 20;
29
30/// The outcome of a single (non-redirect-following) HTTP request.
31enum FetchStep {
32    /// The server responded with a redirect to this absolute URL. Its host must
33    /// be authorized before the redirect is followed.
34    Redirect(String),
35    /// A terminal response was received and converted to Markdown.
36    Complete(String),
37}
38
39/// Prepends `https://` when the URL has no explicit HTTP(S) scheme, matching the
40/// behavior the fetch tool has always had for user/model-supplied URLs.
41fn normalize_url(url: &str) -> Cow<'_, str> {
42    if !url.starts_with("https://") && !url.starts_with("http://") {
43        Cow::Owned(format!("https://{url}"))
44    } else {
45        Cow::Borrowed(url)
46    }
47}
48
49/// Fetches a URL and returns the content as Markdown.
50///
51/// This tool is not run inside the terminal OS sandbox, but it still refuses to
52/// reach any host that hasn't been granted network access. It shares the same
53/// per-host grants as the `terminal` tool: approving a host for one authorizes
54/// it for the other, whether the grant is for this thread or saved permanently.
55/// HTTP redirects are followed one hop at a time, and each hop's host must be
56/// granted the same way, so a granted host can't redirect the request to a host
57/// that hasn't been approved.
58/// When unsandboxed access has been granted, these restrictions are lifted
59/// entirely, matching the terminal, which is also how loopback and IP-literal
60/// hosts (which can't be granted individually) become reachable.
61#[derive(Debug, Serialize, Deserialize, JsonSchema)]
62pub struct FetchToolInput {
63    /// The URL to fetch.
64    url: String,
65}
66
67pub struct FetchTool {
68    http_client: Arc<HttpClientWithUrl>,
69}
70
71impl FetchTool {
72    pub fn new(http_client: Arc<HttpClientWithUrl>) -> Self {
73        Self { http_client }
74    }
75
76    /// Performs a single HTTP GET *without* following redirects, so the tool can
77    /// re-authorize each hop against the shared network grants before following
78    /// it. Returns the redirect target when the server responds with a 3xx, or
79    /// the final content converted to Markdown otherwise.
80    async fn fetch_step(http_client: Arc<HttpClientWithUrl>, url: &str) -> Result<FetchStep> {
81        let normalized = normalize_url(url);
82
83        let mut response = http_client
84            .get(&normalized, AsyncBody::default(), false)
85            .await?;
86
87        let status = response.status();
88        if status.is_redirection() {
89            let location = response
90                .headers()
91                .get("location")
92                .context("redirect response is missing a Location header")?
93                .to_str()
94                .context("redirect response has an invalid Location header")?;
95            let target = url::Url::parse(&normalized)
96                .with_context(|| format!("could not parse URL {normalized:?}"))?
97                .join(location)
98                .with_context(|| format!("invalid redirect target {location:?}"))?;
99            anyhow::ensure!(
100                matches!(target.scheme(), "http" | "https"),
101                "refusing to follow redirect to non-HTTP(S) URL {target}"
102            );
103            return Ok(FetchStep::Redirect(target.to_string()));
104        }
105
106        let mut body = Vec::new();
107        response
108            .body_mut()
109            .read_to_end(&mut body)
110            .await
111            .context("error reading response body")?;
112
113        if status.is_client_error() {
114            let text = String::from_utf8_lossy(body.as_slice());
115            bail!("status error {}, response: {text:?}", status.as_u16());
116        }
117
118        let Some(content_type) = response.headers().get("content-type") else {
119            bail!("missing Content-Type header");
120        };
121        let content_type = content_type
122            .to_str()
123            .context("invalid Content-Type header")?;
124
125        let content_type = if content_type.starts_with("text/plain") {
126            ContentType::Plaintext
127        } else if content_type.starts_with("application/json") {
128            ContentType::Json
129        } else {
130            ContentType::Html
131        };
132
133        let text = match content_type {
134            ContentType::Html => {
135                let mut handlers: Vec<TagHandler> = vec![
136                    Rc::new(RefCell::new(markdown::WebpageChromeRemover)),
137                    Rc::new(RefCell::new(markdown::ParagraphHandler)),
138                    Rc::new(RefCell::new(markdown::HeadingHandler)),
139                    Rc::new(RefCell::new(markdown::ListHandler)),
140                    Rc::new(RefCell::new(markdown::TableHandler::new())),
141                    Rc::new(RefCell::new(markdown::StyledTextHandler)),
142                ];
143                if normalized.contains("wikipedia.org") {
144                    use html_to_markdown::structure::wikipedia;
145
146                    handlers.push(Rc::new(RefCell::new(wikipedia::WikipediaChromeRemover)));
147                    handlers.push(Rc::new(RefCell::new(wikipedia::WikipediaInfoboxHandler)));
148                    handlers.push(Rc::new(
149                        RefCell::new(wikipedia::WikipediaCodeHandler::new()),
150                    ));
151                } else {
152                    handlers.push(Rc::new(RefCell::new(markdown::CodeHandler)));
153                }
154
155                convert_html_to_markdown(&body[..], &mut handlers)?
156            }
157            ContentType::Plaintext => std::str::from_utf8(&body)?.to_owned(),
158            ContentType::Json => {
159                let json: serde_json::Value = serde_json::from_slice(&body)?;
160
161                format!("```json\n{}\n```", serde_json::to_string_pretty(&json)?)
162            }
163        };
164
165        Ok(FetchStep::Complete(text))
166    }
167}
168
169/// Resolve the host of `url` and confirm it doesn't point into loopback /
170/// private / link-local space, applying the same forbidden-IP policy the
171/// terminal sandbox's proxy uses. Returns an error (including "resolves only to
172/// forbidden addresses") that aborts the fetch.
173///
174/// DNS resolution blocks, so callers should run this off the foreground thread.
175/// See the caller for why this is a gate rather than a full resolve-to-connect
176/// pin.
177fn verify_host_not_forbidden(url: &str) -> Result<()> {
178    let normalized = normalize_url(url);
179    let parsed =
180        url::Url::parse(&normalized).with_context(|| format!("could not parse URL {url:?}"))?;
181    let host = parsed
182        .host_str()
183        .with_context(|| format!("URL {url:?} has no host to reach"))?;
184    // Default to the scheme's port when the URL omits one; resolution needs a
185    // port but the value doesn't affect which IPs a host resolves to.
186    let port = parsed
187        .port_or_known_default()
188        .unwrap_or(if parsed.scheme() == "http" { 80 } else { 443 });
189
190    http_proxy::PinnedHost::resolve(host, port).map(|_pinned| ())?;
191    Ok(())
192}
193
194/// Extracts the host from a fetch URL as a [`http_proxy::HostPattern`] so it can
195/// be matched against the shared network grants. Mirrors the scheme handling in
196/// [`normalize_url`] (defaulting to `https://` when none is given).
197fn host_pattern_for_url(url: &str) -> Result<http_proxy::HostPattern> {
198    let normalized = normalize_url(url);
199    let parsed =
200        url::Url::parse(&normalized).with_context(|| format!("could not parse URL {url:?}"))?;
201    let host = parsed
202        .host_str()
203        .with_context(|| format!("URL {url:?} has no host to authorize network access for"))?;
204    http_proxy::HostPattern::parse(host).map_err(|error| match error {
205        http_proxy::HostPatternError::IpLiteral(_) => anyhow::anyhow!(
206            "cannot fetch {host:?}: loopback and IP-literal hosts can't be granted network \
207             access individually. They are only reachable once unsandboxed access has been \
208             granted (for example, via a terminal command that requests it)."
209        ),
210        error => anyhow::anyhow!("cannot authorize network access to {host:?}: {error}"),
211    })
212}
213
214impl AgentTool for FetchTool {
215    type Input = FetchToolInput;
216    type Output = String;
217
218    const NAME: &'static str = "fetch";
219
220    fn kind() -> acp::ToolKind {
221        acp::ToolKind::Fetch
222    }
223
224    fn allow_in_restricted_mode() -> bool {
225        false
226    }
227
228    fn initial_title(
229        &self,
230        input: Result<Self::Input, serde_json::Value>,
231        _cx: &mut App,
232    ) -> SharedString {
233        match input {
234            Ok(input) => format!("Fetch {}", MarkdownEscaped(&input.url)).into(),
235            Err(_) => "Fetch URL".into(),
236        }
237    }
238
239    fn run(
240        self: Arc<Self>,
241        input: ToolInput<Self::Input>,
242        event_stream: ToolCallEventStream,
243        cx: &mut App,
244    ) -> Task<Result<Self::Output, Self::Output>> {
245        let http_client = self.http_client.clone();
246        cx.spawn(async move |cx| {
247            let input: FetchToolInput = input.recv().await.map_err(|e| e.to_string())?;
248
249            // First, the standard tool-permission gate (honors the fetch tool's
250            // allow/deny/confirm rules).
251            let authorize = cx.update(|cx| {
252                let context =
253                    crate::ToolPermissionContext::new(Self::NAME, vec![input.url.clone()]);
254
255                event_stream.authorize(
256                    format!("Fetch {}", MarkdownInlineCode(&input.url)),
257                    context,
258                    cx,
259                )
260            });
261            futures::select! {
262                result = authorize.fuse() => result.map_err(|e| e.to_string())?,
263                _ = event_stream.cancelled_by_user().fuse() => {
264                    return Err("Fetch cancelled by user".to_string());
265                }
266            };
267
268            // Then, unless unsandboxed access is already in effect, the per-host
269            // network grant shared with the terminal tool. If the host isn't
270            // already granted (for this thread or in saved settings) the user is
271            // shown the same escalation prompt the terminal uses; a denial
272            // aborts the fetch. This tool never runs inside the OS sandbox, so
273            // the grant is only consulted to decide whether the request may
274            // proceed. When unsandboxed access has been granted the terminal
275            // already runs without isolation, so we drop fetch's restrictions
276            // too — including reaching hosts that can't be granted individually
277            // (loopback and IP literals).
278            //
279            // Crucially, this authorization is applied to every redirect hop as
280            // well as the initial URL, so a granted host can't 30x-redirect the
281            // fetch to a host the user never approved. We disable the HTTP
282            // client's own redirect following and re-run the grant for each hop
283            // before requesting it.
284            //
285            // When the sandboxing feature flag is off the terminal isn't
286            // sandboxed either, so gating fetch by host would provide no
287            // isolation; skip it entirely (the pre-sandboxing behavior).
288            let unsandboxed = cx.update(|cx| {
289                !crate::sandboxing::sandboxing_enabled(cx)
290                    || event_stream.unsandboxed_access_granted(cx)
291            });
292
293            let mut current_url = input.url.clone();
294            let mut redirects = 0;
295            let text = loop {
296                if !unsandboxed {
297                    let host = host_pattern_for_url(&current_url).map_err(|e| e.to_string())?;
298                    let authorize_host = cx.update(|cx| {
299                        let request = SandboxRequest {
300                            network: NetworkRequest::Hosts(vec![host]),
301                            ..Default::default()
302                        };
303                        event_stream.authorize_sandbox(request, String::new(), cx)
304                    });
305                    futures::select! {
306                        result = authorize_host.fuse() => result.map_err(|e| e.to_string())?,
307                        _ = event_stream.cancelled_by_user().fuse() => {
308                            return Err("Fetch cancelled by user".to_string());
309                        }
310                    };
311
312                    // Authorizing the *hostname* is not enough: a granted host
313                    // (or a redirect to one) whose DNS points into loopback /
314                    // private / link-local space would otherwise let the model
315                    // reach the local machine or LAN (SSRF / DNS rebinding).
316                    // Resolve and vet the host now, applying the same
317                    // forbidden-IP policy the terminal sandbox's proxy enforces.
318                    //
319                    // NOTE: this is a gate, not a full pin. `HttpClientWithUrl`
320                    // resolves the hostname again when it connects, so a DNS
321                    // answer that flips between this check and that connect could
322                    // still slip through. Closing that residual window would
323                    // require the HTTP client to connect to a pre-vetted IP
324                    // (`PinnedHost::socket_addrs`) rather than re-resolving;
325                    // until then this blocks the realistic case of a stably
326                    // resolving host that points at forbidden space.
327                    let verify_task = cx.background_spawn({
328                        let url = current_url.clone();
329                        async move { verify_host_not_forbidden(&url) }
330                    });
331                    futures::select! {
332                        result = verify_task.fuse() => result.map_err(|e| e.to_string())?,
333                        _ = event_stream.cancelled_by_user().fuse() => {
334                            return Err("Fetch cancelled by user".to_string());
335                        }
336                    };
337                }
338
339                let fetch_task = cx.background_spawn({
340                    let http_client = http_client.clone();
341                    let url = current_url.clone();
342                    async move { Self::fetch_step(http_client, &url).await }
343                });
344
345                let step = futures::select! {
346                    result = fetch_task.fuse() => result.map_err(|e| e.to_string())?,
347                    _ = event_stream.cancelled_by_user().fuse() => {
348                        return Err("Fetch cancelled by user".to_string());
349                    }
350                };
351
352                match step {
353                    FetchStep::Complete(text) => break text,
354                    FetchStep::Redirect(target) => {
355                        redirects += 1;
356                        if redirects > MAX_REDIRECTS {
357                            return Err(format!(
358                                "exceeded the maximum of {MAX_REDIRECTS} redirects"
359                            ));
360                        }
361                        current_url = target;
362                    }
363                }
364            };
365
366            if text.trim().is_empty() {
367                return Err("no textual content found".to_string());
368            }
369            Ok(text)
370        })
371    }
372}
373
374#[cfg(test)]
375mod tests {
376    use super::*;
377
378    // These use IP-literal URLs, which "resolve" to themselves, so the SSRF gate
379    // is exercised without depending on real DNS. IP literals can't be *granted*
380    // network access (that's a separate, earlier check), but they can be the
381    // target a granted hostname redirects to or resolves into — which is exactly
382    // the case this gate defends.
383
384    #[test]
385    fn verify_host_rejects_loopback_literal() {
386        let error = verify_host_not_forbidden("http://127.0.0.1/internal")
387            .expect_err("loopback must be refused");
388        assert!(
389            error.to_string().contains("loopback"),
390            "error should explain the forbidden range, got: {error}"
391        );
392    }
393
394    #[test]
395    fn verify_host_rejects_private_and_metadata_literals() {
396        for url in [
397            "http://10.0.0.5/",
398            "https://192.168.1.1/",
399            "http://169.254.169.254/latest/meta-data/", // cloud metadata
400            "http://[::1]/",
401        ] {
402            assert!(
403                verify_host_not_forbidden(url).is_err(),
404                "expected {url} to be refused as a forbidden destination"
405            );
406        }
407    }
408
409    #[test]
410    fn verify_host_allows_public_literal() {
411        verify_host_not_forbidden("https://93.184.215.14/")
412            .expect("a public address must be allowed through the gate");
413    }
414}
415