tenant.openagents/omega

1use anyhow::{Context, Result};
2use mdbook::BookItem;
3use mdbook::book::Book;
4use regex::Regex;
5use std::collections::HashMap;
6use std::path::{Path, PathBuf};
7use std::sync::OnceLock;
8
9use crate::FRONT_MATTER_COMMENT;
10
11#[derive(Debug)]
12pub(crate) struct DocsPage {
13    section: String,
14    title: String,
15    description: Option<String>,
16    pub(crate) source_path: PathBuf,
17    content: String,
18}
19
20pub(crate) fn write_ai_discovery_artifacts(
21    pages: &[DocsPage],
22    destination: &Path,
23    site_url: &str,
24) -> Result<()> {
25    copy_markdown_sources(destination, site_url, pages)?;
26    write_llms_txt(destination, site_url, pages)?;
27    write_sitemap_xml(destination, site_url, pages)?;
28    Ok(())
29}
30
31pub(crate) fn docs_pages(book: &Book) -> Result<Vec<DocsPage>> {
32    let mut pages = Vec::new();
33    let mut section = "Docs".to_string();
34    for item in book.iter() {
35        let BookItem::Chapter(chapter) = item else {
36            if let BookItem::PartTitle(part_title) = item {
37                section.clone_from(part_title);
38            }
39            continue;
40        };
41        let Some(source_path) = chapter.source_path.as_ref() else {
42            continue;
43        };
44        if source_path == Path::new("SUMMARY.md") {
45            continue;
46        }
47        pages.push(DocsPage {
48            section: section.clone(),
49            title: chapter.name.clone(),
50            description: docs_page_description(&chapter.content),
51            source_path: source_path.clone(),
52            content: chapter.content.clone(),
53        });
54    }
55    Ok(pages)
56}
57
58fn copy_markdown_sources(destination: &Path, site_url: &str, pages: &[DocsPage]) -> Result<()> {
59    for page in pages {
60        let destination = destination.join(&page.source_path);
61        if let Some(parent) = destination.parent() {
62            std::fs::create_dir_all(parent).with_context(|| {
63                format!("failed to create markdown destination {}", parent.display())
64            })?;
65        }
66        let contents = rewrite_docs_links(&markdown_source_contents(&page.content), site_url);
67        std::fs::write(
68            &destination,
69            add_llms_markdown_directive(&contents, site_url),
70        )
71        .with_context(|| {
72            format!(
73                "failed to write markdown page {} to {}",
74                page.source_path.display(),
75                destination.display()
76            )
77        })?;
78    }
79    let getting_started = destination.join("getting-started.md");
80    if getting_started.exists() {
81        std::fs::copy(&getting_started, destination.join("index.md"))
82            .context("failed to write index.md markdown alias")?;
83    }
84    Ok(())
85}
86
87fn markdown_source_contents(contents: &str) -> String {
88    front_matter_comment_regex()
89        .replace(contents, "")
90        .trim_start()
91        .to_string()
92}
93
94fn docs_page_description(contents: &str) -> Option<String> {
95    docs_page_metadata(contents).and_then(|metadata| {
96        metadata
97            .get("description")
98            .map(|description| {
99                description
100                    .trim()
101                    .trim_matches('"')
102                    .split_whitespace()
103                    .collect::<Vec<_>>()
104                    .join(" ")
105            })
106            .filter(|description| !description.is_empty())
107    })
108}
109
110fn docs_page_metadata(contents: &str) -> Option<HashMap<String, String>> {
111    let captures = front_matter_comment_regex().captures(contents)?;
112    serde_json::from_str(&captures[1]).ok()
113}
114
115fn front_matter_comment_regex() -> &'static Regex {
116    static FRONT_MATTER_COMMENT_REGEX: OnceLock<Regex> = OnceLock::new();
117    FRONT_MATTER_COMMENT_REGEX
118        .get_or_init(|| Regex::new(&FRONT_MATTER_COMMENT.replace("{}", "([^\\n]*)")).unwrap())
119}
120
121fn write_llms_txt(destination: &Path, site_url: &str, pages: &[DocsPage]) -> Result<()> {
122    let mut contents = String::new();
123    contents.push_str("# Zed Docs\n\n");
124    contents.push_str(
125        "> Official Omega documentation index with links to Markdown versions of each docs page.\n\n",
126    );
127    contents.push_str(
128        "Use these links for concise Markdown copies of Omega documentation pages. Each linked page mirrors the corresponding `/docs/*.html` page without site navigation or styling.\n\n",
129    );
130    let mut current_section = None;
131    for page in pages {
132        if current_section != Some(page.section.as_str()) {
133            if current_section.is_some() {
134                contents.push('\n');
135            }
136            contents.push_str("## ");
137            contents.push_str(&markdown_text(&page.section));
138            contents.push_str("\n\n");
139            current_section = Some(page.section.as_str());
140        }
141        contents.push_str("- [");
142        contents.push_str(&markdown_text(&page.title));
143        contents.push_str("](");
144        contents.push_str(&absolute_docs_url(site_url, &page.source_path));
145        contents.push(')');
146        if let Some(description) = &page.description {
147            contents.push_str(": ");
148            contents.push_str(&markdown_text(description));
149        }
150        contents.push('\n');
151    }
152    std::fs::write(destination.join("llms.txt"), contents).context("failed to write llms.txt")?;
153    Ok(())
154}
155
156fn markdown_text(text: &str) -> String {
157    text.replace('\\', "\\\\")
158        .replace('[', "\\[")
159        .replace(']', "\\]")
160}
161
162fn write_sitemap_xml(destination: &Path, site_url: &str, pages: &[DocsPage]) -> Result<()> {
163    let mut contents = String::new();
164    contents.push_str("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
165    contents.push_str("<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">\n");
166    for page in pages {
167        contents.push_str("  <url><loc>");
168        contents.push_str(&xml_escape(&absolute_docs_url(
169            site_url,
170            &page.source_path.with_extension("html"),
171        )));
172        contents.push_str("</loc>");
173        contents.push_str("</url>\n");
174    }
175    contents.push_str("</urlset>\n");
176    std::fs::write(destination.join("sitemap.xml"), contents)
177        .context("failed to write sitemap.xml")?;
178    Ok(())
179}
180
181pub(crate) fn write_pages_redirects(
182    destination: &Path,
183    redirects: &[(String, String)],
184    site_url: &str,
185) -> Result<()> {
186    let Some(deploy_root) = destination.parent() else {
187        return Ok(());
188    };
189    let mut contents = String::new();
190    for (source, destination) in redirects {
191        write_redirect_line(
192            &mut contents,
193            &docs_path("/docs/", source),
194            &redirect_destination(site_url, destination),
195        );
196        if let Some(extensionless_source) = strip_html_suffix(source) {
197            write_redirect_line(
198                &mut contents,
199                &docs_path("/docs/", &extensionless_source),
200                &redirect_destination(
201                    site_url,
202                    &strip_html_suffix(destination).unwrap_or_else(|| destination.to_string()),
203                ),
204            );
205        }
206        if let Some(markdown_source) = html_path_to_markdown(source) {
207            if let Some(markdown_destination) = html_path_to_markdown(destination) {
208                write_redirect_line(
209                    &mut contents,
210                    &docs_path("/docs/", &markdown_source),
211                    &redirect_destination(site_url, &markdown_destination),
212                );
213            }
214        }
215    }
216    std::fs::write(deploy_root.join("_redirects"), contents)
217        .context("failed to write Cloudflare Pages _redirects")?;
218    Ok(())
219}
220
221pub(crate) fn write_markdown_redirect_aliases(
222    destination: &Path,
223    redirects: &[(String, String)],
224    site_url: &str,
225) -> Result<()> {
226    for (source, redirect_destination_path) in redirects {
227        let Some(source_markdown) = html_path_to_markdown(source) else {
228            continue;
229        };
230        let Some(destination_markdown) = html_path_to_markdown(redirect_destination_path) else {
231            continue;
232        };
233        let source_markdown = destination.join(source_markdown.trim_start_matches('/'));
234        let destination_markdown =
235            destination.join(destination_markdown.trim_start_matches("/docs/"));
236        if !destination_markdown.exists() {
237            continue;
238        }
239        if let Some(parent) = source_markdown.parent() {
240            std::fs::create_dir_all(parent).with_context(|| {
241                format!(
242                    "failed to create markdown alias directory {}",
243                    parent.display()
244                )
245            })?;
246        }
247        let contents = format!(
248            "# Moved\n\n> For the complete documentation index and Markdown links, see [llms.txt]({}).\n\nThis page moved to [the current docs page]({}).\n",
249            docs_url(site_url, Path::new("llms.txt")),
250            html_path_to_markdown(redirect_destination_path)
251                .map(|path| redirect_destination(site_url, &path))
252                .unwrap_or_else(|| redirect_destination(site_url, redirect_destination_path))
253        );
254        std::fs::write(&source_markdown, contents).with_context(|| {
255            format!(
256                "failed to write markdown redirect alias from {} to {}",
257                redirect_destination_path,
258                source_markdown.display()
259            )
260        })?;
261    }
262    Ok(())
263}
264
265fn write_redirect_line(contents: &mut String, source: &str, destination: &str) {
266    contents.push_str(source);
267    contents.push(' ');
268    contents.push_str(destination);
269    contents.push_str(" 301\n");
270}
271
272fn docs_path(site_url: &str, path: &str) -> String {
273    docs_url(site_url, Path::new(path.trim_start_matches('/')))
274}
275
276fn redirect_destination(site_url: &str, destination: &str) -> String {
277    if let Some(path) = destination.strip_prefix("/docs/") {
278        docs_url(site_url, Path::new(path))
279    } else if destination == "/docs" {
280        docs_url(site_url, Path::new(""))
281    } else {
282        destination.to_string()
283    }
284}
285
286fn strip_html_suffix(path: &str) -> Option<String> {
287    let (path, fragment) = split_fragment(path);
288    let path = path.strip_suffix(".html")?;
289    Some(format!("{path}{fragment}"))
290}
291
292fn html_path_to_markdown(path: &str) -> Option<String> {
293    let (path, fragment) = split_fragment(path);
294    if !path.starts_with("/docs/") && path != "/docs" && !path.ends_with(".html") {
295        return None;
296    }
297    let markdown_path = path.strip_suffix(".html").unwrap_or(path);
298    Some(format!("{markdown_path}.md{fragment}"))
299}
300
301fn split_fragment(path: &str) -> (&str, &str) {
302    match path.find('#') {
303        Some(index) => (&path[..index], &path[index..]),
304        None => (path, ""),
305    }
306}
307
308pub(crate) fn rewrite_docs_links(contents: &str, site_url: &str) -> String {
309    const STABLE_DOCS_PREFIX: &str = "https://zed.dev/docs/";
310    let channel_docs_prefix = absolute_docs_url(site_url, Path::new(""));
311    if channel_docs_prefix == STABLE_DOCS_PREFIX {
312        return contents.to_string();
313    }
314
315    let mut output = String::with_capacity(contents.len());
316    let mut remaining = contents;
317    while let Some(index) = remaining.find(STABLE_DOCS_PREFIX) {
318        output.push_str(&remaining[..index]);
319        let after_prefix = &remaining[index + STABLE_DOCS_PREFIX.len()..];
320        if after_prefix.starts_with("preview/") || after_prefix.starts_with("nightly/") {
321            output.push_str(STABLE_DOCS_PREFIX);
322        } else {
323            output.push_str(&channel_docs_prefix);
324        }
325        remaining = after_prefix;
326    }
327    output.push_str(remaining);
328    output
329}
330
331pub(crate) fn add_markdown_alternate_link(
332    contents: &str,
333    html_file: &Path,
334    root_dir: &Path,
335    site_url: &str,
336) -> String {
337    let Ok(relative_path) = html_file.strip_prefix(root_dir) else {
338        return contents.to_string();
339    };
340    let markdown_path = relative_path.with_extension("md");
341    if !root_dir.join(&markdown_path).exists() {
342        return contents.to_string();
343    }
344    let markdown_url = docs_url(site_url, &markdown_path);
345    let link = format!(
346        "        <link rel=\"alternate\" type=\"text/markdown\" href=\"{}\">\n",
347        markdown_url
348    );
349    contents.replacen("</head>", &(link + "    </head>"), 1)
350}
351
352fn add_llms_markdown_directive(contents: &str, site_url: &str) -> String {
353    let directive = format!(
354        "> For the complete documentation index and Markdown links, see [llms.txt]({}).\n\n",
355        docs_url(site_url, Path::new("llms.txt")),
356    );
357    if let Some(rest) = contents.strip_prefix("---\n") {
358        if let Some(frontmatter_end) = rest.find("\n---\n") {
359            let split_at = "---\n".len() + frontmatter_end + "\n---\n".len();
360            let mut output = String::with_capacity(contents.len() + directive.len());
361            output.push_str(&contents[..split_at]);
362            output.push('\n');
363            output.push_str(&directive);
364            output.push_str(&contents[split_at..]);
365            return output;
366        }
367    }
368
369    let mut output = String::with_capacity(contents.len() + directive.len());
370    output.push_str(&directive);
371    output.push_str(contents);
372    output
373}
374
375fn docs_url(site_url: &str, path: &Path) -> String {
376    let mut url = site_url.to_string();
377    if !url.ends_with('/') {
378        url.push('/');
379    }
380    url.push_str(&path.to_string_lossy().replace('\\', "/"));
381    url
382}
383
384fn absolute_docs_url(site_url: &str, path: &Path) -> String {
385    let url = docs_url(site_url, path);
386    if url.starts_with("http://") || url.starts_with("https://") {
387        url
388    } else {
389        format!("https://zed.dev{}", url)
390    }
391}
392
393fn xml_escape(value: &str) -> String {
394    value
395        .replace('&', "&amp;")
396        .replace('<', "&lt;")
397        .replace('>', "&gt;")
398        .replace('"', "&quot;")
399        .replace('\'', "&apos;")
400}
401
402#[cfg(test)]
403mod tests {
404    use super::*;
405
406    #[test]
407    fn test_add_llms_markdown_directive_inserts_after_frontmatter() {
408        let contents = "---\ntitle: Example\n---\n# Example\n";
409        let output = add_llms_markdown_directive(contents, "/docs/");
410
411        assert!(output.starts_with("---\ntitle: Example\n---\n\n"));
412        assert!(output.contains(
413            "> For the complete documentation index and Markdown links, see [llms.txt](/docs/llms.txt)."
414        ));
415    }
416
417    #[test]
418    fn test_redirect_destination_uses_channel_site_url_for_docs_paths() {
419        assert_eq!(
420            redirect_destination("/docs/preview/", "/docs/ai/overview.html"),
421            "/docs/preview/ai/overview.html"
422        );
423        assert_eq!(
424            redirect_destination("/docs/preview/", "/community-links"),
425            "/community-links"
426        );
427    }
428
429    #[test]
430    fn test_rewrite_docs_links_uses_channel_site_url() {
431        assert_eq!(
432            rewrite_docs_links(
433                "See [Code Actions](https://zed.dev/docs/configuring-languages#code-actions) and [Preview](https://zed.dev/docs/preview/ai/overview.html).",
434                "/docs/preview/"
435            ),
436            "See [Code Actions](https://zed.dev/docs/preview/configuring-languages#code-actions) and [Preview](https://zed.dev/docs/preview/ai/overview.html)."
437        );
438    }
439
440    #[test]
441    fn test_docs_path_uses_channel_site_url() {
442        assert_eq!(
443            docs_path("/docs/preview/", "/assistant.md"),
444            "/docs/preview/assistant.md"
445        );
446    }
447
448    #[test]
449    fn test_write_pages_redirects_keeps_sources_on_internal_pages_path() -> Result<()> {
450        let deploy_root = std::env::temp_dir().join(format!(
451            "docs_preprocessor_pages_redirects_test_{}_{}",
452            std::process::id(),
453            std::time::SystemTime::now()
454                .duration_since(std::time::UNIX_EPOCH)?
455                .as_nanos()
456        ));
457        let destination = deploy_root.join("docs");
458        std::fs::create_dir_all(&destination)?;
459        let redirects = vec![
460            (
461                "/assistant.html".to_string(),
462                "/docs/ai/overview.html".to_string(),
463            ),
464            (
465                "/community/feedback.html".to_string(),
466                "/community-links".to_string(),
467            ),
468        ];
469
470        write_pages_redirects(&destination, &redirects, "/docs/preview/")?;
471
472        assert_eq!(
473            std::fs::read_to_string(deploy_root.join("_redirects"))?,
474            "/docs/assistant.html /docs/preview/ai/overview.html 301\n\
475/docs/assistant /docs/preview/ai/overview 301\n\
476/docs/assistant.md /docs/preview/ai/overview.md 301\n\
477/docs/community/feedback.html /community-links 301\n\
478/docs/community/feedback /community-links 301\n"
479        );
480        std::fs::remove_dir_all(&deploy_root)?;
481        Ok(())
482    }
483
484    #[test]
485    fn test_write_ai_discovery_artifacts_generates_agent_facing_metadata() -> Result<()> {
486        let destination = std::env::temp_dir().join(format!(
487            "docs_preprocessor_ai_discovery_test_{}_{}",
488            std::process::id(),
489            std::time::SystemTime::now()
490                .duration_since(std::time::UNIX_EPOCH)?
491                .as_nanos()
492        ));
493        std::fs::create_dir_all(&destination)?;
494
495        let pages = vec![
496            DocsPage {
497                section: "Docs".to_string(),
498                title: "Getting Started".to_string(),
499                description: Some("Start using Zed.".to_string()),
500                source_path: PathBuf::from("getting-started.md"),
501                content: format!(
502                    "{}\n# Getting Started\n",
503                    FRONT_MATTER_COMMENT.replace("{}", r#"{"description":"Start using Zed."}"#)
504                ),
505            },
506            DocsPage {
507                section: "AI".to_string(),
508                title: "MCP".to_string(),
509                description: Some("Connect model context servers.".to_string()),
510                source_path: PathBuf::from("ai/mcp.md"),
511                content: format!(
512                    "{}\n# MCP\n",
513                    FRONT_MATTER_COMMENT
514                        .replace("{}", r#"{"description":"Connect model context servers."}"#)
515                ),
516            },
517        ];
518
519        write_ai_discovery_artifacts(&pages, &destination, "/docs/")?;
520
521        let llms_txt = std::fs::read_to_string(destination.join("llms.txt"))?;
522        assert!(llms_txt.contains("## Docs"));
523        assert!(llms_txt.contains(
524            "- [Getting Started](https://zed.dev/docs/getting-started.md): Start using Zed."
525        ));
526        assert!(llms_txt.contains("## AI"));
527        assert!(
528            llms_txt.contains(
529                "- [MCP](https://zed.dev/docs/ai/mcp.md): Connect model context servers."
530            )
531        );
532
533        let sitemap_xml = std::fs::read_to_string(destination.join("sitemap.xml"))?;
534        assert!(sitemap_xml.contains("<loc>https://zed.dev/docs/getting-started.html</loc>"));
535        assert!(sitemap_xml.contains("<loc>https://zed.dev/docs/ai/mcp.html</loc>"));
536
537        let mcp_markdown = std::fs::read_to_string(destination.join("ai/mcp.md"))?;
538        assert!(mcp_markdown.starts_with(
539            "> For the complete documentation index and Markdown links, see [llms.txt](/docs/llms.txt).\n\n# MCP"
540        ));
541        assert!(!mcp_markdown.contains("ZED_META"));
542
543        let index_markdown = std::fs::read_to_string(destination.join("index.md"))?;
544        assert!(index_markdown.contains("# Getting Started"));
545
546        std::fs::remove_dir_all(&destination)?;
547        Ok(())
548    }
549}
550