Skip to repository content1832 lines · 68.8 KB · rust
tenant.openagents/omega
No repository description is available.
OpenAgents Git authority 2026-07-28T04:58:37.282Z Public web read
NIP-34 coordinate
30617:7649603503856e5148d571eac2766b288a8ff1e9e35d380337a1d2b0015b4f92:omegaMaintainersHidden in public view
References2 branches · 1 tag
Read-only clone
git clone https://openagents.com/git/tenant.openagents/omega.gitBrowse files
parser.rs
1use collections::{BTreeMap, HashMap, HashSet};
2use gpui::SharedString;
3use linkify::LinkFinder;
4pub use pulldown_cmark::TagEnd as MarkdownTagEnd;
5use pulldown_cmark::{
6 Alignment, CowStr, HeadingLevel, LinkType, MetadataBlockKind, Options, Parser,
7};
8use std::{ops::Range, sync::Arc};
9use util::markdown::generate_heading_slug;
10
11use crate::{html, path_range::PathWithRange};
12
13pub const PARSE_OPTIONS: Options = Options::ENABLE_TABLES
14 .union(Options::ENABLE_FOOTNOTES)
15 .union(Options::ENABLE_STRIKETHROUGH)
16 .union(Options::ENABLE_TASKLISTS)
17 .union(Options::ENABLE_SMART_PUNCTUATION)
18 .union(Options::ENABLE_HEADING_ATTRIBUTES)
19 .union(Options::ENABLE_PLUSES_DELIMITED_METADATA_BLOCKS)
20 .union(Options::ENABLE_OLD_FOOTNOTES)
21 .union(Options::ENABLE_GFM)
22 .union(Options::ENABLE_SUPERSCRIPT)
23 .union(Options::ENABLE_SUBSCRIPT);
24
25#[derive(Default)]
26struct ParseState {
27 events: Vec<(Range<usize>, MarkdownEvent)>,
28 root_block_starts: Vec<usize>,
29 depth: usize,
30}
31
32#[derive(Debug, Default)]
33#[cfg_attr(test, derive(PartialEq))]
34pub(crate) struct ParsedMarkdownData {
35 pub events: Vec<(Range<usize>, MarkdownEvent)>,
36 pub language_names: HashSet<SharedString>,
37 pub language_paths: HashSet<Arc<str>>,
38 pub root_block_starts: Vec<usize>,
39 pub html_blocks: BTreeMap<usize, html::html_parser::ParsedHtmlBlock>,
40 pub metadata_blocks: BTreeMap<usize, ParsedMetadataBlock>,
41 pub heading_slugs: HashMap<SharedString, usize>,
42 pub footnote_definitions: HashMap<SharedString, usize>,
43}
44
45#[derive(Clone, Debug, PartialEq)]
46pub(crate) struct ParsedMetadataBlock {
47 pub content_range: Range<usize>,
48 pub rows: Option<Vec<MetadataRow>>,
49}
50
51#[derive(Clone, Debug, PartialEq)]
52pub(crate) struct MetadataRow {
53 pub key: Range<usize>,
54 pub value: Range<usize>,
55}
56
57impl ParseState {
58 fn push_event(&mut self, range: Range<usize>, event: MarkdownEvent) {
59 match &event {
60 MarkdownEvent::Start(_) => {
61 if self.depth == 0 {
62 self.root_block_starts.push(range.start);
63 self.events.push((range.clone(), MarkdownEvent::RootStart));
64 }
65 self.depth += 1;
66 self.events.push((range, event));
67 }
68 MarkdownEvent::End(_) => {
69 self.events.push((range.clone(), event));
70 if self.depth > 0 {
71 self.depth -= 1;
72 if self.depth == 0 {
73 let root_block_index = self.root_block_starts.len() - 1;
74 self.events
75 .push((range, MarkdownEvent::RootEnd(root_block_index)));
76 }
77 }
78 }
79 MarkdownEvent::Rule => {
80 if self.depth == 0 && !range.is_empty() {
81 self.root_block_starts.push(range.start);
82 let root_block_index = self.root_block_starts.len() - 1;
83 self.events.push((range.clone(), MarkdownEvent::RootStart));
84 self.events.push((range.clone(), event));
85 self.events
86 .push((range, MarkdownEvent::RootEnd(root_block_index)));
87 } else {
88 self.events.push((range, event));
89 }
90 }
91 _ => {
92 self.events.push((range, event));
93 }
94 }
95 }
96}
97
98const MAX_DUPLICATE_HEADING_SLUGS: usize = 128;
99
100fn build_heading_slugs(
101 source: &str,
102 events: &[(Range<usize>, MarkdownEvent)],
103) -> HashMap<SharedString, usize> {
104 let mut slugs = HashMap::default();
105 let mut slug_counts: HashMap<String, usize> = HashMap::default();
106 let mut inside_heading = false;
107 let mut heading_text = String::new();
108 let mut heading_source_start: Option<usize> = None;
109
110 for (range, event) in events {
111 match event {
112 MarkdownEvent::Start(MarkdownTag::Heading { .. }) => {
113 inside_heading = true;
114 heading_text.clear();
115 heading_source_start = None;
116 }
117 MarkdownEvent::End(MarkdownTagEnd::Heading(_)) => {
118 if inside_heading {
119 let source_offset = heading_source_start.unwrap_or(range.start);
120 let base_slug = generate_heading_slug(&heading_text);
121 let count = slug_counts.entry(base_slug.clone()).or_insert(0);
122 let mut slug = if *count == 0 {
123 base_slug.clone()
124 } else {
125 format!("{base_slug}-{count}")
126 };
127 *count += 1;
128 while slugs.contains_key(slug.as_str()) {
129 let Some(count) = slug_counts.get_mut(&base_slug) else {
130 slug.clear();
131 break;
132 };
133 if *count >= MAX_DUPLICATE_HEADING_SLUGS {
134 slug.clear();
135 break;
136 }
137 slug = format!("{base_slug}-{count}");
138 *count += 1;
139 }
140 if !slug.is_empty() {
141 slugs.insert(SharedString::from(slug), source_offset);
142 }
143 inside_heading = false;
144 }
145 }
146 MarkdownEvent::Text | MarkdownEvent::Code if inside_heading => {
147 if heading_source_start.is_none() {
148 heading_source_start = Some(range.start);
149 }
150 heading_text.push_str(&source[range.clone()]);
151 }
152 MarkdownEvent::SubstitutedCode(substituted) if inside_heading => {
153 if heading_source_start.is_none() {
154 heading_source_start = Some(range.start);
155 }
156 heading_text.push_str(substituted);
157 }
158 MarkdownEvent::SubstitutedText(substituted) if inside_heading => {
159 if heading_source_start.is_none() {
160 heading_source_start = Some(range.start);
161 }
162 heading_text.push_str(substituted);
163 }
164 _ => {}
165 }
166 }
167
168 slugs
169}
170
171fn parse_metadata_table_rows(source: &str, source_range: Range<usize>) -> Option<Vec<MetadataRow>> {
172 let mut rows = Vec::new();
173 let mut line_start = source_range.start;
174
175 for line in source[source_range].split_inclusive('\n') {
176 let line_end = line_start + line.len();
177 let content_end = line_start + line.trim_end_matches(['\r', '\n']).len();
178 let content_range = line_start..content_end;
179 let line_text = &source[content_range.clone()];
180
181 if line_text.is_empty()
182 || line_text
183 .chars()
184 .next()
185 .is_some_and(|character| character.is_whitespace())
186 {
187 return None;
188 }
189
190 let delimiter = line_text.find(':')?;
191 let key = trim_metadata_range(source, content_range.start..content_range.start + delimiter);
192 let value = trim_metadata_range(
193 source,
194 content_range.start + delimiter + 1..content_range.end,
195 );
196 if key.is_empty() || value.is_empty() {
197 return None;
198 }
199
200 rows.push(MetadataRow { key, value });
201 line_start = line_end;
202 }
203
204 if rows.is_empty() { None } else { Some(rows) }
205}
206
207fn trim_metadata_range(source: &str, range: Range<usize>) -> Range<usize> {
208 let text = &source[range.clone()];
209 let start_offset = text.len() - text.trim_start().len();
210 let end_offset = text.trim_end().len();
211 let start = range.start + start_offset;
212 let end = (range.start + end_offset).max(start);
213 start..end
214}
215
216fn is_br_tag(html: &str) -> bool {
217 let Some(inner) = html
218 .trim()
219 .strip_prefix('<')
220 .and_then(|s| s.strip_suffix('>'))
221 else {
222 return false;
223 };
224 let inner = inner.strip_suffix('/').unwrap_or(inner);
225 inner
226 .split_ascii_whitespace()
227 .next()
228 .is_some_and(|name| name.eq_ignore_ascii_case("br"))
229}
230
231pub(crate) fn parse_markdown_with_options(
232 text: &str,
233 parse_html: bool,
234 parse_heading_slugs: bool,
235 parse_metadata_blocks: bool,
236) -> ParsedMarkdownData {
237 let mut state = ParseState::default();
238 let mut language_names = HashSet::default();
239 let mut language_paths = HashSet::default();
240 let mut html_blocks = BTreeMap::default();
241 let mut metadata_blocks = BTreeMap::default();
242 let mut within_link = false;
243 let mut within_code_block = false;
244 let mut within_metadata = false;
245 let mut within_table = false;
246 let mut current_metadata_block_start = None;
247 let mut metadata_block_content_range: Option<Range<usize>> = None;
248 let parse_options = if parse_metadata_blocks {
249 PARSE_OPTIONS.union(Options::ENABLE_YAML_STYLE_METADATA_BLOCKS)
250 } else {
251 PARSE_OPTIONS
252 };
253 let mut parser = Parser::new_ext(text, parse_options)
254 .into_offset_iter()
255 .peekable();
256 while let Some((pulldown_event, range)) = parser.next() {
257 if within_metadata && !parse_metadata_blocks {
258 if let pulldown_cmark::Event::End(pulldown_cmark::TagEnd::MetadataBlock(_)) =
259 pulldown_event
260 {
261 within_metadata = false;
262 current_metadata_block_start = None;
263 metadata_block_content_range = None;
264 }
265 continue;
266 }
267 match pulldown_event {
268 pulldown_cmark::Event::Start(tag) => {
269 if let pulldown_cmark::Tag::HtmlBlock = &tag {
270 state.push_event(range.clone(), MarkdownEvent::Start(MarkdownTag::HtmlBlock));
271
272 if parse_html {
273 if let Some(block) =
274 html::html_parser::parse_html_block(&text[range.clone()], range.clone())
275 {
276 html_blocks.insert(range.start, block);
277
278 while let Some((event, end_range)) = parser.next() {
279 if let pulldown_cmark::Event::End(
280 pulldown_cmark::TagEnd::HtmlBlock,
281 ) = event
282 {
283 state.push_event(
284 end_range,
285 MarkdownEvent::End(MarkdownTagEnd::HtmlBlock),
286 );
287 break;
288 }
289 }
290 }
291 }
292 continue;
293 }
294
295 let tag = match tag {
296 pulldown_cmark::Tag::Link {
297 link_type,
298 dest_url,
299 title,
300 id,
301 } => {
302 within_link = true;
303 MarkdownTag::Link {
304 link_type,
305 dest_url: SharedString::from(dest_url.into_string()),
306 title: SharedString::from(title.into_string()),
307 id: SharedString::from(id.into_string()),
308 }
309 }
310 pulldown_cmark::Tag::MetadataBlock(kind) => {
311 within_metadata = true;
312 current_metadata_block_start = Some(range.start);
313 metadata_block_content_range = None;
314 if !parse_metadata_blocks {
315 continue;
316 }
317 MarkdownTag::MetadataBlock(kind)
318 }
319 pulldown_cmark::Tag::CodeBlock(pulldown_cmark::CodeBlockKind::Indented) => {
320 within_code_block = true;
321 MarkdownTag::CodeBlock {
322 kind: CodeBlockKind::Indented,
323 metadata: CodeBlockMetadata {
324 content_range: range.clone(),
325 line_count: 1,
326 is_fenced_closed: false,
327 },
328 }
329 }
330 pulldown_cmark::Tag::CodeBlock(pulldown_cmark::CodeBlockKind::Fenced(
331 ref info,
332 )) => {
333 within_code_block = true;
334 let content_range = extract_code_block_content_range(&text[range.clone()]);
335 let content_range =
336 content_range.start + range.start..content_range.end + range.start;
337
338 // Valid to use bytes since multi-byte UTF-8 doesn't use ASCII chars.
339 let line_count = text[content_range.clone()]
340 .bytes()
341 .filter(|c| *c == b'\n')
342 .count();
343 let is_fenced_closed = {
344 let code_block_source = &text[range.clone()];
345 code_block_source
346 .trim_end()
347 .lines()
348 .last()
349 .is_some_and(|line| {
350 let trimmed = line.trim_start();
351 trimmed.len() >= 3 && trimmed.chars().all(|c| c == '`')
352 })
353 };
354
355 let metadata = CodeBlockMetadata {
356 content_range,
357 line_count,
358 is_fenced_closed,
359 };
360
361 let info = info.trim();
362 let kind = if info.is_empty() {
363 CodeBlockKind::Fenced
364 // Languages should never contain a slash, and PathRanges always should.
365 // (Models are told to specify them relative to a workspace root.)
366 } else if info.contains('/') {
367 let path_range = PathWithRange::new(info);
368 language_paths.insert(path_range.path.clone());
369 CodeBlockKind::FencedSrc(path_range)
370 } else {
371 let language = SharedString::from(info.to_string());
372 language_names.insert(language.clone());
373 CodeBlockKind::FencedLang(language)
374 };
375
376 MarkdownTag::CodeBlock { kind, metadata }
377 }
378 pulldown_cmark::Tag::Paragraph => MarkdownTag::Paragraph,
379 pulldown_cmark::Tag::Heading {
380 level,
381 id,
382 classes,
383 attrs,
384 } => {
385 let id = id.map(|id| SharedString::from(id.into_string()));
386 let classes = classes
387 .into_iter()
388 .map(|c| SharedString::from(c.into_string()))
389 .collect();
390 let attrs = attrs
391 .into_iter()
392 .map(|(key, value)| {
393 (
394 SharedString::from(key.into_string()),
395 value.map(|v| SharedString::from(v.into_string())),
396 )
397 })
398 .collect();
399 MarkdownTag::Heading {
400 level,
401 id,
402 classes,
403 attrs,
404 }
405 }
406 pulldown_cmark::Tag::BlockQuote(kind) => MarkdownTag::BlockQuote(kind),
407 pulldown_cmark::Tag::List(start_number) => MarkdownTag::List(start_number),
408 pulldown_cmark::Tag::Item => MarkdownTag::Item,
409 pulldown_cmark::Tag::FootnoteDefinition(label) => {
410 MarkdownTag::FootnoteDefinition(SharedString::from(label.to_string()))
411 }
412 pulldown_cmark::Tag::Table(alignments) => {
413 within_table = true;
414 MarkdownTag::Table(alignments)
415 }
416 pulldown_cmark::Tag::TableHead => MarkdownTag::TableHead,
417 pulldown_cmark::Tag::TableRow => MarkdownTag::TableRow,
418 pulldown_cmark::Tag::TableCell => MarkdownTag::TableCell,
419 pulldown_cmark::Tag::Emphasis => MarkdownTag::Emphasis,
420 pulldown_cmark::Tag::Strong => MarkdownTag::Strong,
421 pulldown_cmark::Tag::Strikethrough => MarkdownTag::Strikethrough,
422 pulldown_cmark::Tag::Superscript => MarkdownTag::Superscript,
423 pulldown_cmark::Tag::Subscript => MarkdownTag::Subscript,
424 pulldown_cmark::Tag::Image {
425 link_type,
426 dest_url,
427 title,
428 id,
429 } => MarkdownTag::Image {
430 link_type,
431 dest_url: SharedString::from(dest_url.into_string()),
432 title: SharedString::from(title.into_string()),
433 id: SharedString::from(id.into_string()),
434 },
435 pulldown_cmark::Tag::HtmlBlock => MarkdownTag::HtmlBlock, // this is handled above separately
436 pulldown_cmark::Tag::DefinitionList => MarkdownTag::DefinitionList,
437 pulldown_cmark::Tag::DefinitionListTitle => MarkdownTag::DefinitionListTitle,
438 pulldown_cmark::Tag::DefinitionListDefinition => {
439 MarkdownTag::DefinitionListDefinition
440 }
441 };
442 state.push_event(range, MarkdownEvent::Start(tag))
443 }
444 pulldown_cmark::Event::End(tag) => {
445 if let pulldown_cmark::TagEnd::Link = tag {
446 within_link = false;
447 } else if let pulldown_cmark::TagEnd::CodeBlock = tag {
448 within_code_block = false;
449 } else if let pulldown_cmark::TagEnd::MetadataBlock(_) = tag {
450 within_metadata = false;
451 let block_start = current_metadata_block_start.take();
452 let content_range = metadata_block_content_range.take();
453 if parse_metadata_blocks
454 && let (Some(block_start), Some(content_range)) =
455 (block_start, content_range)
456 {
457 metadata_blocks.insert(
458 block_start,
459 ParsedMetadataBlock {
460 rows: parse_metadata_table_rows(text, content_range.clone()),
461 content_range,
462 },
463 );
464 }
465 if !parse_metadata_blocks {
466 continue;
467 }
468 } else if let pulldown_cmark::TagEnd::Table = tag {
469 within_table = false;
470 }
471 state.push_event(range, MarkdownEvent::End(tag));
472 }
473 pulldown_cmark::Event::Text(parsed) => {
474 fn event_for(
475 text: &str,
476 range: Range<usize>,
477 str: &str,
478 ) -> (Range<usize>, MarkdownEvent) {
479 if str == &text[range.clone()] {
480 (range, MarkdownEvent::Text)
481 } else {
482 (range, MarkdownEvent::SubstitutedText(str.to_owned()))
483 }
484 }
485
486 if within_metadata {
487 match &mut metadata_block_content_range {
488 Some(content_range) => {
489 content_range.start = content_range.start.min(range.start);
490 content_range.end = content_range.end.max(range.end);
491 }
492 None => metadata_block_content_range = Some(range.clone()),
493 }
494 state.push_event(range, MarkdownEvent::Text);
495 continue;
496 }
497
498 if within_code_block {
499 let (range, event) = event_for(text, range, &parsed);
500 state.push_event(range, event);
501 continue;
502 }
503
504 #[derive(Debug)]
505 struct TextRange<'a> {
506 source_range: Range<usize>,
507 merged_range: Range<usize>,
508 parsed: CowStr<'a>,
509 }
510
511 let mut last_len = parsed.len();
512 let mut ranges = vec![TextRange {
513 source_range: range.clone(),
514 merged_range: 0..last_len,
515 parsed,
516 }];
517
518 while match parser.peek() {
519 Some((pulldown_cmark::Event::Text(_), _)) => true,
520 Some((pulldown_cmark::Event::InlineHtml(html), _)) => {
521 parse_html && !is_br_tag(html)
522 }
523 _ => false,
524 } {
525 let Some((next_event, next_range)) = parser.next() else {
526 unreachable!()
527 };
528 let next_text = match next_event {
529 pulldown_cmark::Event::Text(next_event) => next_event,
530 pulldown_cmark::Event::InlineHtml(_) => CowStr::Borrowed(""),
531 _ => unreachable!(),
532 };
533 let next_len = last_len + next_text.len();
534 ranges.push(TextRange {
535 source_range: next_range.clone(),
536 merged_range: last_len..next_len,
537 parsed: next_text,
538 });
539 last_len = next_len;
540 }
541
542 let mut merged_text =
543 String::with_capacity(ranges.last().unwrap().merged_range.end);
544 for range in &ranges {
545 merged_text.push_str(&range.parsed);
546 }
547
548 let mut ranges = ranges.into_iter().peekable();
549
550 if !within_link && !within_code_block {
551 let mut finder = LinkFinder::new();
552 finder.kinds(&[linkify::LinkKind::Url]);
553
554 // Find links in the merged text
555 for link in finder.links(&merged_text) {
556 let link_start_in_merged = link.start();
557 let link_end_in_merged = link.end();
558
559 while ranges
560 .peek()
561 .is_some_and(|range| range.merged_range.end <= link_start_in_merged)
562 {
563 let range = ranges.next().unwrap();
564 let (range, event) = event_for(text, range.source_range, &range.parsed);
565 state.push_event(range, event);
566 }
567
568 let Some(range) = ranges.peek_mut() else {
569 continue;
570 };
571 let prefix_len = link_start_in_merged - range.merged_range.start;
572 if prefix_len > 0 {
573 let (head, tail) = range.parsed.split_at(prefix_len);
574 let (event_range, event) = event_for(
575 text,
576 range.source_range.start..range.source_range.start + prefix_len,
577 head,
578 );
579 state.push_event(event_range, event);
580 range.parsed = CowStr::Boxed(tail.into());
581 range.merged_range.start += prefix_len;
582 range.source_range.start += prefix_len;
583 }
584
585 let link_start_in_source = range.source_range.start;
586 let mut link_end_in_source = range.source_range.end;
587 let mut link_events = Vec::new();
588
589 while ranges
590 .peek()
591 .is_some_and(|range| range.merged_range.end <= link_end_in_merged)
592 {
593 let range = ranges.next().unwrap();
594 link_end_in_source = range.source_range.end;
595 link_events.push(event_for(text, range.source_range, &range.parsed));
596 }
597
598 if let Some(range) = ranges.peek_mut() {
599 let prefix_len = link_end_in_merged - range.merged_range.start;
600 if prefix_len > 0 {
601 let (head, tail) = range.parsed.split_at(prefix_len);
602 link_events.push(event_for(
603 text,
604 range.source_range.start..range.source_range.start + prefix_len,
605 head,
606 ));
607 range.parsed = CowStr::Boxed(tail.into());
608 range.merged_range.start += prefix_len;
609 range.source_range.start += prefix_len;
610 link_end_in_source = range.source_range.start;
611 }
612 }
613 let link_range = link_start_in_source..link_end_in_source;
614
615 state.push_event(
616 link_range.clone(),
617 MarkdownEvent::Start(MarkdownTag::Link {
618 link_type: LinkType::Autolink,
619 dest_url: SharedString::from(link.as_str().to_string()),
620 title: SharedString::default(),
621 id: SharedString::default(),
622 }),
623 );
624 for (range, event) in link_events {
625 state.push_event(range, event);
626 }
627 state.push_event(
628 link_range.clone(),
629 MarkdownEvent::End(MarkdownTagEnd::Link),
630 );
631 }
632 }
633
634 for range in ranges {
635 let (range, event) = event_for(text, range.source_range, &range.parsed);
636 state.push_event(range, event);
637 }
638 }
639 pulldown_cmark::Event::Code(parsed) => {
640 let content_range = extract_code_content_range(&text[range.clone()]);
641 let content_range =
642 content_range.start + range.start..content_range.end + range.start;
643 let source = &text[content_range.clone()];
644 let event = if within_table && source.contains(r"\|") {
645 MarkdownEvent::SubstitutedCode(parsed.to_string())
646 } else {
647 MarkdownEvent::Code
648 };
649 state.push_event(content_range, event)
650 }
651 pulldown_cmark::Event::Html(_) => state.push_event(range, MarkdownEvent::Html),
652 pulldown_cmark::Event::InlineHtml(html) => {
653 if parse_html && is_br_tag(&html) {
654 state.push_event(range, MarkdownEvent::HardBreak)
655 } else {
656 state.push_event(range, MarkdownEvent::InlineHtml)
657 }
658 }
659 pulldown_cmark::Event::FootnoteReference(label) => state.push_event(
660 range,
661 MarkdownEvent::FootnoteReference(SharedString::from(label.to_string())),
662 ),
663 pulldown_cmark::Event::SoftBreak => state.push_event(range, MarkdownEvent::SoftBreak),
664 pulldown_cmark::Event::HardBreak => state.push_event(range, MarkdownEvent::HardBreak),
665 pulldown_cmark::Event::Rule => state.push_event(range, MarkdownEvent::Rule),
666 pulldown_cmark::Event::TaskListMarker(checked) => {
667 state.push_event(range, MarkdownEvent::TaskListMarker(checked))
668 }
669 pulldown_cmark::Event::InlineMath(_) | pulldown_cmark::Event::DisplayMath(_) => {}
670 }
671 }
672
673 let heading_slugs = if parse_heading_slugs {
674 build_heading_slugs(text, &state.events)
675 } else {
676 HashMap::default()
677 };
678 let footnote_definitions = build_footnote_definitions(&state.events);
679
680 ParsedMarkdownData {
681 events: state.events,
682 language_names,
683 language_paths,
684 root_block_starts: state.root_block_starts,
685 html_blocks,
686 metadata_blocks,
687 heading_slugs,
688 footnote_definitions,
689 }
690}
691
692fn build_footnote_definitions(
693 events: &[(Range<usize>, MarkdownEvent)],
694) -> HashMap<SharedString, usize> {
695 let mut definitions = HashMap::default();
696 let mut current_label: Option<SharedString> = None;
697
698 for (range, event) in events {
699 match event {
700 MarkdownEvent::Start(MarkdownTag::FootnoteDefinition(label)) => {
701 current_label = Some(label.clone());
702 }
703 MarkdownEvent::End(MarkdownTagEnd::FootnoteDefinition) => {
704 current_label = None;
705 }
706 MarkdownEvent::Text if current_label.is_some() => {
707 if let Some(label) = current_label.take() {
708 definitions.entry(label).or_insert(range.start);
709 }
710 }
711 _ => {}
712 }
713 }
714
715 definitions
716}
717
718pub fn parse_links_only(text: &str) -> Vec<(Range<usize>, MarkdownEvent)> {
719 let mut events = Vec::new();
720 let mut finder = LinkFinder::new();
721 finder.kinds(&[linkify::LinkKind::Url]);
722 let mut text_range = Range {
723 start: 0,
724 end: text.len(),
725 };
726 for link in finder.links(text) {
727 let link_range = link.start()..link.end();
728
729 if link_range.start > text_range.start {
730 events.push((text_range.start..link_range.start, MarkdownEvent::Text));
731 }
732
733 events.push((
734 link_range.clone(),
735 MarkdownEvent::Start(MarkdownTag::Link {
736 link_type: LinkType::Autolink,
737 dest_url: SharedString::from(link.as_str().to_string()),
738 title: SharedString::default(),
739 id: SharedString::default(),
740 }),
741 ));
742 events.push((link_range.clone(), MarkdownEvent::Text));
743 events.push((link_range.clone(), MarkdownEvent::End(MarkdownTagEnd::Link)));
744
745 text_range.start = link_range.end;
746 }
747
748 if text_range.end > text_range.start {
749 events.push((text_range, MarkdownEvent::Text));
750 }
751
752 events
753}
754
755/// A static-lifetime equivalent of pulldown_cmark::Event so we can cache the
756/// parse result for rendering without resorting to unsafe lifetime coercion.
757#[derive(Clone, Debug, PartialEq)]
758pub enum MarkdownEvent {
759 /// Start of a tagged element. Events that are yielded after this event
760 /// and before its corresponding `End` event are inside this element.
761 /// Start and end events are guaranteed to be balanced.
762 Start(MarkdownTag),
763 /// End of a tagged element.
764 End(MarkdownTagEnd),
765 /// Text that uses the associated range from the markdown source.
766 Text,
767 /// Text that differs from the markdown source - typically due to substitution of HTML entities
768 /// and smart punctuation.
769 SubstitutedText(String),
770 /// An inline code node.
771 Code,
772 /// An inline code node that differs from the markdown source due to escape decoding.
773 SubstitutedCode(String),
774 /// An HTML node.
775 Html,
776 /// An inline HTML node.
777 InlineHtml,
778 /// A reference to a footnote with given label, which may or may not be defined
779 /// by an event with a `Tag::FootnoteDefinition` tag. Definitions and references to them may
780 /// occur in any order.
781 FootnoteReference(SharedString),
782 /// A soft line break.
783 SoftBreak,
784 /// A hard line break.
785 HardBreak,
786 /// A horizontal ruler.
787 Rule,
788 /// A task list marker, rendered as a checkbox in HTML. Contains a true when it is checked.
789 TaskListMarker(bool),
790 /// Start of a root-level block (a top-level structural element like a paragraph, heading, list, etc.).
791 RootStart,
792 /// End of a root-level block. Contains the root block index.
793 RootEnd(usize),
794}
795
796/// Tags for elements that can contain other elements.
797#[derive(Clone, Debug, PartialEq)]
798pub enum MarkdownTag {
799 /// A paragraph of text and other inline elements.
800 Paragraph,
801
802 /// A heading, with optional identifier, classes and custom attributes.
803 /// The identifier is prefixed with `#` and the last one in the attributes
804 /// list is chosen, classes are prefixed with `.` and custom attributes
805 /// have no prefix and can optionally have a value (`myattr` o `myattr=myvalue`).
806 Heading {
807 level: HeadingLevel,
808 id: Option<SharedString>,
809 classes: Vec<SharedString>,
810 /// The first item of the tuple is the attr and second one the value.
811 attrs: Vec<(SharedString, Option<SharedString>)>,
812 },
813
814 BlockQuote(Option<pulldown_cmark::BlockQuoteKind>),
815
816 /// A code block.
817 CodeBlock {
818 kind: CodeBlockKind,
819 metadata: CodeBlockMetadata,
820 },
821
822 /// A HTML block.
823 HtmlBlock,
824
825 /// A list. If the list is ordered the field indicates the number of the first item.
826 /// Contains only list items.
827 List(Option<u64>), // TODO: add delim and tight for ast (not needed for html)
828
829 /// A list item.
830 Item,
831
832 /// A footnote definition. The value contained is the footnote's label by which it can
833 /// be referred to.
834 FootnoteDefinition(SharedString),
835
836 /// A table. Contains a vector describing the text-alignment for each of its columns.
837 Table(Vec<Alignment>),
838
839 /// A table header. Contains only `TableCell`s. Note that the table body starts immediately
840 /// after the closure of the `TableHead` tag. There is no `TableBody` tag.
841 TableHead,
842
843 /// A table row. Is used both for header rows as body rows. Contains only `TableCell`s.
844 TableRow,
845 TableCell,
846
847 // span-level tags
848 Emphasis,
849 Strong,
850 Strikethrough,
851 Superscript,
852 Subscript,
853
854 /// A link.
855 Link {
856 link_type: LinkType,
857 dest_url: SharedString,
858 title: SharedString,
859 /// Identifier of reference links, e.g. `world` in the link `[hello][world]`.
860 id: SharedString,
861 },
862
863 /// An image. The first field is the link type, the second the destination URL and the third is a title,
864 /// the fourth is the link identifier.
865 Image {
866 link_type: LinkType,
867 dest_url: SharedString,
868 title: SharedString,
869 /// Identifier of reference links, e.g. `world` in the link `[hello][world]`.
870 id: SharedString,
871 },
872
873 /// A metadata block.
874 MetadataBlock(MetadataBlockKind),
875
876 DefinitionList,
877 DefinitionListTitle,
878 DefinitionListDefinition,
879}
880
881#[derive(Clone, Debug, PartialEq)]
882pub enum CodeBlockKind {
883 Indented,
884 /// "Fenced" means "surrounded by triple backticks."
885 /// There can optionally be either a language after the backticks (like in traditional Markdown)
886 /// or, if an agent is specifying a path for a source location in the project, it can be a PathRange,
887 /// e.g. ```path/to/foo.rs#L123-456 instead of ```rust
888 Fenced,
889 FencedLang(SharedString),
890 FencedSrc(PathWithRange),
891}
892
893#[derive(Default, Clone, Debug, PartialEq)]
894pub struct CodeBlockMetadata {
895 pub content_range: Range<usize>,
896 pub line_count: usize,
897 pub is_fenced_closed: bool,
898}
899
900fn extract_code_content_range(text: &str) -> Range<usize> {
901 let text_len = text.len();
902 if text_len == 0 {
903 return 0..0;
904 }
905
906 let start_ticks = text.chars().take_while(|&c| c == '`').count();
907
908 if start_ticks == 0 || start_ticks > text_len {
909 return 0..text_len;
910 }
911
912 let end_ticks = text.chars().rev().take_while(|&c| c == '`').count();
913
914 if end_ticks != start_ticks || text_len < start_ticks + end_ticks {
915 return 0..text_len;
916 }
917
918 start_ticks..text_len - end_ticks
919}
920
921pub(crate) fn extract_code_block_content_range(text: &str) -> Range<usize> {
922 let mut range = 0..text.len();
923 if text.starts_with("```") {
924 range.start += 3;
925
926 if let Some(newline_ix) = text[range.clone()].find('\n') {
927 range.start += newline_ix + 1;
928 }
929 }
930
931 if !range.is_empty() && text.ends_with("```") {
932 range.end -= 3;
933 }
934 if range.start > range.end {
935 range.end = range.start;
936 }
937 range
938}
939
940#[cfg(test)]
941mod tests {
942 use super::MarkdownEvent::*;
943 use super::MarkdownTag::*;
944 use super::*;
945
946 const CONDITIONAL_OPTIONS: Options = Options::ENABLE_YAML_STYLE_METADATA_BLOCKS;
947 const UNWANTED_OPTIONS: Options = Options::ENABLE_MATH
948 .union(Options::ENABLE_DEFINITION_LIST)
949 .union(Options::ENABLE_WIKILINKS);
950
951 #[test]
952 fn all_options_considered() {
953 // The purpose of this is to fail when new options are added to pulldown_cmark, so that they
954 // can be evaluated for inclusion.
955 assert_eq!(
956 PARSE_OPTIONS
957 .union(CONDITIONAL_OPTIONS)
958 .union(UNWANTED_OPTIONS),
959 Options::all()
960 );
961 }
962
963 #[test]
964 fn wanted_and_unwanted_options_disjoint() {
965 assert_eq!(
966 PARSE_OPTIONS
967 .union(CONDITIONAL_OPTIONS)
968 .intersection(UNWANTED_OPTIONS),
969 Options::empty()
970 );
971 }
972
973 #[test]
974 fn test_yaml_style_metadata_block() {
975 assert_eq!(
976 parse_markdown_with_options("---\ntitle: Post\n---\n# Heading", false, false, true),
977 ParsedMarkdownData {
978 events: vec![
979 (0..19, RootStart),
980 (0..19, Start(MetadataBlock(MetadataBlockKind::YamlStyle))),
981 (4..16, Text),
982 (
983 0..19,
984 End(MarkdownTagEnd::MetadataBlock(MetadataBlockKind::YamlStyle))
985 ),
986 (0..19, RootEnd(0)),
987 (20..29, RootStart),
988 (
989 20..29,
990 Start(Heading {
991 level: HeadingLevel::H1,
992 id: None,
993 classes: Vec::new(),
994 attrs: Vec::new(),
995 })
996 ),
997 (22..29, Text),
998 (20..29, End(MarkdownTagEnd::Heading(HeadingLevel::H1))),
999 (20..29, RootEnd(1)),
1000 ],
1001 root_block_starts: vec![0, 20],
1002 metadata_blocks: BTreeMap::from_iter([(
1003 0,
1004 ParsedMetadataBlock {
1005 content_range: 4..16,
1006 rows: Some(vec![MetadataRow {
1007 key: 4..9,
1008 value: 11..15,
1009 }]),
1010 },
1011 )]),
1012 ..Default::default()
1013 }
1014 )
1015 }
1016
1017 #[test]
1018 fn test_metadata_block_text_is_verbatim() {
1019 let parsed =
1020 parse_markdown_with_options("---\nurl: https://zed.dev\n---\nBody", false, false, true);
1021 assert!(
1022 parsed
1023 .events
1024 .iter()
1025 .all(|(_, event)| !matches!(event, Start(Link { .. })))
1026 );
1027 }
1028
1029 #[test]
1030 fn test_metadata_blocks_store_table_rows() {
1031 let parsed = parse_markdown_with_options(
1032 "---\ntitle: Post\nauthor: Zed\n---\nBody",
1033 false,
1034 false,
1035 true,
1036 );
1037
1038 assert_eq!(
1039 parsed.metadata_blocks,
1040 BTreeMap::from_iter([(
1041 0,
1042 ParsedMetadataBlock {
1043 content_range: 4..28,
1044 rows: Some(vec![
1045 MetadataRow {
1046 key: 4..9,
1047 value: 11..15,
1048 },
1049 MetadataRow {
1050 key: 16..22,
1051 value: 24..27,
1052 },
1053 ]),
1054 },
1055 )])
1056 );
1057 }
1058
1059 #[test]
1060 fn test_metadata_blocks_store_fallback_for_nested_yaml() {
1061 let parsed =
1062 parse_markdown_with_options("---\ntags:\n - zed\n---\nBody", false, false, true);
1063
1064 assert_eq!(
1065 parsed.metadata_blocks,
1066 BTreeMap::from_iter([(
1067 0,
1068 ParsedMetadataBlock {
1069 content_range: 4..18,
1070 rows: None,
1071 },
1072 )])
1073 );
1074 }
1075
1076 #[test]
1077 fn test_metadata_table_rows_parse_simple_colon_pairs() {
1078 let source = "title: Post\nauthor: Zed\n";
1079 let Some(rows) = parse_metadata_table_rows(source, 0..source.len()) else {
1080 panic!("expected metadata rows");
1081 };
1082 let pairs = rows
1083 .into_iter()
1084 .map(|row| (&source[row.key], &source[row.value]))
1085 .collect::<Vec<_>>();
1086
1087 assert_eq!(pairs, vec![("title", "Post"), ("author", "Zed")]);
1088 }
1089
1090 #[test]
1091 fn test_metadata_table_rows_reject_non_simple_colon_pairs() {
1092 for source in [
1093 "tags:\n - zed\n",
1094 "title = Post\n",
1095 "title:\n",
1096 "title: \n",
1097 ": Post\n",
1098 " title: Post\n",
1099 "\n",
1100 ] {
1101 assert!(parse_metadata_table_rows(source, 0..source.len()).is_none());
1102 }
1103 }
1104
1105 #[test]
1106 fn test_trim_metadata_range_returns_valid_empty_range() {
1107 let source = "key: \n";
1108 let trimmed = trim_metadata_range(source, 4..7);
1109
1110 assert_eq!(trimmed, 7..7);
1111 assert!(source[trimmed].is_empty());
1112 }
1113
1114 #[test]
1115 fn test_html_comments() {
1116 assert_eq!(
1117 parse_markdown_with_options(
1118 " <!--\nrdoc-file=string.c\n-->\nReturns",
1119 false,
1120 false,
1121 false
1122 ),
1123 ParsedMarkdownData {
1124 events: vec![
1125 (2..30, RootStart),
1126 (2..30, Start(HtmlBlock)),
1127 (2..2, SubstitutedText(" ".into())),
1128 (2..7, Html),
1129 (7..26, Html),
1130 (26..30, Html),
1131 (2..30, End(MarkdownTagEnd::HtmlBlock)),
1132 (2..30, RootEnd(0)),
1133 (30..37, RootStart),
1134 (30..37, Start(Paragraph)),
1135 (30..37, Text),
1136 (30..37, End(MarkdownTagEnd::Paragraph)),
1137 (30..37, RootEnd(1)),
1138 ],
1139 root_block_starts: vec![2, 30],
1140 ..Default::default()
1141 }
1142 )
1143 }
1144
1145 #[test]
1146 fn test_plain_urls_and_escaped_text() {
1147 assert_eq!(
1148 parse_markdown_with_options(
1149 " https://some.url some \\`►\\` text",
1150 false,
1151 false,
1152 false,
1153 ),
1154 ParsedMarkdownData {
1155 events: vec![
1156 (0..51, RootStart),
1157 (0..51, Start(Paragraph)),
1158 (0..6, SubstitutedText("\u{a0}".into())),
1159 (6..12, SubstitutedText("\u{a0}".into())),
1160 (12..13, Text),
1161 (
1162 13..29,
1163 Start(Link {
1164 link_type: LinkType::Autolink,
1165 dest_url: "https://some.url".into(),
1166 title: "".into(),
1167 id: "".into(),
1168 })
1169 ),
1170 (13..29, Text),
1171 (13..29, End(MarkdownTagEnd::Link)),
1172 (29..35, Text),
1173 (36..37, Text), // Escaped backtick
1174 (37..44, SubstitutedText("►".into())),
1175 (45..46, Text), // Escaped backtick
1176 (46..51, Text),
1177 (0..51, End(MarkdownTagEnd::Paragraph)),
1178 (0..51, RootEnd(0)),
1179 ],
1180 root_block_starts: vec![0],
1181 ..Default::default()
1182 }
1183 );
1184 }
1185
1186 #[test]
1187 fn test_incomplete_link() {
1188 assert_eq!(
1189 parse_markdown_with_options(
1190 "You can use the [GitHub Search API](https://docs.github.com/en",
1191 false,
1192 false,
1193 false,
1194 )
1195 .events,
1196 vec![
1197 (0..62, RootStart),
1198 (0..62, Start(Paragraph)),
1199 (0..16, Text),
1200 (16..17, Text),
1201 (17..34, Text),
1202 (34..35, Text),
1203 (35..36, Text),
1204 (
1205 36..62,
1206 Start(Link {
1207 link_type: LinkType::Autolink,
1208 dest_url: "https://docs.github.com/en".into(),
1209 title: "".into(),
1210 id: "".into()
1211 })
1212 ),
1213 (36..62, Text),
1214 (36..62, End(MarkdownTagEnd::Link)),
1215 (0..62, End(MarkdownTagEnd::Paragraph)),
1216 (0..62, RootEnd(0)),
1217 ],
1218 );
1219 }
1220
1221 #[test]
1222 fn test_smart_punctuation() {
1223 assert_eq!(
1224 parse_markdown_with_options(
1225 "-- --- ... \"double quoted\" 'single quoted' ----------",
1226 false,
1227 false,
1228 false,
1229 ),
1230 ParsedMarkdownData {
1231 events: vec![
1232 (0..53, RootStart),
1233 (0..53, Start(Paragraph)),
1234 (0..2, SubstitutedText("–".into())),
1235 (2..3, Text),
1236 (3..6, SubstitutedText("—".into())),
1237 (6..7, Text),
1238 (7..10, SubstitutedText("…".into())),
1239 (10..11, Text),
1240 (11..12, SubstitutedText("\u{201c}".into())),
1241 (12..25, Text),
1242 (25..26, SubstitutedText("\u{201d}".into())),
1243 (26..27, Text),
1244 (27..28, SubstitutedText("\u{2018}".into())),
1245 (28..41, Text),
1246 (41..42, SubstitutedText("\u{2019}".into())),
1247 (42..43, Text),
1248 (43..53, SubstitutedText("–––––".into())),
1249 (0..53, End(MarkdownTagEnd::Paragraph)),
1250 (0..53, RootEnd(0)),
1251 ],
1252 root_block_starts: vec![0],
1253 ..Default::default()
1254 }
1255 )
1256 }
1257
1258 #[test]
1259 fn test_code_block_metadata() {
1260 assert_eq!(
1261 parse_markdown_with_options(
1262 "```rust\nfn main() {\n let a = 1;\n}\n```",
1263 false,
1264 false,
1265 false
1266 ),
1267 ParsedMarkdownData {
1268 events: vec![
1269 (0..37, RootStart),
1270 (
1271 0..37,
1272 Start(CodeBlock {
1273 kind: CodeBlockKind::FencedLang("rust".into()),
1274 metadata: CodeBlockMetadata {
1275 content_range: 8..34,
1276 line_count: 3,
1277 is_fenced_closed: true,
1278 }
1279 })
1280 ),
1281 (8..34, Text),
1282 (0..37, End(MarkdownTagEnd::CodeBlock)),
1283 (0..37, RootEnd(0)),
1284 ],
1285 language_names: {
1286 let mut h = HashSet::default();
1287 h.insert("rust".into());
1288 h
1289 },
1290 root_block_starts: vec![0],
1291 ..Default::default()
1292 }
1293 );
1294 assert_eq!(
1295 parse_markdown_with_options(" fn main() {}", false, false, false),
1296 ParsedMarkdownData {
1297 events: vec![
1298 (4..16, RootStart),
1299 (
1300 4..16,
1301 Start(CodeBlock {
1302 kind: CodeBlockKind::Indented,
1303 metadata: CodeBlockMetadata {
1304 content_range: 4..16,
1305 line_count: 1,
1306 is_fenced_closed: false,
1307 }
1308 })
1309 ),
1310 (4..16, Text),
1311 (4..16, End(MarkdownTagEnd::CodeBlock)),
1312 (4..16, RootEnd(0)),
1313 ],
1314 root_block_starts: vec![4],
1315 ..Default::default()
1316 }
1317 );
1318 }
1319
1320 fn assert_code_block_does_not_emit_links(markdown: &str) {
1321 let parsed = parse_markdown_with_options(markdown, false, false, false);
1322 let mut code_block_depth = 0;
1323 let mut code_block_count = 0;
1324 let mut saw_text_inside_code_block = false;
1325
1326 for (_, event) in &parsed.events {
1327 match event {
1328 Start(CodeBlock { .. }) => {
1329 code_block_depth += 1;
1330 code_block_count += 1;
1331 }
1332 End(MarkdownTagEnd::CodeBlock) => {
1333 assert!(
1334 code_block_depth > 0,
1335 "encountered a code block end without a matching start"
1336 );
1337 code_block_depth -= 1;
1338 }
1339 Start(Link { .. }) | End(MarkdownTagEnd::Link) => {
1340 assert_eq!(
1341 code_block_depth, 0,
1342 "code blocks should not emit link events"
1343 );
1344 }
1345 Text | SubstitutedText(_) if code_block_depth > 0 => {
1346 saw_text_inside_code_block = true;
1347 }
1348 _ => {}
1349 }
1350 }
1351
1352 assert_eq!(code_block_count, 1, "expected exactly one code block");
1353 assert_eq!(code_block_depth, 0, "unterminated code block");
1354 assert!(
1355 saw_text_inside_code_block,
1356 "expected text inside the code block"
1357 );
1358 }
1359
1360 #[test]
1361 fn test_code_blocks_do_not_autolink_urls() {
1362 assert_code_block_does_not_emit_links("```txt\nhttps://example.com\n```");
1363 assert_code_block_does_not_emit_links(" https://example.com");
1364 assert_code_block_does_not_emit_links(
1365 "```txt\r\nhttps:/\\/example.com\r\nhttps://example.com\r\n```",
1366 );
1367 assert_code_block_does_not_emit_links(
1368 " https:/\\/example.com\r\n https://example.com",
1369 );
1370 }
1371
1372 #[test]
1373 fn test_metadata_blocks_are_root_blocks() {
1374 assert_eq!(
1375 parse_markdown_with_options(
1376 "+++\ntitle = \"Example\"\n+++\n\nParagraph",
1377 false,
1378 false,
1379 true
1380 ),
1381 ParsedMarkdownData {
1382 events: vec![
1383 (0..25, RootStart),
1384 (0..25, Start(MetadataBlock(MetadataBlockKind::PlusesStyle))),
1385 (4..22, Text),
1386 (
1387 0..25,
1388 End(MarkdownTagEnd::MetadataBlock(
1389 MetadataBlockKind::PlusesStyle
1390 ))
1391 ),
1392 (0..25, RootEnd(0)),
1393 (27..36, RootStart),
1394 (27..36, Start(Paragraph)),
1395 (27..36, Text),
1396 (27..36, End(MarkdownTagEnd::Paragraph)),
1397 (27..36, RootEnd(1)),
1398 ],
1399 root_block_starts: vec![0, 27],
1400 metadata_blocks: BTreeMap::from_iter([(
1401 0,
1402 ParsedMetadataBlock {
1403 content_range: 4..22,
1404 rows: None,
1405 },
1406 )]),
1407 ..Default::default()
1408 }
1409 );
1410 }
1411
1412 #[test]
1413 fn test_metadata_blocks_are_omitted_by_default() {
1414 assert_eq!(
1415 parse_markdown_with_options(
1416 "+++\ntitle = \"Example\"\n+++\n\nParagraph",
1417 false,
1418 false,
1419 false
1420 ),
1421 ParsedMarkdownData {
1422 events: vec![
1423 (27..36, RootStart),
1424 (27..36, Start(Paragraph)),
1425 (27..36, Text),
1426 (27..36, End(MarkdownTagEnd::Paragraph)),
1427 (27..36, RootEnd(0)),
1428 ],
1429 root_block_starts: vec![27],
1430 ..Default::default()
1431 }
1432 );
1433 }
1434
1435 #[test]
1436 fn test_table_checkboxes_remain_text_in_cells() {
1437 let markdown = "\
1438| Done | Task |
1439|------|---------|
1440| [x] | Fix bug |
1441| [ ] | Add feature |";
1442 let parsed = parse_markdown_with_options(markdown, false, false, false);
1443
1444 let mut in_table = false;
1445 let mut saw_task_list_marker = false;
1446 let mut cell_texts = Vec::new();
1447 let mut current_cell = String::new();
1448
1449 for (range, event) in &parsed.events {
1450 match event {
1451 Start(Table(_)) => in_table = true,
1452 End(MarkdownTagEnd::Table) => in_table = false,
1453 Start(TableCell) => current_cell.clear(),
1454 End(MarkdownTagEnd::TableCell) => {
1455 if in_table {
1456 cell_texts.push(current_cell.clone());
1457 }
1458 }
1459 Text if in_table => current_cell.push_str(&markdown[range.clone()]),
1460 TaskListMarker(_) if in_table => saw_task_list_marker = true,
1461 _ => {}
1462 }
1463 }
1464
1465 let checkbox_cells: Vec<&str> = cell_texts
1466 .iter()
1467 .map(|cell| cell.trim())
1468 .filter(|cell| *cell == "[x]" || *cell == "[X]" || *cell == "[ ]")
1469 .collect();
1470
1471 assert!(
1472 !saw_task_list_marker,
1473 "Table checkboxes should remain text, not task-list markers"
1474 );
1475 assert_eq!(checkbox_cells, vec!["[x]", "[ ]"]);
1476 }
1477
1478 #[test]
1479 fn test_extract_code_content_range() {
1480 let input = "```let x = 5;```";
1481 assert_eq!(extract_code_content_range(input), 3..13);
1482
1483 let input = "``let x = 5;``";
1484 assert_eq!(extract_code_content_range(input), 2..12);
1485
1486 let input = "`let x = 5;`";
1487 assert_eq!(extract_code_content_range(input), 1..11);
1488
1489 let input = "plain text";
1490 assert_eq!(extract_code_content_range(input), 0..10);
1491
1492 let input = "``let x = 5;`";
1493 assert_eq!(extract_code_content_range(input), 0..13);
1494 }
1495
1496 #[test]
1497 fn test_inline_code_substitutes_escaped_pipes() {
1498 let markdown = r"| Pattern |
1499| --- |
1500| `a\|b` |";
1501 let parsed = parse_markdown_with_options(markdown, false, false, false);
1502 let code_range = {
1503 let start = markdown.find(r"a\|b").expect("inline code source");
1504 start..start + r"a\|b".len()
1505 };
1506
1507 assert!(
1508 parsed
1509 .events
1510 .iter()
1511 .any(|(range, event)| range == &code_range
1512 && event == &SubstitutedCode("a|b".into())),
1513 "expected escaped pipe in table inline code to render as decoded inline code: {:?}",
1514 parsed.events
1515 );
1516 }
1517
1518 #[test]
1519 fn test_inline_code_keeps_escaped_pipes_outside_tables() {
1520 let markdown = r"`a\|b`";
1521 let parsed = parse_markdown_with_options(markdown, false, false, false);
1522
1523 assert!(
1524 parsed
1525 .events
1526 .iter()
1527 .any(|(range, event)| range == &(1..5) && event == &Code),
1528 "expected escaped pipe outside a table to remain normal inline code: {:?}",
1529 parsed.events
1530 );
1531 }
1532
1533 #[test]
1534 fn test_extract_code_block_content_range() {
1535 let input = "```rust\nlet x = 5;\n```";
1536 assert_eq!(extract_code_block_content_range(input), 8..19);
1537
1538 let input = "plain text";
1539 assert_eq!(extract_code_block_content_range(input), 0..10);
1540
1541 let input = "```python\nprint('hello')\nprint('world')\n```";
1542 assert_eq!(extract_code_block_content_range(input), 10..40);
1543
1544 // Malformed input
1545 let input = "`````";
1546 assert_eq!(extract_code_block_content_range(input), 3..3);
1547 }
1548
1549 #[test]
1550 fn test_footnotes() {
1551 let parsed = parse_markdown_with_options(
1552 "Text with a footnote[^1] and some more text.\n\n[^1]: This is the footnote content.",
1553 false,
1554 false,
1555 false,
1556 );
1557 assert_eq!(
1558 parsed.events,
1559 vec![
1560 (0..45, RootStart),
1561 (0..45, Start(Paragraph)),
1562 (0..20, Text),
1563 (20..24, FootnoteReference("1".into())),
1564 (24..44, Text),
1565 (0..45, End(MarkdownTagEnd::Paragraph)),
1566 (0..45, RootEnd(0)),
1567 (46..81, RootStart),
1568 (46..81, Start(FootnoteDefinition("1".into()))),
1569 (52..81, Start(Paragraph)),
1570 (52..81, Text),
1571 (52..81, End(MarkdownTagEnd::Paragraph)),
1572 (46..81, End(MarkdownTagEnd::FootnoteDefinition)),
1573 (46..81, RootEnd(1)),
1574 ]
1575 );
1576 assert_eq!(parsed.footnote_definitions.len(), 1);
1577 assert_eq!(parsed.footnote_definitions.get("1").copied(), Some(52));
1578 }
1579
1580 #[test]
1581 fn test_footnote_definitions_multiple() {
1582 let parsed = parse_markdown_with_options(
1583 "Text[^a] and[^b].\n\n[^a]: First.\n\n[^b]: Second.",
1584 false,
1585 false,
1586 false,
1587 );
1588 assert_eq!(parsed.footnote_definitions.len(), 2);
1589 assert!(parsed.footnote_definitions.contains_key("a"));
1590 assert!(parsed.footnote_definitions.contains_key("b"));
1591 }
1592
1593 #[test]
1594 fn test_links_split_across_fragments() {
1595 // This test verifies that links split across multiple text fragments due to escaping or other issues
1596 // are correctly detected and processed
1597 // Note: In real usage, pulldown_cmark creates separate text events for the escaped character
1598 // We're verifying our parser can handle this correctly
1599 assert_eq!(
1600 parse_markdown_with_options(
1601 "https:/\\/example.com is equivalent to https://example.com!",
1602 false,
1603 false,
1604 false,
1605 )
1606 .events,
1607 vec![
1608 (0..62, RootStart),
1609 (0..62, Start(Paragraph)),
1610 (
1611 0..20,
1612 Start(Link {
1613 link_type: LinkType::Autolink,
1614 dest_url: "https://example.com".into(),
1615 title: "".into(),
1616 id: "".into()
1617 })
1618 ),
1619 (0..7, Text),
1620 (8..20, Text),
1621 (0..20, End(MarkdownTagEnd::Link)),
1622 (20..38, Text),
1623 (
1624 38..61,
1625 Start(Link {
1626 link_type: LinkType::Autolink,
1627 dest_url: "https://example.com".into(),
1628 title: "".into(),
1629 id: "".into()
1630 })
1631 ),
1632 (38..53, Text),
1633 (53..58, SubstitutedText(".".into())),
1634 (58..61, Text),
1635 (38..61, End(MarkdownTagEnd::Link)),
1636 (61..62, Text),
1637 (0..62, End(MarkdownTagEnd::Paragraph)),
1638 (0..62, RootEnd(0)),
1639 ],
1640 );
1641
1642 assert_eq!(
1643 parse_markdown_with_options(
1644 "Visit https://example.com/cat\\/é‍☕ for coffee!",
1645 false,
1646 false,
1647 false,
1648 )
1649 .events,
1650 [
1651 (0..55, RootStart),
1652 (0..55, Start(Paragraph)),
1653 (0..6, Text),
1654 (
1655 6..43,
1656 Start(Link {
1657 link_type: LinkType::Autolink,
1658 dest_url: "https://example.com/cat/é\u{200d}☕".into(),
1659 title: "".into(),
1660 id: "".into()
1661 })
1662 ),
1663 (6..29, Text),
1664 (30..33, Text),
1665 (33..40, SubstitutedText("\u{200d}".into())),
1666 (40..43, Text),
1667 (6..43, End(MarkdownTagEnd::Link)),
1668 (43..55, Text),
1669 (0..55, End(MarkdownTagEnd::Paragraph)),
1670 (0..55, RootEnd(0)),
1671 ]
1672 );
1673 }
1674
1675 #[test]
1676 fn test_heading_slugs() {
1677 let parsed = parse_markdown_with_options(
1678 "# Hello World\n\n## Code `block`\n\n### Third Level\n\n#### Fourth Level\n\n## Hello World",
1679 false,
1680 true,
1681 false,
1682 );
1683 assert_eq!(parsed.heading_slugs.len(), 5);
1684 assert!(parsed.heading_slugs.contains_key("hello-world"));
1685 assert!(parsed.heading_slugs.contains_key("code-block"));
1686 assert!(parsed.heading_slugs.contains_key("third-level"));
1687 assert!(parsed.heading_slugs.contains_key("fourth-level"));
1688 assert!(parsed.heading_slugs.contains_key("hello-world-1"));
1689 }
1690
1691 #[test]
1692 fn test_heading_source_index_for_slug() {
1693 let parsed = parse_markdown_with_options(
1694 "# Duplicate\n\nText\n\n## Duplicate\n\nMore text",
1695 false,
1696 true,
1697 false,
1698 );
1699 let first = parsed.heading_slugs.get("duplicate").copied();
1700 let second = parsed.heading_slugs.get("duplicate-1").copied();
1701 assert!(first.is_some());
1702 assert!(second.is_some());
1703 assert!(first.expect("first slug missing") < second.expect("second slug missing"));
1704 }
1705
1706 #[test]
1707 fn test_heading_slug_collision_with_dedup_suffix() {
1708 let parsed = parse_markdown_with_options("# Foo\n\n## Foo\n\n## Foo 1", false, true, false);
1709 assert_eq!(parsed.heading_slugs.len(), 3);
1710 assert!(parsed.heading_slugs.contains_key("foo"));
1711 assert!(parsed.heading_slugs.contains_key("foo-1"));
1712 assert!(parsed.heading_slugs.contains_key("foo-1-1"));
1713 }
1714
1715 #[test]
1716 fn test_gfm_alert_block_quote_kinds() {
1717 use pulldown_cmark::BlockQuoteKind;
1718
1719 let markdown = "\n> [!NOTE]\n> A note.\n\n> [!TIP]\n> A tip.\n\n> [!IMPORTANT]\n> Important.\n\n> [!WARNING]\n> A warning.\n\n> [!CAUTION]\n> A caution.\n\n> Plain quote.\n";
1720 let parsed = parse_markdown_with_options(markdown, false, false, false);
1721
1722 let block_quote_kinds: Vec<_> = parsed
1723 .events
1724 .iter()
1725 .filter_map(|(_, event)| match event {
1726 Start(BlockQuote(kind)) => Some(*kind),
1727 _ => None,
1728 })
1729 .collect();
1730
1731 assert_eq!(
1732 block_quote_kinds,
1733 vec![
1734 Some(BlockQuoteKind::Note),
1735 Some(BlockQuoteKind::Tip),
1736 Some(BlockQuoteKind::Important),
1737 Some(BlockQuoteKind::Warning),
1738 Some(BlockQuoteKind::Caution),
1739 None,
1740 ]
1741 );
1742 }
1743
1744 #[test]
1745 fn test_br_tag_emits_hard_break() {
1746 for input in [
1747 "hello<br>world",
1748 "hello<br/>world",
1749 "hello<br />world",
1750 "hello<br >world",
1751 "hello<BR>world",
1752 "hello<br class=\"x\">world",
1753 "hello<br class=\"x\"/>world",
1754 ] {
1755 let parsed = parse_markdown_with_options(input, true, false, false);
1756 let has_hard_break = parsed
1757 .events
1758 .iter()
1759 .any(|(_, event)| matches!(event, MarkdownEvent::HardBreak));
1760 let has_empty_substituted_text = parsed.events.iter().any(|(_, event)| {
1761 matches!(event, MarkdownEvent::SubstitutedText(text) if text.is_empty())
1762 });
1763 assert!(has_hard_break, "<br> in \"{input}\" should emit HardBreak");
1764 assert!(
1765 !has_empty_substituted_text,
1766 "<br> in \"{input}\" should not produce empty SubstitutedText"
1767 );
1768 }
1769 }
1770
1771 #[test]
1772 fn test_br_tag_not_a_hard_break_without_parse_html() {
1773 for input in ["hello<br>world", "hello<br/>world", "hello<br />world"] {
1774 let parsed = parse_markdown_with_options(input, false, false, false);
1775 let has_hard_break = parsed
1776 .events
1777 .iter()
1778 .any(|(_, event)| matches!(event, MarkdownEvent::HardBreak));
1779 let has_inline_html = parsed
1780 .events
1781 .iter()
1782 .any(|(_, event)| matches!(event, MarkdownEvent::InlineHtml));
1783 assert!(
1784 !has_hard_break,
1785 "<br> in \"{input}\" should not emit HardBreak when parse_html is disabled"
1786 );
1787 assert!(
1788 has_inline_html,
1789 "<br> in \"{input}\" should be preserved as InlineHtml when parse_html is disabled"
1790 );
1791 }
1792 }
1793
1794 #[test]
1795 fn test_br_prefixed_tag_is_not_a_hard_break() {
1796 for input in ["a<break>b", "a<brick>b", "a<b>bold</b>c"] {
1797 let parsed = parse_markdown_with_options(input, true, false, false);
1798 let has_hard_break = parsed
1799 .events
1800 .iter()
1801 .any(|(_, event)| matches!(event, MarkdownEvent::HardBreak));
1802 assert!(
1803 !has_hard_break,
1804 "\"{input}\" should not be treated as a <br> hard break"
1805 );
1806 }
1807 }
1808
1809 #[test]
1810 fn test_unrecognized_inline_html_preserved_as_inline_html() {
1811 for input in ["a<span>b</span>c", "a<em>b</em>c", "a<strong>b</strong>c"] {
1812 let parsed = parse_markdown_with_options(input, false, false, false);
1813 let has_inline_html = parsed
1814 .events
1815 .iter()
1816 .any(|(_, event)| matches!(event, MarkdownEvent::InlineHtml));
1817 let has_hard_break = parsed
1818 .events
1819 .iter()
1820 .any(|(_, event)| matches!(event, MarkdownEvent::HardBreak));
1821 assert!(
1822 has_inline_html,
1823 "unrecognized inline HTML \"{input}\" should emit InlineHtml"
1824 );
1825 assert!(
1826 !has_hard_break,
1827 "unrecognized inline HTML \"{input}\" should not emit HardBreak"
1828 );
1829 }
1830 }
1831}
1832