Skip to repository content818 lines · 32.5 KB · rust
tenant.openagents/omega
No repository description is available.
OpenAgents Git authority 2026-07-28T02:32:52.264Z Public web read
NIP-34 coordinate
30617:7649603503856e5148d571eac2766b288a8ff1e9e35d380337a1d2b0015b4f92:omegaMaintainersHidden in public view
References2 branches · 1 tag
Read-only clone
git clone https://openagents.com/git/tenant.openagents/omega.gitBrowse files
rewrap.rs
1use super::*;
2
3impl Editor {
4 pub fn rewrap(&mut self, options: RewrapOptions, cx: &mut Context<Self>) {
5 if self.read_only(cx) || self.mode.is_single_line() {
6 return;
7 }
8 let buffer = self.buffer.read(cx).snapshot(cx);
9 let selections = self.selections.all::<Point>(&self.display_snapshot(cx));
10
11 #[derive(Clone, Debug, PartialEq)]
12 enum CommentFormat {
13 /// single line comment, with prefix for line
14 Line(String),
15 /// single line within a block comment, with prefix for line
16 BlockLine(String),
17 /// a single line of a block comment that includes the initial delimiter
18 BlockCommentWithStart(BlockCommentConfig),
19 /// a single line of a block comment that includes the ending delimiter
20 BlockCommentWithEnd(BlockCommentConfig),
21 }
22
23 // Split selections to respect paragraph, indent, and comment prefix boundaries.
24 let wrap_ranges = selections.into_iter().flat_map(|selection| {
25 let language_settings = buffer.language_settings_at(selection.head(), cx);
26 let language_scope = buffer.language_scope_at(selection.head());
27
28 let indent_and_prefix_for_row =
29 |row: u32| -> (IndentSize, Option<CommentFormat>, Option<String>) {
30 let indent = buffer.indent_size_for_line(MultiBufferRow(row));
31 let (comment_prefix, rewrap_prefix) = if let Some(language_scope) =
32 &language_scope
33 {
34 let indent_end = Point::new(row, indent.len);
35 let line_end = Point::new(row, buffer.line_len(MultiBufferRow(row)));
36 let line_text_after_indent = buffer
37 .text_for_range(indent_end..line_end)
38 .collect::<String>();
39
40 let is_within_comment_override = buffer
41 .language_scope_at(indent_end)
42 .is_some_and(|scope| scope.override_name() == Some("comment"));
43 let comment_delimiters = if is_within_comment_override {
44 // we are within a comment syntax node, but we don't
45 // yet know what kind of comment: block, doc or line
46 match (
47 language_scope.documentation_comment(),
48 language_scope.block_comment(),
49 ) {
50 (Some(config), _) | (_, Some(config))
51 if buffer.contains_str_at(indent_end, &config.start) =>
52 {
53 Some(CommentFormat::BlockCommentWithStart(config.clone()))
54 }
55 (Some(config), _) | (_, Some(config))
56 if line_text_after_indent.ends_with(config.end.as_ref()) =>
57 {
58 Some(CommentFormat::BlockCommentWithEnd(config.clone()))
59 }
60 (Some(config), _) | (_, Some(config))
61 if !config.prefix.is_empty()
62 && buffer.contains_str_at(indent_end, &config.prefix) =>
63 {
64 Some(CommentFormat::BlockLine(config.prefix.to_string()))
65 }
66 (_, _) => language_scope
67 .line_comment_prefixes()
68 .iter()
69 .find(|prefix| buffer.contains_str_at(indent_end, prefix))
70 .map(|prefix| CommentFormat::Line(prefix.to_string())),
71 }
72 } else {
73 // we not in an overridden comment node, but we may
74 // be within a non-overridden line comment node
75 language_scope
76 .line_comment_prefixes()
77 .iter()
78 .find(|prefix| buffer.contains_str_at(indent_end, prefix))
79 .map(|prefix| CommentFormat::Line(prefix.to_string()))
80 };
81
82 let rewrap_prefix = language_scope
83 .rewrap_prefixes()
84 .iter()
85 .find_map(|prefix_regex| {
86 prefix_regex.find(&line_text_after_indent).map(|mat| {
87 if mat.start() == 0 {
88 Some(mat.as_str().to_string())
89 } else {
90 None
91 }
92 })
93 })
94 .flatten();
95 (comment_delimiters, rewrap_prefix)
96 } else {
97 (None, None)
98 };
99 (indent, comment_prefix, rewrap_prefix)
100 };
101
102 let mut start_row = selection.start.row;
103 let mut end_row = selection.end.row;
104
105 if selection.is_empty() {
106 let cursor_row = selection.start.row;
107
108 let (mut indent_size, comment_prefix, _) = indent_and_prefix_for_row(cursor_row);
109 let line_prefix = match &comment_prefix {
110 Some(CommentFormat::Line(prefix) | CommentFormat::BlockLine(prefix)) => {
111 Some(prefix.as_str())
112 }
113 Some(CommentFormat::BlockCommentWithEnd(BlockCommentConfig {
114 prefix, ..
115 })) => Some(prefix.as_ref()),
116 Some(CommentFormat::BlockCommentWithStart(BlockCommentConfig {
117 start: _,
118 end: _,
119 prefix,
120 tab_size,
121 })) => {
122 indent_size.len += tab_size;
123 Some(prefix.as_ref())
124 }
125 None => None,
126 };
127 let indent_prefix = indent_size.chars().collect::<String>();
128 let line_prefix = format!("{indent_prefix}{}", line_prefix.unwrap_or(""));
129
130 'expand_upwards: while start_row > 0 {
131 let prev_row = start_row - 1;
132 if buffer.contains_str_at(Point::new(prev_row, 0), &line_prefix)
133 && buffer.line_len(MultiBufferRow(prev_row)) as usize > line_prefix.len()
134 && !buffer.is_line_blank(MultiBufferRow(prev_row))
135 {
136 start_row = prev_row;
137 } else {
138 break 'expand_upwards;
139 }
140 }
141
142 'expand_downwards: while end_row < buffer.max_point().row {
143 let next_row = end_row + 1;
144 if buffer.contains_str_at(Point::new(next_row, 0), &line_prefix)
145 && buffer.line_len(MultiBufferRow(next_row)) as usize > line_prefix.len()
146 && !buffer.is_line_blank(MultiBufferRow(next_row))
147 {
148 end_row = next_row;
149 } else {
150 break 'expand_downwards;
151 }
152 }
153 }
154
155 let mut non_blank_rows_iter = (start_row..=end_row)
156 .filter(|row| !buffer.is_line_blank(MultiBufferRow(*row)))
157 .peekable();
158
159 let first_row = if let Some(&row) = non_blank_rows_iter.peek() {
160 row
161 } else {
162 return Vec::new();
163 };
164
165 let mut ranges = Vec::new();
166
167 let mut current_range_start = first_row;
168 let mut prev_row = first_row;
169 let (
170 mut current_range_indent,
171 mut current_range_comment_delimiters,
172 mut current_range_rewrap_prefix,
173 ) = indent_and_prefix_for_row(first_row);
174
175 for row in non_blank_rows_iter.skip(1) {
176 let has_paragraph_break = row > prev_row + 1;
177
178 let (row_indent, row_comment_delimiters, row_rewrap_prefix) =
179 indent_and_prefix_for_row(row);
180
181 let has_indent_change = row_indent != current_range_indent;
182 let has_comment_change = row_comment_delimiters != current_range_comment_delimiters;
183
184 let has_boundary_change = has_comment_change
185 || row_rewrap_prefix.is_some()
186 || (has_indent_change && current_range_comment_delimiters.is_some());
187
188 if has_paragraph_break || has_boundary_change {
189 ranges.push((
190 language_settings.clone(),
191 Point::new(current_range_start, 0)
192 ..Point::new(prev_row, buffer.line_len(MultiBufferRow(prev_row))),
193 current_range_indent,
194 current_range_comment_delimiters.clone(),
195 current_range_rewrap_prefix.clone(),
196 ));
197 current_range_start = row;
198 current_range_indent = row_indent;
199 current_range_comment_delimiters = row_comment_delimiters;
200 current_range_rewrap_prefix = row_rewrap_prefix;
201 }
202 prev_row = row;
203 }
204
205 ranges.push((
206 language_settings.clone(),
207 Point::new(current_range_start, 0)
208 ..Point::new(prev_row, buffer.line_len(MultiBufferRow(prev_row))),
209 current_range_indent,
210 current_range_comment_delimiters,
211 current_range_rewrap_prefix,
212 ));
213
214 ranges
215 });
216
217 let mut edits = Vec::new();
218 let mut rewrapped_row_ranges = Vec::<RangeInclusive<u32>>::new();
219
220 for (language_settings, wrap_range, mut indent_size, comment_prefix, rewrap_prefix) in
221 wrap_ranges
222 {
223 let start_row = wrap_range.start.row;
224 let end_row = wrap_range.end.row;
225
226 // Skip selections that overlap with a range that has already been rewrapped.
227 let selection_range = start_row..end_row;
228 if rewrapped_row_ranges
229 .iter()
230 .any(|range| range.overlaps(&selection_range))
231 {
232 continue;
233 }
234
235 let tab_size = language_settings.tab_size;
236
237 let (line_prefix, inside_comment) = match &comment_prefix {
238 Some(CommentFormat::Line(prefix) | CommentFormat::BlockLine(prefix)) => {
239 (Some(prefix.as_str()), true)
240 }
241 Some(CommentFormat::BlockCommentWithEnd(BlockCommentConfig { prefix, .. })) => {
242 (Some(prefix.as_ref()), true)
243 }
244 Some(CommentFormat::BlockCommentWithStart(BlockCommentConfig {
245 start: _,
246 end: _,
247 prefix,
248 tab_size,
249 })) => {
250 indent_size.len += tab_size;
251 (Some(prefix.as_ref()), true)
252 }
253 None => (None, false),
254 };
255 let indent_prefix = indent_size.chars().collect::<String>();
256 let line_prefix = format!("{indent_prefix}{}", line_prefix.unwrap_or(""));
257
258 let allow_rewrap_based_on_language = match language_settings.allow_rewrap {
259 RewrapBehavior::InComments => inside_comment,
260 RewrapBehavior::InSelections => !wrap_range.is_empty(),
261 RewrapBehavior::Anywhere => true,
262 };
263
264 let should_rewrap = options.override_language_settings
265 || allow_rewrap_based_on_language
266 || self.hard_wrap.is_some();
267 if !should_rewrap {
268 continue;
269 }
270
271 let start = Point::new(start_row, 0);
272 let start_offset = ToOffset::to_offset(&start, &buffer);
273 let end = Point::new(end_row, buffer.line_len(MultiBufferRow(end_row)));
274 let selection_text = buffer.text_for_range(start..end).collect::<String>();
275 let mut first_line_delimiter = None;
276 let mut last_line_delimiter = None;
277 let Some(lines_without_prefixes) = selection_text
278 .lines()
279 .enumerate()
280 .map(|(ix, line)| {
281 let line_trimmed = line.trim_start();
282 if rewrap_prefix.is_some() && ix > 0 {
283 Ok(line_trimmed)
284 } else if let Some(
285 CommentFormat::BlockCommentWithStart(BlockCommentConfig {
286 start,
287 prefix,
288 end,
289 tab_size,
290 })
291 | CommentFormat::BlockCommentWithEnd(BlockCommentConfig {
292 start,
293 prefix,
294 end,
295 tab_size,
296 }),
297 ) = &comment_prefix
298 {
299 let line_trimmed = line_trimmed
300 .strip_prefix(start.as_ref())
301 .map(|s| {
302 let mut indent_size = indent_size;
303 indent_size.len -= tab_size;
304 let indent_prefix: String = indent_size.chars().collect();
305 first_line_delimiter = Some((indent_prefix, start));
306 s.trim_start()
307 })
308 .unwrap_or(line_trimmed);
309 let line_trimmed = line_trimmed
310 .strip_suffix(end.as_ref())
311 .map(|s| {
312 last_line_delimiter = Some(end);
313 s.trim_end()
314 })
315 .unwrap_or(line_trimmed);
316 let line_trimmed = line_trimmed
317 .strip_prefix(prefix.as_ref())
318 .unwrap_or(line_trimmed);
319 Ok(line_trimmed)
320 } else if let Some(CommentFormat::BlockLine(prefix)) = &comment_prefix {
321 line_trimmed.strip_prefix(prefix).with_context(|| {
322 format!("line did not start with prefix {prefix:?}: {line:?}")
323 })
324 } else {
325 line_trimmed
326 .strip_prefix(&line_prefix.trim_start())
327 .with_context(|| {
328 format!("line did not start with prefix {line_prefix:?}: {line:?}")
329 })
330 }
331 })
332 .collect::<Result<Vec<_>, _>>()
333 .log_err()
334 else {
335 continue;
336 };
337
338 let wrap_column = options.line_length.or(self.hard_wrap).unwrap_or_else(|| {
339 buffer
340 .language_settings_at(Point::new(start_row, 0), cx)
341 .preferred_line_length as usize
342 });
343
344 let subsequent_lines_prefix = if let Some(rewrap_prefix_str) = &rewrap_prefix {
345 format!("{}{}", indent_prefix, " ".repeat(rewrap_prefix_str.len()))
346 } else {
347 line_prefix.clone()
348 };
349
350 let wrapped_text = {
351 let mut wrapped_text = wrap_with_prefix(
352 line_prefix,
353 subsequent_lines_prefix,
354 lines_without_prefixes.join("\n"),
355 wrap_column,
356 tab_size,
357 options.preserve_existing_whitespace,
358 );
359
360 if let Some((indent, delimiter)) = first_line_delimiter {
361 wrapped_text = format!("{indent}{delimiter}\n{wrapped_text}");
362 }
363 if let Some(last_line) = last_line_delimiter {
364 wrapped_text = format!("{wrapped_text}\n{indent_prefix}{last_line}");
365 }
366
367 wrapped_text
368 };
369
370 // TODO: should always use char-based diff while still supporting cursor behavior that
371 // matches vim.
372 let mut diff_options = DiffOptions::default();
373 if options.override_language_settings {
374 diff_options.max_word_diff_len = 0;
375 diff_options.max_word_diff_line_count = 0;
376 } else {
377 diff_options.max_word_diff_len = usize::MAX;
378 diff_options.max_word_diff_line_count = usize::MAX;
379 }
380
381 for (old_range, new_text) in
382 text_diff_with_options(&selection_text, &wrapped_text, diff_options)
383 {
384 let edit_start = buffer.anchor_after(start_offset + old_range.start);
385 let edit_end = buffer.anchor_after(start_offset + old_range.end);
386 edits.push((edit_start..edit_end, new_text));
387 }
388
389 rewrapped_row_ranges.push(start_row..=end_row);
390 }
391
392 self.buffer
393 .update(cx, |buffer, cx| buffer.edit(edits, None, cx));
394 }
395}
396
397fn char_len_with_expanded_tabs(offset: usize, text: &str, tab_size: NonZeroU32) -> usize {
398 let tab_size = tab_size.get() as usize;
399 let mut width = offset;
400
401 for ch in text.chars() {
402 width += if ch == '\t' {
403 tab_size - (width % tab_size)
404 } else {
405 1
406 };
407 }
408
409 width - offset
410}
411
412/// Tokenizes a string into runs of text that should stick together, or that is whitespace.
413struct WordBreakingTokenizer<'a> {
414 input: &'a str,
415}
416
417impl<'a> WordBreakingTokenizer<'a> {
418 fn new(input: &'a str) -> Self {
419 Self { input }
420 }
421}
422
423fn is_char_ideographic(ch: char) -> bool {
424 use unicode_script::Script::*;
425 use unicode_script::UnicodeScript;
426 matches!(ch.script(), Han | Tangut | Yi)
427}
428
429fn is_grapheme_ideographic(text: &str) -> bool {
430 text.chars().any(is_char_ideographic)
431}
432
433fn is_non_breaking_whitespace(character: char) -> bool {
434 matches!(character, '\u{00A0}' | '\u{2007}' | '\u{202F}')
435}
436
437fn is_grapheme_whitespace(text: &str) -> bool {
438 text.chars()
439 .any(|character| character.is_whitespace() && !is_non_breaking_whitespace(character))
440}
441
442fn should_stay_with_preceding_ideograph(text: &str) -> bool {
443 text.chars()
444 .next()
445 .is_some_and(|ch| matches!(ch, '。' | '、' | '，' | '？' | '！' | '：' | '；' | '…'))
446}
447
448#[derive(PartialEq, Eq, Debug, Clone, Copy)]
449enum WordBreakToken<'a> {
450 Word { token: &'a str, grapheme_len: usize },
451 InlineWhitespace { token: &'a str, grapheme_len: usize },
452 Newline,
453}
454
455impl<'a> Iterator for WordBreakingTokenizer<'a> {
456 /// Yields a span, the count of graphemes in the token, and whether it was
457 /// whitespace. Note that it also breaks at word boundaries.
458 type Item = WordBreakToken<'a>;
459
460 fn next(&mut self) -> Option<Self::Item> {
461 use unicode_segmentation::UnicodeSegmentation;
462 if self.input.is_empty() {
463 return None;
464 }
465
466 let mut iter = self.input.graphemes(true).peekable();
467 let mut offset = 0;
468 let mut grapheme_len = 0;
469 if let Some(first_grapheme) = iter.next() {
470 let is_newline = first_grapheme == "\n";
471 let is_whitespace = is_grapheme_whitespace(first_grapheme);
472 offset += first_grapheme.len();
473 grapheme_len += 1;
474 if is_grapheme_ideographic(first_grapheme) && !is_whitespace {
475 if let Some(grapheme) = iter.peek().copied()
476 && should_stay_with_preceding_ideograph(grapheme)
477 {
478 offset += grapheme.len();
479 grapheme_len += 1;
480 }
481 } else {
482 let mut words = self.input[offset..].split_word_bound_indices().peekable();
483 let mut next_word_bound = words.peek().copied();
484 if next_word_bound.is_some_and(|(i, _)| i == 0) {
485 next_word_bound = words.next();
486 }
487 while let Some(grapheme) = iter.peek().copied() {
488 if next_word_bound.is_some_and(|(i, _)| i == offset) {
489 break;
490 };
491 if is_grapheme_whitespace(grapheme) != is_whitespace
492 || (grapheme == "\n") != is_newline
493 {
494 break;
495 };
496 offset += grapheme.len();
497 grapheme_len += 1;
498 iter.next();
499 }
500 }
501 let token = &self.input[..offset];
502 self.input = &self.input[offset..];
503 if token == "\n" {
504 Some(WordBreakToken::Newline)
505 } else if is_whitespace {
506 Some(WordBreakToken::InlineWhitespace {
507 token,
508 grapheme_len,
509 })
510 } else {
511 Some(WordBreakToken::Word {
512 token,
513 grapheme_len,
514 })
515 }
516 } else {
517 None
518 }
519 }
520}
521
522fn wrap_with_prefix(
523 first_line_prefix: String,
524 subsequent_lines_prefix: String,
525 unwrapped_text: String,
526 wrap_column: usize,
527 tab_size: NonZeroU32,
528 preserve_existing_whitespace: bool,
529) -> String {
530 let first_line_prefix_len = char_len_with_expanded_tabs(0, &first_line_prefix, tab_size);
531 let subsequent_lines_prefix_len =
532 char_len_with_expanded_tabs(0, &subsequent_lines_prefix, tab_size);
533 let mut wrapped_text = String::new();
534 let mut current_line = first_line_prefix;
535 let mut is_first_line = true;
536
537 let tokenizer = WordBreakingTokenizer::new(&unwrapped_text);
538 let mut current_line_len = first_line_prefix_len;
539 let mut in_whitespace = false;
540 for token in tokenizer {
541 let have_preceding_whitespace = in_whitespace;
542 match token {
543 WordBreakToken::Word {
544 token,
545 grapheme_len,
546 } => {
547 in_whitespace = false;
548 let current_prefix_len = if is_first_line {
549 first_line_prefix_len
550 } else {
551 subsequent_lines_prefix_len
552 };
553 if current_line_len + grapheme_len > wrap_column
554 && current_line_len != current_prefix_len
555 {
556 wrapped_text.push_str(current_line.trim_end());
557 wrapped_text.push('\n');
558 is_first_line = false;
559 current_line = subsequent_lines_prefix.clone();
560 current_line_len = subsequent_lines_prefix_len;
561 }
562 current_line.push_str(token);
563 current_line_len += grapheme_len;
564 }
565 WordBreakToken::InlineWhitespace {
566 mut token,
567 mut grapheme_len,
568 } => {
569 in_whitespace = true;
570 if have_preceding_whitespace && !preserve_existing_whitespace {
571 continue;
572 }
573 if !preserve_existing_whitespace {
574 // Keep a single whitespace grapheme as-is
575 if let Some(first) =
576 unicode_segmentation::UnicodeSegmentation::graphemes(token, true).next()
577 {
578 token = first;
579 } else {
580 token = " ";
581 }
582 grapheme_len = 1;
583 }
584 let current_prefix_len = if is_first_line {
585 first_line_prefix_len
586 } else {
587 subsequent_lines_prefix_len
588 };
589 if current_line_len + grapheme_len > wrap_column {
590 wrapped_text.push_str(current_line.trim_end());
591 wrapped_text.push('\n');
592 is_first_line = false;
593 current_line = subsequent_lines_prefix.clone();
594 current_line_len = subsequent_lines_prefix_len;
595 } else if current_line_len != current_prefix_len || preserve_existing_whitespace {
596 current_line.push_str(token);
597 current_line_len += grapheme_len;
598 }
599 }
600 WordBreakToken::Newline => {
601 in_whitespace = true;
602 let current_prefix_len = if is_first_line {
603 first_line_prefix_len
604 } else {
605 subsequent_lines_prefix_len
606 };
607 if preserve_existing_whitespace {
608 wrapped_text.push_str(current_line.trim_end());
609 wrapped_text.push('\n');
610 is_first_line = false;
611 current_line = subsequent_lines_prefix.clone();
612 current_line_len = subsequent_lines_prefix_len;
613 } else if have_preceding_whitespace {
614 continue;
615 } else if current_line_len + 1 > wrap_column
616 && current_line_len != current_prefix_len
617 {
618 wrapped_text.push_str(current_line.trim_end());
619 wrapped_text.push('\n');
620 is_first_line = false;
621 current_line = subsequent_lines_prefix.clone();
622 current_line_len = subsequent_lines_prefix_len;
623 } else if current_line_len != current_prefix_len {
624 current_line.push(' ');
625 current_line_len += 1;
626 }
627 }
628 }
629 }
630
631 if !current_line.is_empty() {
632 wrapped_text.push_str(¤t_line);
633 }
634 wrapped_text
635}
636
637#[cfg(test)]
638mod tests {
639 use super::*;
640
641 #[test]
642 fn test_string_size_with_expanded_tabs() {
643 let nz = |val| NonZeroU32::new(val).unwrap();
644 assert_eq!(char_len_with_expanded_tabs(0, "", nz(4)), 0);
645 assert_eq!(char_len_with_expanded_tabs(0, "hello", nz(4)), 5);
646 assert_eq!(char_len_with_expanded_tabs(0, "\thello", nz(4)), 9);
647 assert_eq!(char_len_with_expanded_tabs(0, "abc\tab", nz(4)), 6);
648 assert_eq!(char_len_with_expanded_tabs(0, "hello\t", nz(4)), 8);
649 assert_eq!(char_len_with_expanded_tabs(0, "\t\t", nz(8)), 16);
650 assert_eq!(char_len_with_expanded_tabs(0, "x\t", nz(8)), 8);
651 assert_eq!(char_len_with_expanded_tabs(7, "x\t", nz(8)), 9);
652 }
653
654 #[test]
655 fn test_word_breaking_tokenizer() {
656 let tests: &[(&str, &[WordBreakToken<'static>])] = &[
657 ("", &[]),
658 (" ", &[whitespace(" ", 2)]),
659 ("Ʒ", &[word("Ʒ", 1)]),
660 ("Ǽ", &[word("Ǽ", 1)]),
661 ("⋑", &[word("⋑", 1)]),
662 ("⋑⋑", &[word("⋑⋑", 2)]),
663 (
664 "原理，进而",
665 &[word("原", 1), word("理，", 2), word("进", 1), word("而", 1)],
666 ),
667 (
668 "hello world",
669 &[word("hello", 5), whitespace(" ", 1), word("world", 5)],
670 ),
671 (
672 "hello, world",
673 &[word("hello,", 6), whitespace(" ", 1), word("world", 5)],
674 ),
675 (
676 " hello world",
677 &[
678 whitespace(" ", 2),
679 word("hello", 5),
680 whitespace(" ", 1),
681 word("world", 5),
682 ],
683 ),
684 (
685 "这是什么 \n 钢笔",
686 &[
687 word("这", 1),
688 word("是", 1),
689 word("什", 1),
690 word("么", 1),
691 whitespace(" ", 1),
692 newline(),
693 whitespace(" ", 1),
694 word("钢", 1),
695 word("笔", 1),
696 ],
697 ),
698 (
699 "\u{2003}mutton",
700 &[whitespace("\u{2003}", 1), word("mutton", 6)],
701 ),
702 (
703 "a\u{2009}b\u{2009}c",
704 &[
705 word("a", 1),
706 whitespace("\u{2009}", 1),
707 word("b", 1),
708 whitespace("\u{2009}", 1),
709 word("c", 1),
710 ],
711 ),
712 ("a\u{a0}b\u{a0}c", &[word("a\u{a0}b\u{a0}c", 5)]),
713 ("a\u{2007}b\u{2007}c", &[word("a\u{2007}b\u{2007}c", 5)]),
714 ("a\u{202f}b\u{202f}c", &[word("a\u{202f}b\u{202f}c", 5)]),
715 ];
716
717 fn word(token: &'static str, grapheme_len: usize) -> WordBreakToken<'static> {
718 WordBreakToken::Word {
719 token,
720 grapheme_len,
721 }
722 }
723
724 fn whitespace(token: &'static str, grapheme_len: usize) -> WordBreakToken<'static> {
725 WordBreakToken::InlineWhitespace {
726 token,
727 grapheme_len,
728 }
729 }
730
731 fn newline() -> WordBreakToken<'static> {
732 WordBreakToken::Newline
733 }
734
735 for (input, result) in tests {
736 assert_eq!(
737 WordBreakingTokenizer::new(input)
738 .collect::<Vec<_>>()
739 .as_slice(),
740 *result,
741 );
742 }
743 }
744
745 #[test]
746 fn test_wrap_with_prefix() {
747 assert_eq!(
748 wrap_with_prefix(
749 "# ".to_string(),
750 "# ".to_string(),
751 "abcdefg".to_string(),
752 4,
753 NonZeroU32::new(4).unwrap(),
754 false,
755 ),
756 "# abcdefg"
757 );
758 assert_eq!(
759 wrap_with_prefix(
760 "".to_string(),
761 "".to_string(),
762 "\thello world".to_string(),
763 8,
764 NonZeroU32::new(4).unwrap(),
765 false,
766 ),
767 "hello\nworld"
768 );
769 assert_eq!(
770 wrap_with_prefix(
771 "// ".to_string(),
772 "// ".to_string(),
773 "xx \nyy zz aa bb cc".to_string(),
774 12,
775 NonZeroU32::new(4).unwrap(),
776 false,
777 ),
778 "// xx yy zz\n// aa bb cc"
779 );
780 assert_eq!(
781 wrap_with_prefix(
782 String::new(),
783 String::new(),
784 "这是什么 \n 钢笔".to_string(),
785 3,
786 NonZeroU32::new(4).unwrap(),
787 false,
788 ),
789 "这是什\n么 钢\n笔"
790 );
791 assert_eq!(
792 wrap_with_prefix(
793 String::new(),
794 String::new(),
795 format!("foo{}bar", '\u{2009}'), // thin space
796 80,
797 NonZeroU32::new(4).unwrap(),
798 false,
799 ),
800 format!("foo{}bar", '\u{2009}')
801 );
802 for non_breaking_space in ['\u{a0}', '\u{2007}', '\u{202f}'] {
803 let input = format!("a{0}a{0}a{0}a{0}a", non_breaking_space);
804 assert_eq!(
805 wrap_with_prefix(
806 String::new(),
807 String::new(),
808 input.clone(),
809 4,
810 NonZeroU32::new(4).unwrap(),
811 false,
812 ),
813 input
814 );
815 }
816 }
817}
818