Skip to repository content1834 lines · 62.6 KB · rust
tenant.openagents/omega
No repository description is available.
OpenAgents Git authority 2026-07-28T03:12:44.788Z Public web read
NIP-34 coordinate
30617:7649603503856e5148d571eac2766b288a8ff1e9e35d380337a1d2b0015b4f92:omegaMaintainersHidden in public view
References2 branches · 1 tag
Read-only clone
git clone https://openagents.com/git/tenant.openagents/omega.gitBrowse files
multi_region.rs
1use anyhow::{Context as _, Result, anyhow};
2
3pub const MARKER_TAG_PREFIX: &str = "<|marker_";
4pub const MARKER_TAG_SUFFIX: &str = "|>";
5pub const RELATIVE_MARKER_TAG_PREFIX: &str = "<|marker";
6const V0316_MIN_BLOCK_LINES: usize = 3;
7const V0316_MAX_BLOCK_LINES: usize = 8;
8const V0318_MIN_BLOCK_LINES: usize = 6;
9const V0318_MAX_BLOCK_LINES: usize = 16;
10const V0618_MIN_BLOCK_LINES: usize = 3;
11const V0618_MAX_BLOCK_LINES: usize = 32;
12const MAX_NUDGE_LINES: usize = 5;
13pub const V0316_END_MARKER: &str = "<[end▁of▁sentence]>";
14pub const V0317_END_MARKER: &str = "<[end▁of▁sentence]>";
15pub const V0318_END_MARKER: &str = "<[end▁of▁sentence]>";
16pub const V0327_END_MARKER: &str = "<[end▁of▁sentence]>";
17
18pub fn marker_tag(number: usize) -> String {
19 format!("{MARKER_TAG_PREFIX}{number}{MARKER_TAG_SUFFIX}")
20}
21
22pub fn marker_tag_relative(delta: isize) -> String {
23 if delta > 0 {
24 format!("<|marker+{delta}|>")
25 } else if delta == 0 {
26 String::from("<|marker-0|>")
27 } else {
28 format!("<|marker{delta}|>")
29 }
30}
31
32struct LineInfo {
33 start: usize,
34 is_blank: bool,
35 is_good_start: bool,
36}
37
38fn collect_line_info(text: &str) -> Vec<LineInfo> {
39 let mut lines = Vec::new();
40 let mut offset = 0;
41 for line in text.split('\n') {
42 let trimmed = line.trim();
43 let is_blank = trimmed.is_empty();
44 let is_good_start = !is_blank && !is_structural_tail(trimmed);
45 lines.push(LineInfo {
46 start: offset,
47 is_blank,
48 is_good_start,
49 });
50 offset += line.len() + 1;
51 }
52 // split('\n') on "abc\n" yields ["abc", ""] — drop the phantom trailing
53 // empty element when the text ends with '\n'.
54 if text.ends_with('\n') && lines.len() > 1 {
55 lines.pop();
56 }
57 lines
58}
59
60/// Whether a trimmed line is a model-friendly place to start a block: it has
61/// content and isn't a structural tail. Exposed for reuse by context
62/// retrieval when snapping excerpt boundaries to block boundaries.
63pub fn is_good_block_start(trimmed_line: &str) -> bool {
64 !trimmed_line.is_empty() && !is_structural_tail(trimmed_line)
65}
66
67fn is_structural_tail(trimmed_line: &str) -> bool {
68 if trimmed_line.starts_with(&['}', ']', ')']) {
69 return true;
70 }
71 matches!(
72 trimmed_line.trim_end_matches(';'),
73 "break" | "continue" | "return" | "throw" | "end"
74 )
75}
76
77/// Starting from line `from`, scan up to `MAX_NUDGE_LINES` forward to find a
78/// line with `is_good_start`. Returns `None` if no suitable line is found.
79fn skip_to_good_start(lines: &[LineInfo], from: usize) -> Option<usize> {
80 (from..lines.len().min(from + MAX_NUDGE_LINES)).find(|&i| lines[i].is_good_start)
81}
82
83/// Compute byte offsets within `editable_text` where marker boundaries should
84/// be placed.
85///
86/// Returns a sorted `Vec<usize>` that always starts with `0` and ends with
87/// `editable_text.len()`. Interior offsets are placed at line boundaries
88/// (right after a `\n`), preferring blank-line boundaries when available and
89/// respecting `min_block_lines` / `max_block_lines` constraints.
90fn compute_marker_offsets_with_limits(
91 editable_text: &str,
92 min_block_lines: usize,
93 max_block_lines: usize,
94) -> Vec<usize> {
95 if editable_text.is_empty() {
96 return vec![0, 0];
97 }
98
99 let lines = collect_line_info(editable_text);
100 let mut offsets = vec![0usize];
101 let mut last_boundary_line = 0;
102 let mut i = 0;
103
104 while i < lines.len() {
105 let gap = i - last_boundary_line;
106
107 // Blank-line split: non-blank line following blank line(s) with enough
108 // accumulated lines.
109 if gap >= min_block_lines && !lines[i].is_blank && i > 0 && lines[i - 1].is_blank {
110 let target = if lines[i].is_good_start {
111 i
112 } else {
113 skip_to_good_start(&lines, i).unwrap_or(i)
114 };
115 if lines.len() - target >= min_block_lines
116 && lines[target].start > *offsets.last().unwrap_or(&0)
117 {
118 offsets.push(lines[target].start);
119 last_boundary_line = target;
120 i = target + 1;
121 continue;
122 }
123 }
124
125 // Hard cap: too many lines without a split.
126 if gap >= max_block_lines {
127 let target = skip_to_good_start(&lines, i).unwrap_or(i);
128 if lines[target].start > *offsets.last().unwrap_or(&0) {
129 offsets.push(lines[target].start);
130 last_boundary_line = target;
131 i = target + 1;
132 continue;
133 }
134 }
135
136 i += 1;
137 }
138
139 let end = editable_text.len();
140 if *offsets.last().unwrap_or(&0) != end {
141 offsets.push(end);
142 }
143
144 offsets
145}
146
147/// Compute byte offsets within `editable_text` for the V0316/V0317 block sizing rules.
148pub fn compute_marker_offsets(editable_text: &str) -> Vec<usize> {
149 compute_marker_offsets_with_limits(editable_text, V0316_MIN_BLOCK_LINES, V0316_MAX_BLOCK_LINES)
150}
151
152pub fn compute_marker_offsets_v0318(editable_text: &str) -> Vec<usize> {
153 compute_marker_offsets_with_limits(editable_text, V0318_MIN_BLOCK_LINES, V0318_MAX_BLOCK_LINES)
154}
155
156pub fn compute_marker_offsets_v0618(editable_text: &str) -> Vec<usize> {
157 compute_marker_offsets_with_limits(editable_text, V0618_MIN_BLOCK_LINES, V0618_MAX_BLOCK_LINES)
158}
159
160fn line_start_at_or_before(text: &str, offset: usize) -> usize {
161 let bounded_offset = text.floor_char_boundary(offset.min(text.len()));
162 text[..bounded_offset]
163 .rfind('\n')
164 .map(|index| index + 1)
165 .unwrap_or(0)
166}
167
168fn line_end_at_or_after(text: &str, offset: usize) -> usize {
169 let bounded_offset = text.floor_char_boundary(offset.min(text.len()));
170 if bounded_offset >= text.len() {
171 return text.len();
172 }
173
174 text[bounded_offset..]
175 .find('\n')
176 .map(|index| bounded_offset + index + 1)
177 .unwrap_or(text.len())
178}
179
180fn grow_v0327_candidate_range(
181 text: &str,
182 cursor_offset: usize,
183 editable_token_limit: usize,
184) -> std::ops::Range<usize> {
185 if text.is_empty() {
186 return 0..0;
187 }
188
189 let byte_budget = editable_token_limit.saturating_mul(3).max(1);
190 let half_budget = byte_budget / 2;
191
192 let mut start = cursor_offset.saturating_sub(half_budget);
193 let mut end = start.saturating_add(byte_budget).min(text.len());
194
195 if end.saturating_sub(start) < byte_budget {
196 start = end.saturating_sub(byte_budget);
197 }
198
199 start = line_start_at_or_before(text, start);
200 end = line_end_at_or_after(text, end);
201
202 if start < end {
203 start..end
204 } else {
205 let line_start = line_start_at_or_before(text, cursor_offset);
206 let line_end = line_end_at_or_after(text, cursor_offset);
207 line_start..line_end.max(line_start)
208 }
209}
210
211fn trim_v0327_candidate_range_to_markers(
212 text: &str,
213 candidate_range: std::ops::Range<usize>,
214 cursor_offset: usize,
215) -> std::ops::Range<usize> {
216 let candidate_text = &text[candidate_range.clone()];
217 let marker_offsets = compute_marker_offsets_v0318(candidate_text);
218
219 if marker_offsets.len() <= 2 {
220 return candidate_range;
221 }
222
223 let candidate_cursor_offset = cursor_offset
224 .saturating_sub(candidate_range.start)
225 .min(candidate_text.len());
226 let first_internal_marker_index = if candidate_cursor_offset >= marker_offsets[1] {
227 1
228 } else {
229 0
230 };
231 let last_internal_marker_index = marker_offsets.len() - 2;
232 let last_marker_index = marker_offsets.len() - 1;
233 let end_marker_index = if candidate_cursor_offset <= marker_offsets[last_internal_marker_index]
234 {
235 last_internal_marker_index
236 } else {
237 last_marker_index
238 };
239
240 let trimmed_start = candidate_range.start + marker_offsets[first_internal_marker_index];
241 let trimmed_end = candidate_range.start + marker_offsets[end_marker_index];
242
243 if trimmed_start < trimmed_end {
244 trimmed_start..trimmed_end
245 } else {
246 let block_index = cursor_block_index(Some(candidate_cursor_offset), &marker_offsets);
247 let start = candidate_range.start + marker_offsets[block_index];
248 let end = candidate_range.start + marker_offsets[block_index + 1];
249 if start < end {
250 start..end
251 } else {
252 candidate_range
253 }
254 }
255}
256
257pub fn compute_v0327_editable_range(
258 text: &str,
259 cursor_offset: usize,
260 editable_token_limit: usize,
261) -> std::ops::Range<usize> {
262 let candidate_range = grow_v0327_candidate_range(text, cursor_offset, editable_token_limit);
263 trim_v0327_candidate_range_to_markers(text, candidate_range, cursor_offset)
264}
265
266/// Write the editable region content with marker tags, inserting the cursor
267/// marker at the given offset within the editable text.
268pub fn write_editable_with_markers(
269 output: &mut String,
270 editable_text: &str,
271 cursor_offset_in_editable: usize,
272 cursor_marker: &str,
273) {
274 let marker_offsets = compute_marker_offsets(editable_text);
275 let mut cursor_placed = false;
276 for (i, &offset) in marker_offsets.iter().enumerate() {
277 let marker_num = i + 1;
278 if !output.is_empty() && !output.ends_with('\n') {
279 output.push('\n');
280 }
281 output.push_str(&marker_tag(marker_num));
282
283 if let Some(&next_offset) = marker_offsets.get(i + 1) {
284 output.push('\n');
285 let block = &editable_text[offset..next_offset];
286 if !cursor_placed
287 && cursor_offset_in_editable >= offset
288 && cursor_offset_in_editable <= next_offset
289 {
290 cursor_placed = true;
291 let cursor_in_block = cursor_offset_in_editable - offset;
292 output.push_str(&block[..cursor_in_block]);
293 output.push_str(cursor_marker);
294 output.push_str(&block[cursor_in_block..]);
295 } else {
296 output.push_str(block);
297 }
298 }
299 }
300}
301
302/// Strip any `<|marker_N|>` tags from `text`.
303///
304/// When a marker tag sits on its own line (followed by `\n`), the trailing
305/// newline is also removed so the surrounding lines stay joined naturally.
306pub(crate) fn strip_marker_tags(text: &str) -> String {
307 let mut result = String::with_capacity(text.len());
308 let mut pos = 0;
309 let bytes = text.as_bytes();
310 while let Some(rel) = text[pos..].find(MARKER_TAG_PREFIX) {
311 result.push_str(&text[pos..pos + rel]);
312 let num_start = pos + rel + MARKER_TAG_PREFIX.len();
313 if let Some(suffix_rel) = text[num_start..].find(MARKER_TAG_SUFFIX) {
314 let mut tag_end = num_start + suffix_rel + MARKER_TAG_SUFFIX.len();
315 if bytes.get(tag_end) == Some(&b'\n') {
316 tag_end += 1;
317 }
318 pos = tag_end;
319 } else {
320 result.push_str(MARKER_TAG_PREFIX);
321 pos = num_start;
322 }
323 }
324 result.push_str(&text[pos..]);
325 result
326}
327
328/// Parse model output that uses the marker format.
329///
330/// Returns `(start_marker_num, end_marker_num, content_between_markers)`.
331/// The leading format-level newline after the start marker is stripped.
332/// Trailing newlines are preserved so blank-line endings in the editable
333/// region are not lost.
334///
335/// Any extra intermediate marker tags that the model may have inserted
336/// between the first and last markers are stripped from the returned content.
337pub fn extract_marker_span(text: &str) -> Result<(usize, usize, String)> {
338 let first_tag_start = text
339 .find(MARKER_TAG_PREFIX)
340 .context("no start marker found in output")?;
341 let first_num_start = first_tag_start + MARKER_TAG_PREFIX.len();
342 let first_num_end = text[first_num_start..]
343 .find(MARKER_TAG_SUFFIX)
344 .map(|i| i + first_num_start)
345 .context("malformed start marker tag")?;
346 let start_num: usize = text[first_num_start..first_num_end]
347 .parse()
348 .context("start marker number is not a valid integer")?;
349 let first_tag_end = first_num_end + MARKER_TAG_SUFFIX.len();
350
351 let last_tag_start = text
352 .rfind(MARKER_TAG_PREFIX)
353 .context("no end marker found in output")?;
354 let last_num_start = last_tag_start + MARKER_TAG_PREFIX.len();
355 let last_num_end = text[last_num_start..]
356 .find(MARKER_TAG_SUFFIX)
357 .map(|i| i + last_num_start)
358 .context("malformed end marker tag")?;
359 let end_num: usize = text[last_num_start..last_num_end]
360 .parse()
361 .context("end marker number is not a valid integer")?;
362
363 if start_num == end_num {
364 return Err(anyhow!(
365 "start and end markers are the same (marker {})",
366 start_num
367 ));
368 }
369
370 let mut content_start = first_tag_end;
371 if text.as_bytes().get(content_start) == Some(&b'\n') {
372 content_start += 1;
373 }
374 let content_end = last_tag_start;
375
376 let content = &text[content_start..content_end.max(content_start)];
377 let content = strip_marker_tags(content);
378 Ok((start_num, end_num, content))
379}
380
381/// Given old editable text and model output with marker span, reconstruct the
382/// full new editable region.
383pub fn apply_marker_span(old_editable: &str, output: &str) -> Result<String> {
384 let (start_num, end_num, raw_new_span) = extract_marker_span(output)?;
385 let marker_offsets = compute_marker_offsets(old_editable);
386
387 let start_idx = start_num
388 .checked_sub(1)
389 .context("marker numbers are 1-indexed")?;
390 let end_idx = end_num
391 .checked_sub(1)
392 .context("marker numbers are 1-indexed")?;
393 let start_byte = *marker_offsets
394 .get(start_idx)
395 .context("start marker number out of range")?;
396 let end_byte = *marker_offsets
397 .get(end_idx)
398 .context("end marker number out of range")?;
399
400 if start_byte > end_byte {
401 return Err(anyhow!("start marker must come before end marker"));
402 }
403
404 let old_span = &old_editable[start_byte..end_byte];
405 let mut new_span = raw_new_span;
406 if old_span.ends_with('\n') && !new_span.ends_with('\n') && !new_span.is_empty() {
407 new_span.push('\n');
408 }
409 if !old_span.ends_with('\n') && new_span.ends_with('\n') {
410 new_span.pop();
411 }
412
413 let mut result = String::new();
414 result.push_str(&old_editable[..start_byte]);
415 result.push_str(&new_span);
416 result.push_str(&old_editable[end_byte..]);
417
418 Ok(result)
419}
420
421/// Compare old and new editable text, find the minimal marker span that covers
422/// all changes, and encode the result with marker tags.
423pub fn encode_from_old_and_new(
424 old_editable: &str,
425 new_editable: &str,
426 cursor_offset_in_new: Option<usize>,
427 cursor_marker: &str,
428 end_marker: &str,
429 no_edits_marker: &str,
430) -> Result<String> {
431 if old_editable == new_editable {
432 return Ok(format!("{no_edits_marker}{end_marker}"));
433 }
434
435 let marker_offsets = compute_marker_offsets(old_editable);
436 let (common_prefix, common_suffix) =
437 common_prefix_suffix(old_editable.as_bytes(), new_editable.as_bytes());
438 let change_end_in_old = old_editable.len() - common_suffix;
439
440 let start_marker_idx = marker_offsets
441 .iter()
442 .rposition(|&offset| offset <= common_prefix)
443 .unwrap_or(0);
444 let end_marker_idx = marker_offsets
445 .iter()
446 .position(|&offset| offset >= change_end_in_old)
447 .unwrap_or(marker_offsets.len() - 1);
448
449 let old_start = marker_offsets[start_marker_idx];
450 let old_end = marker_offsets[end_marker_idx];
451
452 let new_start = old_start;
453 let new_end = new_editable
454 .len()
455 .saturating_sub(old_editable.len().saturating_sub(old_end));
456
457 let new_span = &new_editable[new_start..new_end];
458
459 let start_marker_num = start_marker_idx + 1;
460 let end_marker_num = end_marker_idx + 1;
461
462 let mut result = String::new();
463 result.push_str(&marker_tag(start_marker_num));
464 result.push('\n');
465
466 if let Some(cursor_offset) = cursor_offset_in_new {
467 if cursor_offset >= new_start && cursor_offset <= new_end {
468 let cursor_in_span = cursor_offset - new_start;
469 let bounded = cursor_in_span.min(new_span.len());
470 result.push_str(&new_span[..bounded]);
471 result.push_str(cursor_marker);
472 result.push_str(&new_span[bounded..]);
473 } else {
474 result.push_str(new_span);
475 }
476 } else {
477 result.push_str(new_span);
478 }
479
480 if !result.ends_with('\n') {
481 result.push('\n');
482 }
483 result.push_str(&marker_tag(end_marker_num));
484 result.push('\n');
485 result.push_str(end_marker);
486
487 Ok(result)
488}
489
490/// Extract the full editable region from text that uses marker tags.
491///
492/// Returns the concatenation of all block contents between the first and last
493/// markers, with intermediate marker tags stripped.
494pub fn extract_editable_region_from_markers(text: &str) -> Option<String> {
495 let first_marker_start = text.find(MARKER_TAG_PREFIX)?;
496
497 let mut markers: Vec<(usize, usize)> = Vec::new();
498 let mut search_start = first_marker_start;
499 while let Some(rel_pos) = text[search_start..].find(MARKER_TAG_PREFIX) {
500 let tag_start = search_start + rel_pos;
501 let num_start = tag_start + MARKER_TAG_PREFIX.len();
502 let num_end = text[num_start..].find(MARKER_TAG_SUFFIX)?;
503 let tag_end = num_start + num_end + MARKER_TAG_SUFFIX.len();
504 markers.push((tag_start, tag_end));
505 search_start = tag_end;
506 }
507
508 if markers.len() < 2 {
509 return None;
510 }
511
512 let (_, first_tag_end) = markers[0];
513 let (last_tag_start, _) = markers[markers.len() - 1];
514
515 let mut content_start = first_tag_end;
516 if text.as_bytes().get(content_start) == Some(&b'\n') {
517 content_start += 1;
518 }
519 let mut content_end = last_tag_start;
520 if content_end > content_start && text.as_bytes().get(content_end - 1) == Some(&b'\n') {
521 content_end -= 1;
522 }
523
524 let raw = &text[content_start..content_end];
525 let result = strip_marker_tags(raw);
526 let result = result.strip_suffix('\n').unwrap_or(&result).to_string();
527 Some(result)
528}
529
530struct ParsedTag {
531 value: isize,
532 tag_start: usize,
533 tag_end: usize,
534}
535
536fn collect_tags(text: &str, prefix: &str, parse: fn(&str) -> Option<isize>) -> Vec<ParsedTag> {
537 let mut tags = Vec::new();
538 let mut search_from = 0;
539 while let Some(rel_pos) = text[search_from..].find(prefix) {
540 let tag_start = search_from + rel_pos;
541 let payload_start = tag_start + prefix.len();
542 if let Some(suffix_rel) = text[payload_start..].find(MARKER_TAG_SUFFIX) {
543 let payload_end = payload_start + suffix_rel;
544 if let Some(value) = parse(&text[payload_start..payload_end]) {
545 let tag_end = payload_end + MARKER_TAG_SUFFIX.len();
546 tags.push(ParsedTag {
547 value,
548 tag_start,
549 tag_end,
550 });
551 search_from = tag_end;
552 continue;
553 }
554 }
555 search_from = tag_start + prefix.len();
556 }
557 tags
558}
559
560fn collect_marker_tags(text: &str) -> Vec<ParsedTag> {
561 collect_tags(text, MARKER_TAG_PREFIX, |s| {
562 s.parse::<usize>().ok().map(|n| n as isize)
563 })
564}
565
566fn collect_relative_marker_tags(text: &str) -> Vec<ParsedTag> {
567 collect_tags(text, RELATIVE_MARKER_TAG_PREFIX, |s| {
568 s.parse::<isize>().ok()
569 })
570}
571
572pub fn nearest_marker_number(cursor_offset: Option<usize>, marker_offsets: &[usize]) -> usize {
573 let cursor = cursor_offset.unwrap_or(0);
574 marker_offsets
575 .iter()
576 .enumerate()
577 .min_by_key(|(_, offset)| (**offset as isize - cursor as isize).unsigned_abs())
578 .map(|(idx, _)| idx + 1)
579 .unwrap_or(1)
580}
581
582fn cursor_block_index(cursor_offset: Option<usize>, marker_offsets: &[usize]) -> usize {
583 let cursor = cursor_offset.unwrap_or(0);
584 marker_offsets
585 .array_windows::<2>()
586 .position(|&[a, b]| cursor >= a && cursor < b)
587 .unwrap_or_else(|| marker_offsets.len().saturating_sub(2))
588}
589
590fn common_prefix_suffix(a: &[u8], b: &[u8]) -> (usize, usize) {
591 let prefix = a.iter().zip(b.iter()).take_while(|(x, y)| x == y).count();
592 let remaining_a = a.len() - prefix;
593 let remaining_b = b.len() - prefix;
594 let max_suffix = remaining_a.min(remaining_b);
595 let suffix = a[a.len() - max_suffix..]
596 .iter()
597 .rev()
598 .zip(b[b.len() - max_suffix..].iter().rev())
599 .take_while(|(x, y)| x == y)
600 .count();
601 (prefix, suffix)
602}
603
604/// Map a byte offset from old span coordinates to new span coordinates,
605/// using common prefix/suffix within the span for accuracy.
606fn map_boundary_offset(
607 old_rel: usize,
608 old_span_len: usize,
609 new_span_len: usize,
610 span_common_prefix: usize,
611 span_common_suffix: usize,
612) -> usize {
613 if old_rel <= span_common_prefix {
614 old_rel
615 } else if old_rel >= old_span_len - span_common_suffix {
616 new_span_len - (old_span_len - old_rel)
617 } else {
618 let old_changed_start = span_common_prefix;
619 let old_changed_len = old_span_len
620 .saturating_sub(span_common_prefix)
621 .saturating_sub(span_common_suffix);
622 let new_changed_start = span_common_prefix;
623 let new_changed_len = new_span_len
624 .saturating_sub(span_common_prefix)
625 .saturating_sub(span_common_suffix);
626
627 new_changed_start
628 + ((old_rel - old_changed_start) * new_changed_len)
629 .checked_div(old_changed_len)
630 .unwrap_or(new_changed_len)
631 }
632}
633
634fn snap_to_line_start(text: &str, offset: usize) -> usize {
635 let bounded = offset.min(text.len());
636 let bounded = text.floor_char_boundary(bounded);
637
638 if bounded >= text.len() {
639 return text.len();
640 }
641
642 if bounded == 0 || text.as_bytes().get(bounded - 1) == Some(&b'\n') {
643 return bounded;
644 }
645
646 if let Some(next_nl_rel) = text[bounded..].find('\n') {
647 let next = bounded + next_nl_rel + 1;
648 return text.floor_char_boundary(next.min(text.len()));
649 }
650
651 let prev_start = text[..bounded].rfind('\n').map(|idx| idx + 1).unwrap_or(0);
652 text.floor_char_boundary(prev_start)
653}
654
655/// Write the editable region content with byte-exact marker tags, inserting the
656/// cursor marker at the given offset within the editable text.
657///
658/// The `tag_for_index` closure maps a boundary index to the marker tag string.
659fn write_editable_with_markers_impl(
660 output: &mut String,
661 editable_text: &str,
662 cursor_offset_in_editable: usize,
663 cursor_marker: &str,
664 marker_offsets: &[usize],
665 tag_for_index: impl Fn(usize) -> String,
666) {
667 let mut cursor_placed = false;
668 for (i, &offset) in marker_offsets.iter().enumerate() {
669 output.push_str(&tag_for_index(i));
670
671 if let Some(&next_offset) = marker_offsets.get(i + 1) {
672 let block = &editable_text[offset..next_offset];
673 if !cursor_placed
674 && cursor_offset_in_editable >= offset
675 && cursor_offset_in_editable <= next_offset
676 {
677 cursor_placed = true;
678 let cursor_in_block = cursor_offset_in_editable - offset;
679 output.push_str(&block[..cursor_in_block]);
680 output.push_str(cursor_marker);
681 output.push_str(&block[cursor_in_block..]);
682 } else {
683 output.push_str(block);
684 }
685 }
686 }
687}
688
689pub fn write_editable_with_markers_v0316(
690 output: &mut String,
691 editable_text: &str,
692 cursor_offset_in_editable: usize,
693 cursor_marker: &str,
694) {
695 let marker_offsets = compute_marker_offsets(editable_text);
696 write_editable_with_markers_impl(
697 output,
698 editable_text,
699 cursor_offset_in_editable,
700 cursor_marker,
701 &marker_offsets,
702 |i| marker_tag(i + 1),
703 );
704}
705
706pub fn write_editable_with_markers_v0317(
707 output: &mut String,
708 editable_text: &str,
709 cursor_offset_in_editable: usize,
710 cursor_marker: &str,
711) {
712 let marker_offsets = compute_marker_offsets(editable_text);
713 let anchor_idx = cursor_block_index(Some(cursor_offset_in_editable), &marker_offsets);
714 write_editable_with_markers_impl(
715 output,
716 editable_text,
717 cursor_offset_in_editable,
718 cursor_marker,
719 &marker_offsets,
720 |i| marker_tag_relative(i as isize - anchor_idx as isize),
721 );
722}
723
724pub fn write_editable_with_markers_v0318(
725 output: &mut String,
726 editable_text: &str,
727 cursor_offset_in_editable: usize,
728 cursor_marker: &str,
729) {
730 let marker_offsets = compute_marker_offsets_v0318(editable_text);
731 write_editable_with_markers_impl(
732 output,
733 editable_text,
734 cursor_offset_in_editable,
735 cursor_marker,
736 &marker_offsets,
737 |i| marker_tag(i + 1),
738 );
739}
740
741/// Parse byte-exact model output and reconstruct the full new editable region.
742///
743/// `resolve_boundary` maps a parsed tag value to an absolute byte offset in
744/// old_editable, given the marker_offsets. Returns `(start_byte, end_byte)` or
745/// an error.
746fn apply_marker_span_impl(
747 old_editable: &str,
748 tags: &[ParsedTag],
749 output: &str,
750 resolve_boundaries: impl Fn(isize, isize) -> Result<(usize, usize)>,
751) -> Result<String> {
752 if tags.is_empty() {
753 return Err(anyhow!("no marker tags found in output"));
754 }
755 if tags.len() == 1 {
756 return Err(anyhow!(
757 "only one marker tag found in output, expected at least two"
758 ));
759 }
760
761 let start_value = tags[0].value;
762 let end_value = tags[tags.len() - 1].value;
763
764 if start_value == end_value {
765 return Ok(old_editable.to_string());
766 }
767
768 let (start_byte, end_byte) = resolve_boundaries(start_value, end_value)?;
769
770 if start_byte > end_byte {
771 return Err(anyhow!("start marker must come before end marker"));
772 }
773
774 let mut new_content = String::new();
775 for i in 0..tags.len() - 1 {
776 let content_start = tags[i].tag_end;
777 let content_end = tags[i + 1].tag_start;
778 if content_start <= content_end {
779 new_content.push_str(&output[content_start..content_end]);
780 }
781 }
782
783 let mut result = String::new();
784 result.push_str(&old_editable[..start_byte]);
785 result.push_str(&new_content);
786 result.push_str(&old_editable[end_byte..]);
787
788 Ok(result)
789}
790
791pub fn apply_marker_span_v0316(old_editable: &str, output: &str) -> Result<String> {
792 let tags = collect_marker_tags(output);
793
794 // Validate monotonically increasing with no gaps (best-effort warning)
795 if tags.len() >= 2 {
796 let start_num = tags[0].value;
797 let end_num = tags[tags.len() - 1].value;
798 if start_num != end_num {
799 let expected: Vec<isize> = (start_num..=end_num).collect();
800 let actual: Vec<isize> = tags.iter().map(|t| t.value).collect();
801 if actual != expected {
802 eprintln!(
803 "V0316 marker sequence validation failed: expected {:?}, got {:?}. Attempting best-effort parse.",
804 expected, actual
805 );
806 }
807 }
808 }
809
810 let marker_offsets = compute_marker_offsets(old_editable);
811 apply_marker_span_impl(old_editable, &tags, output, |start_val, end_val| {
812 let start_idx = (start_val as usize)
813 .checked_sub(1)
814 .context("marker numbers are 1-indexed")?;
815 let end_idx = (end_val as usize)
816 .checked_sub(1)
817 .context("marker numbers are 1-indexed")?;
818 let start_byte = *marker_offsets
819 .get(start_idx)
820 .context("start marker number out of range")?;
821 let end_byte = *marker_offsets
822 .get(end_idx)
823 .context("end marker number out of range")?;
824 Ok((start_byte, end_byte))
825 })
826}
827
828pub fn apply_marker_span_v0317(
829 old_editable: &str,
830 output: &str,
831 cursor_offset_in_old: Option<usize>,
832) -> Result<String> {
833 let tags = collect_relative_marker_tags(output);
834 let marker_offsets = compute_marker_offsets(old_editable);
835 let anchor_idx = cursor_block_index(cursor_offset_in_old, &marker_offsets);
836
837 apply_marker_span_impl(old_editable, &tags, output, |start_delta, end_delta| {
838 let start_idx_signed = anchor_idx as isize + start_delta;
839 let end_idx_signed = anchor_idx as isize + end_delta;
840 if start_idx_signed < 0 || end_idx_signed < 0 {
841 return Err(anyhow!("relative marker maps before first marker"));
842 }
843 let start_idx = usize::try_from(start_idx_signed).context("invalid start marker index")?;
844 let end_idx = usize::try_from(end_idx_signed).context("invalid end marker index")?;
845 let start_byte = *marker_offsets
846 .get(start_idx)
847 .context("start marker number out of range")?;
848 let end_byte = *marker_offsets
849 .get(end_idx)
850 .context("end marker number out of range")?;
851 Ok((start_byte, end_byte))
852 })
853}
854
855pub fn apply_marker_span_v0318(old_editable: &str, output: &str) -> Result<String> {
856 let tags = collect_marker_tags(output);
857
858 if tags.len() >= 2 {
859 let start_num = tags[0].value;
860 let end_num = tags[tags.len() - 1].value;
861 if start_num != end_num {
862 let expected: Vec<isize> = (start_num..=end_num).collect();
863 let actual: Vec<isize> = tags.iter().map(|t| t.value).collect();
864 if actual != expected {
865 eprintln!(
866 "V0318 marker sequence validation failed: expected {:?}, got {:?}. Attempting best-effort parse.",
867 expected, actual
868 );
869 }
870 }
871 }
872
873 let marker_offsets = compute_marker_offsets_v0318(old_editable);
874 apply_marker_span_impl(old_editable, &tags, output, |start_val, end_val| {
875 let start_idx = (start_val as usize)
876 .checked_sub(1)
877 .context("marker numbers are 1-indexed")?;
878 let end_idx = (end_val as usize)
879 .checked_sub(1)
880 .context("marker numbers are 1-indexed")?;
881 let start_byte = *marker_offsets
882 .get(start_idx)
883 .context("start marker number out of range")?;
884 let end_byte = *marker_offsets
885 .get(end_idx)
886 .context("end marker number out of range")?;
887 Ok((start_byte, end_byte))
888 })
889}
890
891/// Encode the training target from old and new editable text.
892///
893/// Shared implementation for V0316, V0317, and V0318. The `tag_for_block_idx`
894/// closure maps a block index to the appropriate marker tag string.
895/// `no_edit_tag` is the marker tag to repeat when there are no edits.
896fn encode_from_old_and_new_impl(
897 old_editable: &str,
898 new_editable: &str,
899 cursor_offset_in_new: Option<usize>,
900 cursor_marker: &str,
901 end_marker: &str,
902 no_edit_tag: &str,
903 marker_offsets: &[usize],
904 tag_for_block_idx: impl Fn(usize) -> String,
905) -> Result<String> {
906 if old_editable == new_editable {
907 return Ok(format!("{no_edit_tag}{no_edit_tag}{end_marker}"));
908 }
909
910 let (common_prefix, common_suffix) =
911 common_prefix_suffix(old_editable.as_bytes(), new_editable.as_bytes());
912 let change_end_in_old = old_editable.len() - common_suffix;
913
914 let mut start_marker_idx = marker_offsets
915 .iter()
916 .rposition(|&offset| offset <= common_prefix)
917 .unwrap_or(0);
918 let mut end_marker_idx = marker_offsets
919 .iter()
920 .position(|&offset| offset >= change_end_in_old)
921 .unwrap_or(marker_offsets.len() - 1);
922
923 if start_marker_idx == end_marker_idx {
924 if end_marker_idx < marker_offsets.len().saturating_sub(1) {
925 end_marker_idx += 1;
926 } else if start_marker_idx > 0 {
927 start_marker_idx -= 1;
928 }
929 }
930
931 let old_start = marker_offsets[start_marker_idx];
932 let old_end = marker_offsets[end_marker_idx];
933
934 let new_start = old_start;
935 let new_end = new_editable
936 .len()
937 .saturating_sub(old_editable.len().saturating_sub(old_end));
938
939 let new_span = &new_editable[new_start..new_end];
940 let old_span = &old_editable[old_start..old_end];
941
942 let (span_common_prefix, span_common_suffix) =
943 common_prefix_suffix(old_span.as_bytes(), new_span.as_bytes());
944
945 let mut result = String::new();
946 let mut prev_new_rel = 0usize;
947 let mut cursor_placed = false;
948
949 for block_idx in start_marker_idx..end_marker_idx {
950 result.push_str(&tag_for_block_idx(block_idx));
951
952 let new_rel_end = if block_idx + 1 == end_marker_idx {
953 new_span.len()
954 } else {
955 let old_rel = marker_offsets[block_idx + 1] - old_start;
956 let mapped = map_boundary_offset(
957 old_rel,
958 old_span.len(),
959 new_span.len(),
960 span_common_prefix,
961 span_common_suffix,
962 );
963 snap_to_line_start(new_span, mapped)
964 };
965
966 let new_rel_end = new_rel_end.max(prev_new_rel);
967 let block_content = &new_span[prev_new_rel..new_rel_end];
968
969 if !cursor_placed {
970 if let Some(cursor_offset) = cursor_offset_in_new {
971 let abs_start = new_start + prev_new_rel;
972 let abs_end = new_start + new_rel_end;
973 if cursor_offset >= abs_start && cursor_offset <= abs_end {
974 cursor_placed = true;
975 let cursor_in_block = cursor_offset - abs_start;
976 let bounded = cursor_in_block.min(block_content.len());
977 result.push_str(&block_content[..bounded]);
978 result.push_str(cursor_marker);
979 result.push_str(&block_content[bounded..]);
980 prev_new_rel = new_rel_end;
981 continue;
982 }
983 }
984 }
985
986 result.push_str(block_content);
987 prev_new_rel = new_rel_end;
988 }
989
990 result.push_str(&tag_for_block_idx(end_marker_idx));
991 result.push_str(end_marker);
992
993 Ok(result)
994}
995
996pub fn encode_from_old_and_new_v0316(
997 old_editable: &str,
998 new_editable: &str,
999 cursor_offset_in_new: Option<usize>,
1000 cursor_marker: &str,
1001 end_marker: &str,
1002) -> Result<String> {
1003 let marker_offsets = compute_marker_offsets(old_editable);
1004 let no_edit_tag = marker_tag(nearest_marker_number(cursor_offset_in_new, &marker_offsets));
1005 encode_from_old_and_new_impl(
1006 old_editable,
1007 new_editable,
1008 cursor_offset_in_new,
1009 cursor_marker,
1010 end_marker,
1011 &no_edit_tag,
1012 &marker_offsets,
1013 |block_idx| marker_tag(block_idx + 1),
1014 )
1015}
1016
1017pub fn encode_from_old_and_new_v0317(
1018 old_editable: &str,
1019 new_editable: &str,
1020 cursor_offset_in_new: Option<usize>,
1021 cursor_marker: &str,
1022 end_marker: &str,
1023) -> Result<String> {
1024 let marker_offsets = compute_marker_offsets(old_editable);
1025 let anchor_idx = cursor_block_index(cursor_offset_in_new, &marker_offsets);
1026 let no_edit_tag = marker_tag_relative(0);
1027 encode_from_old_and_new_impl(
1028 old_editable,
1029 new_editable,
1030 cursor_offset_in_new,
1031 cursor_marker,
1032 end_marker,
1033 &no_edit_tag,
1034 &marker_offsets,
1035 |block_idx| marker_tag_relative(block_idx as isize - anchor_idx as isize),
1036 )
1037}
1038
1039pub fn encode_from_old_and_new_v0318(
1040 old_editable: &str,
1041 new_editable: &str,
1042 cursor_offset_in_new: Option<usize>,
1043 cursor_marker: &str,
1044 end_marker: &str,
1045) -> Result<String> {
1046 let marker_offsets = compute_marker_offsets_v0318(old_editable);
1047 let no_edit_tag = marker_tag(nearest_marker_number(cursor_offset_in_new, &marker_offsets));
1048 encode_from_old_and_new_impl(
1049 old_editable,
1050 new_editable,
1051 cursor_offset_in_new,
1052 cursor_marker,
1053 end_marker,
1054 &no_edit_tag,
1055 &marker_offsets,
1056 |block_idx| marker_tag(block_idx + 1),
1057 )
1058}
1059
1060#[cfg(test)]
1061mod tests {
1062 use super::*;
1063
1064 #[test]
1065 fn test_compute_marker_offsets_small_block() {
1066 let text = "aaa\nbbb\nccc\n";
1067 let offsets = compute_marker_offsets(text);
1068 assert_eq!(offsets, vec![0, text.len()]);
1069 }
1070
1071 #[test]
1072 fn test_compute_marker_offsets_blank_line_split() {
1073 let text = "aaa\nbbb\nccc\n\nddd\neee\nfff\n";
1074 let offsets = compute_marker_offsets(text);
1075 assert_eq!(offsets[0], 0);
1076 assert!(offsets.contains(&13), "offsets: {:?}", offsets);
1077 assert_eq!(*offsets.last().unwrap(), text.len());
1078 }
1079
1080 #[test]
1081 fn test_compute_marker_offsets_blank_line_split_overrides_pending_hard_cap_boundary() {
1082 let text = "\
1083class OCRDataframe(BaseModel):
1084 model_config = ConfigDict(arbitrary_types_allowed=True)
1085
1086 df: pl.DataFrame
1087
1088 def page(self, page_number: int = 0) -> \"OCRDataframe\":
1089 # Filter dataframe on specific page
1090 df_page = self.df.filter(pl.col(\"page\") == page_number)
1091 return OCRDataframe(df=df_page)
1092
1093 def get_text_cell(
1094 self,
1095 cell: Cell,
1096 margin: int = 0,
1097 page_number: Optional[int] = None,
1098 min_confidence: int = 50,
1099 ) -> Optional[str]:
1100 \"\"\"
1101 Get text corresponding to cell
1102";
1103 let offsets = compute_marker_offsets(text);
1104
1105 let def_start = text
1106 .find(" def get_text_cell(")
1107 .expect("def line exists");
1108 let self_start = text.find(" self,").expect("self line exists");
1109
1110 assert!(
1111 offsets.contains(&def_start),
1112 "expected boundary at def line start ({def_start}), got {offsets:?}"
1113 );
1114 assert!(
1115 !offsets.contains(&self_start),
1116 "did not expect boundary at self line start ({self_start}), got {offsets:?}"
1117 );
1118 }
1119
1120 #[test]
1121 fn test_compute_marker_offsets_blank_line_split_skips_closer_line() {
1122 let text = "\
1123impl Plugin for AhoySchedulePlugin {
1124 fn build(&self, app: &mut App) {
1125 app.configure_sets(
1126 self.schedule,
1127 (
1128 AhoySystems::MoveCharacters,
1129 AhoySystems::ApplyForcesToDynamicRigidBodies,
1130 )
1131 .chain()
1132 .before(PhysicsSystems::First),
1133 );
1134
1135 }
1136}
1137
1138/// System set used by all systems of `bevy_ahoy`.
1139#[derive(SystemSet, Debug, Clone, Copy, Hash, PartialEq, Eq)]
1140pub enum AhoySystems {
1141 MoveCharacters,
1142 ApplyForcesToDynamicRigidBodies,
1143}
1144";
1145 let offsets = compute_marker_offsets(text);
1146
1147 let closer_start = text.find(" }\n").expect("closer line exists");
1148 let doc_start = text
1149 .find("/// System set used by all systems of `bevy_ahoy`.")
1150 .expect("doc line exists");
1151
1152 assert!(
1153 !offsets.contains(&closer_start),
1154 "did not expect boundary at closer line start ({closer_start}), got {offsets:?}"
1155 );
1156 assert!(
1157 offsets.contains(&doc_start),
1158 "expected boundary at doc line start ({doc_start}), got {offsets:?}"
1159 );
1160 }
1161
1162 #[test]
1163 fn test_compute_marker_offsets_max_lines_split() {
1164 let text = "1\n2\n3\n4\n5\n6\n7\n8\n9\n10\n";
1165 let offsets = compute_marker_offsets(text);
1166 assert!(offsets.len() >= 3, "offsets: {:?}", offsets);
1167 }
1168
1169 #[test]
1170 fn test_compute_marker_offsets_hard_cap_nudges_past_closer_to_case_line() {
1171 let text = "a1\na2\na3\na4\na5\na6\na7\na8\n}\ncase 'x': {\nbody\n";
1172 let offsets = compute_marker_offsets(text);
1173
1174 let expected = text.find("case 'x': {").expect("case line exists");
1175 assert!(
1176 offsets.contains(&expected),
1177 "expected nudged boundary at case line start ({expected}), got {offsets:?}"
1178 );
1179 }
1180
1181 #[test]
1182 fn test_compute_marker_offsets_hard_cap_nudge_respects_max_forward_lines() {
1183 let text = "a1\na2\na3\na4\na5\na6\na7\na8\n}\n}\n}\n}\n}\ncase 'x': {\nbody\n";
1184 let offsets = compute_marker_offsets(text);
1185
1186 let case_start = text.find("case 'x': {").expect("case line exists");
1187 assert!(
1188 !offsets.contains(&case_start),
1189 "boundary should not nudge beyond max forward lines; offsets: {offsets:?}"
1190 );
1191 }
1192
1193 #[test]
1194 fn test_compute_marker_offsets_stay_sorted_when_hard_cap_boundary_nudges_forward() {
1195 let text = "\
1196aaaaaaaaaa = 1;
1197bbbbbbbbbb = 2;
1198cccccccccc = 3;
1199dddddddddd = 4;
1200eeeeeeeeee = 5;
1201ffffffffff = 6;
1202gggggggggg = 7;
1203hhhhhhhhhh = 8;
1204 };
1205 };
1206
1207 grafanaDashboards = {
1208 cluster-overview.spec = {
1209 inherit instanceSelector;
1210 folderRef = \"infrastructure\";
1211 json = builtins.readFile ./grafana/dashboards/cluster-overview.json;
1212 };
1213 };
1214";
1215 let offsets = compute_marker_offsets(text);
1216
1217 assert_eq!(offsets.first().copied(), Some(0), "offsets: {offsets:?}");
1218 assert_eq!(
1219 offsets.last().copied(),
1220 Some(text.len()),
1221 "offsets: {offsets:?}"
1222 );
1223 assert!(offsets.is_sorted(), "offsets must be sorted: {offsets:?}");
1224 }
1225
1226 #[test]
1227 fn test_compute_marker_offsets_empty() {
1228 let offsets = compute_marker_offsets("");
1229 assert_eq!(offsets, vec![0, 0]);
1230 }
1231
1232 #[test]
1233 fn test_compute_v0327_editable_range_trims_to_marker_boundaries() {
1234 let text = (0..80).map(|_| "x\n").collect::<String>();
1235 let cursor_offset = text.find("x\nx\nx\nx\nx\n").expect("cursor anchor exists") + 40;
1236
1237 let candidate_range = grow_v0327_candidate_range(&text, cursor_offset, 20);
1238 let editable_range = compute_v0327_editable_range(&text, cursor_offset, 20);
1239 let marker_offsets = compute_marker_offsets_v0318(&text[candidate_range.clone()]);
1240 let relative_start = editable_range.start - candidate_range.start;
1241 let relative_end = editable_range.end - candidate_range.start;
1242
1243 assert!(
1244 marker_offsets.len() > 2,
1245 "expected interior markers: {marker_offsets:?}"
1246 );
1247 assert!(marker_offsets.contains(&relative_start));
1248 assert!(marker_offsets.contains(&relative_end));
1249 assert!(editable_range.start <= cursor_offset);
1250 assert!(editable_range.end >= cursor_offset);
1251 assert!(
1252 editable_range.start > candidate_range.start
1253 || editable_range.end < candidate_range.end,
1254 "expected at least one side to trim from {candidate_range:?} down to {editable_range:?}"
1255 );
1256 }
1257
1258 #[test]
1259 fn test_compute_marker_offsets_avoid_short_markdown_blocks() {
1260 let text = "\
1261# Spree Posts
1262
1263This is a Posts extension for [Spree Commerce](https://spreecommerce.org), built with Ruby on Rails.
1264
1265## Installation
1266
12671. Add this extension to your Gemfile with this line:
1268
1269 ```ruby
1270 bundle add spree_posts
1271 ```
1272
12732. Run the install generator
1274
1275 ```ruby
1276 bundle exec rails g spree_posts:install
1277 ```
1278
12793. Restart your server
1280
1281 If your server was running, restart it so that it can find the assets properly.
1282
1283## Developing
1284
12851. Create a dummy app
1286
1287 ```bash
1288 bundle update
1289 bundle exec rake test_app
1290 ```
1291
12922. Add your new code
12933. Run tests
1294
1295 ```bash
1296 bundle exec rspec
1297 ```
1298
1299When testing your applications integration with this extension you may use it's factories.
1300Simply add this require statement to your spec_helper:
1301
1302```ruby
1303require 'spree_posts/factories'
1304```
1305
1306## Releasing a new version
1307
1308```shell
1309bundle exec gem bump -p -t
1310bundle exec gem release
1311```
1312
1313For more options please see [gem-release README](https://github.com/svenfuchs/gem-release)
1314
1315## Contributing
1316
1317If you'd like to contribute, please take a look at the contributing guide.
1318";
1319 let offsets = compute_marker_offsets(text);
1320
1321 assert_eq!(offsets.first().copied(), Some(0), "offsets: {offsets:?}");
1322 assert_eq!(
1323 offsets.last().copied(),
1324 Some(text.len()),
1325 "offsets: {offsets:?}"
1326 );
1327
1328 for window in offsets.windows(2) {
1329 let block = &text[window[0]..window[1]];
1330 let line_count = block.lines().count();
1331 assert!(
1332 line_count >= V0316_MIN_BLOCK_LINES,
1333 "block too short: {line_count} lines in block {block:?} with offsets {offsets:?}"
1334 );
1335 }
1336 }
1337
1338 #[test]
1339 fn test_extract_marker_span() {
1340 let text = "<|marker_2|>\n new content\n<|marker_3|>\n";
1341 let (start, end, content) = extract_marker_span(text).unwrap();
1342 assert_eq!(start, 2);
1343 assert_eq!(end, 3);
1344 assert_eq!(content, " new content\n");
1345 }
1346
1347 #[test]
1348 fn test_extract_marker_span_multi_line() {
1349 let text = "<|marker_1|>\nline1\nline2\nline3\n<|marker_4|>";
1350 let (start, end, content) = extract_marker_span(text).unwrap();
1351 assert_eq!(start, 1);
1352 assert_eq!(end, 4);
1353 assert_eq!(content, "line1\nline2\nline3\n");
1354 }
1355
1356 #[test]
1357 fn test_apply_marker_span_basic() {
1358 let old = "aaa\nbbb\nccc\n";
1359 let output = "<|marker_1|>\naaa\nBBB\nccc\n<|marker_2|>";
1360 let result = apply_marker_span(old, output).unwrap();
1361 assert_eq!(result, "aaa\nBBB\nccc\n");
1362 }
1363
1364 #[test]
1365 fn test_apply_marker_span_preserves_trailing_blank_line() {
1366 let old = "/\nresult\n\n";
1367 let output = "<|marker_1|>\n//\nresult\n\n<|marker_2|>";
1368 let result = apply_marker_span(old, output).unwrap();
1369 assert_eq!(result, "//\nresult\n\n");
1370 }
1371
1372 #[test]
1373 fn test_encode_no_edits() {
1374 let old = "aaa\nbbb\nccc\n";
1375 let result = encode_from_old_and_new(
1376 old,
1377 old,
1378 None,
1379 "<|user_cursor|>",
1380 ">>>>>>> UPDATED\n",
1381 "NO_EDITS\n",
1382 )
1383 .unwrap();
1384 assert_eq!(result, "NO_EDITS\n>>>>>>> UPDATED\n");
1385 }
1386
1387 #[test]
1388 fn test_encode_with_change() {
1389 let old = "aaa\nbbb\nccc\n";
1390 let new = "aaa\nBBB\nccc\n";
1391 let result = encode_from_old_and_new(
1392 old,
1393 new,
1394 None,
1395 "<|user_cursor|>",
1396 ">>>>>>> UPDATED\n",
1397 "NO_EDITS\n",
1398 )
1399 .unwrap();
1400 assert!(result.contains("<|marker_1|>"));
1401 assert!(result.contains("<|marker_2|>"));
1402 assert!(result.contains("aaa\nBBB\nccc\n"));
1403 assert!(result.ends_with(">>>>>>> UPDATED\n"));
1404 }
1405
1406 #[test]
1407 fn test_roundtrip_encode_apply() {
1408 let old = "line1\nline2\nline3\n\nline5\nline6\nline7\nline8\nline9\nline10\n";
1409 let new = "line1\nline2\nline3\n\nline5\nLINE6\nline7\nline8\nline9\nline10\n";
1410 let encoded = encode_from_old_and_new(
1411 old,
1412 new,
1413 None,
1414 "<|user_cursor|>",
1415 ">>>>>>> UPDATED\n",
1416 "NO_EDITS\n",
1417 )
1418 .unwrap();
1419 let output = encoded
1420 .strip_suffix(">>>>>>> UPDATED\n")
1421 .expect("should have end marker");
1422 let reconstructed = apply_marker_span(old, output).unwrap();
1423 assert_eq!(reconstructed, new);
1424 }
1425
1426 #[test]
1427 fn test_extract_editable_region_from_markers_multi() {
1428 let text = "prefix\n<|marker_1|>\naaa\nbbb\n<|marker_2|>\nccc\nddd\n<|marker_3|>\nsuffix";
1429 let parsed = extract_editable_region_from_markers(text).unwrap();
1430 assert_eq!(parsed, "aaa\nbbb\nccc\nddd");
1431 }
1432
1433 #[test]
1434 fn test_extract_editable_region_two_markers() {
1435 let text = "<|marker_1|>\none\ntwo three\n<|marker_2|>";
1436 let parsed = extract_editable_region_from_markers(text).unwrap();
1437 assert_eq!(parsed, "one\ntwo three");
1438 }
1439
1440 #[test]
1441 fn test_encode_with_cursor() {
1442 let old = "aaa\nbbb\nccc\n";
1443 let new = "aaa\nBBB\nccc\n";
1444 let result = encode_from_old_and_new(
1445 old,
1446 new,
1447 Some(5),
1448 "<|user_cursor|>",
1449 ">>>>>>> UPDATED\n",
1450 "NO_EDITS\n",
1451 )
1452 .unwrap();
1453 assert!(result.contains("<|user_cursor|>"), "result: {result}");
1454 assert!(result.contains("B<|user_cursor|>BB"), "result: {result}");
1455 }
1456
1457 #[test]
1458 fn test_extract_marker_span_strips_intermediate_markers() {
1459 let text = "<|marker_2|>\nline1\n<|marker_3|>\nline2\n<|marker_4|>";
1460 let (start, end, content) = extract_marker_span(text).unwrap();
1461 assert_eq!(start, 2);
1462 assert_eq!(end, 4);
1463 assert_eq!(content, "line1\nline2\n");
1464 }
1465
1466 #[test]
1467 fn test_extract_marker_span_strips_multiple_intermediate_markers() {
1468 let text = "<|marker_1|>\naaa\n<|marker_2|>\nbbb\n<|marker_3|>\nccc\n<|marker_4|>";
1469 let (start, end, content) = extract_marker_span(text).unwrap();
1470 assert_eq!(start, 1);
1471 assert_eq!(end, 4);
1472 assert_eq!(content, "aaa\nbbb\nccc\n");
1473 }
1474
1475 #[test]
1476 fn test_apply_marker_span_with_extra_intermediate_marker() {
1477 let old = "aaa\nbbb\nccc\n";
1478 let output = "<|marker_1|>\naaa\n<|marker_1|>\nBBB\nccc\n<|marker_2|>";
1479 let result = apply_marker_span(old, output).unwrap();
1480 assert_eq!(result, "aaa\nBBB\nccc\n");
1481 }
1482
1483 #[test]
1484 fn test_strip_marker_tags_inline() {
1485 assert_eq!(strip_marker_tags("no markers here"), "no markers here");
1486 assert_eq!(strip_marker_tags("before<|marker_5|>after"), "beforeafter");
1487 assert_eq!(
1488 strip_marker_tags("line1\n<|marker_3|>\nline2"),
1489 "line1\nline2"
1490 );
1491 }
1492
1493 #[test]
1494 fn test_write_editable_with_markers_v0316_byte_exact() {
1495 let editable = "aaa\nbbb\nccc\n";
1496 let mut output = String::new();
1497 write_editable_with_markers_v0316(&mut output, editable, 4, "<|user_cursor|>");
1498 assert!(output.starts_with("<|marker_1|>"));
1499 assert!(output.contains("<|user_cursor|>"));
1500 let stripped = output.replace("<|user_cursor|>", "");
1501 let stripped = strip_marker_tags(&stripped);
1502 assert_eq!(stripped, editable);
1503 }
1504
1505 #[test]
1506 fn test_apply_marker_span_v0316_basic() {
1507 let old = "aaa\nbbb\nccc\n";
1508 let output = "<|marker_1|>aaa\nBBB\nccc\n<|marker_2|>";
1509 let result = apply_marker_span_v0316(old, output).unwrap();
1510 assert_eq!(result, "aaa\nBBB\nccc\n");
1511 }
1512
1513 #[test]
1514 fn test_apply_marker_span_v0316_no_edit() {
1515 let old = "aaa\nbbb\nccc\n";
1516 let output = "<|marker_1|><|marker_1|>";
1517 let result = apply_marker_span_v0316(old, output).unwrap();
1518 assert_eq!(result, old);
1519 }
1520
1521 #[test]
1522 fn test_apply_marker_span_v0316_no_edit_any_marker() {
1523 let old = "aaa\nbbb\nccc\n";
1524 let output = "<|marker_2|>ignored content<|marker_2|>";
1525 let result = apply_marker_span_v0316(old, output).unwrap();
1526 assert_eq!(result, old);
1527 }
1528
1529 #[test]
1530 fn test_apply_marker_span_v0316_multi_block() {
1531 let old = "line1\nline2\nline3\n\nline5\nline6\nline7\nline8\n";
1532 let marker_offsets = compute_marker_offsets(old);
1533 assert!(
1534 marker_offsets.len() >= 3,
1535 "expected at least 3 offsets, got {:?}",
1536 marker_offsets
1537 );
1538
1539 let new_content = "LINE1\nLINE2\nLINE3\n\nLINE5\nLINE6\nLINE7\nLINE8\n";
1540 let mut output = String::new();
1541 output.push_str("<|marker_1|>");
1542 for i in 0..marker_offsets.len() - 1 {
1543 if i > 0 {
1544 output.push_str(&marker_tag(i + 1));
1545 }
1546 let start = marker_offsets[i];
1547 let end = marker_offsets[i + 1];
1548 let block_len = end - start;
1549 output.push_str(&new_content[start..start + block_len]);
1550 }
1551 let last_marker_num = marker_offsets.len();
1552 output.push_str(&marker_tag(last_marker_num));
1553 let result = apply_marker_span_v0316(old, &output).unwrap();
1554 assert_eq!(result, new_content);
1555 }
1556
1557 #[test]
1558 fn test_apply_marker_span_v0316_byte_exact_no_normalization() {
1559 let old = "aaa\nbbb\nccc\n";
1560 let output = "<|marker_1|>aaa\nBBB\nccc<|marker_2|>";
1561 let result = apply_marker_span_v0316(old, output).unwrap();
1562 assert_eq!(result, "aaa\nBBB\nccc");
1563 }
1564
1565 #[test]
1566 fn test_encode_v0316_no_edits() {
1567 let old = "aaa\nbbb\nccc\n";
1568 let result =
1569 encode_from_old_and_new_v0316(old, old, Some(5), "<|user_cursor|>", "<|end|>").unwrap();
1570 assert!(result.ends_with("<|end|>"));
1571 let stripped = result.strip_suffix("<|end|>").unwrap();
1572 let result_parsed = apply_marker_span_v0316(old, stripped).unwrap();
1573 assert_eq!(result_parsed, old);
1574 }
1575
1576 #[test]
1577 fn test_encode_v0316_with_change() {
1578 let old = "aaa\nbbb\nccc\n";
1579 let new = "aaa\nBBB\nccc\n";
1580 let result =
1581 encode_from_old_and_new_v0316(old, new, None, "<|user_cursor|>", "<|end|>").unwrap();
1582 assert!(result.contains("<|marker_1|>"));
1583 assert!(result.contains("<|marker_2|>"));
1584 assert!(result.ends_with("<|end|>"));
1585 }
1586
1587 #[test]
1588 fn test_roundtrip_v0316() {
1589 let old = "line1\nline2\nline3\n\nline5\nline6\nline7\nline8\nline9\nline10\n";
1590 let new = "line1\nline2\nline3\n\nline5\nLINE6\nline7\nline8\nline9\nline10\n";
1591 let encoded =
1592 encode_from_old_and_new_v0316(old, new, None, "<|user_cursor|>", "<|end|>").unwrap();
1593 let stripped = encoded
1594 .strip_suffix("<|end|>")
1595 .expect("should have end marker");
1596 let reconstructed = apply_marker_span_v0316(old, stripped).unwrap();
1597 assert_eq!(reconstructed, new);
1598 }
1599
1600 #[test]
1601 fn test_roundtrip_v0316_with_cursor() {
1602 let old = "aaa\nbbb\nccc\n";
1603 let new = "aaa\nBBB\nccc\n";
1604 let result =
1605 encode_from_old_and_new_v0316(old, new, Some(5), "<|user_cursor|>", "<|end|>").unwrap();
1606 assert!(result.contains("<|user_cursor|>"), "result: {result}");
1607 assert!(result.contains("B<|user_cursor|>BB"), "result: {result}");
1608 }
1609
1610 #[test]
1611 fn test_roundtrip_v0316_multi_block_change() {
1612 let old = "line1\nline2\nline3\n\nline5\nline6\nline7\nline8\n";
1613 let new = "line1\nLINE2\nline3\n\nline5\nLINE6\nline7\nline8\n";
1614 let encoded =
1615 encode_from_old_and_new_v0316(old, new, None, "<|user_cursor|>", "<|end|>").unwrap();
1616 let stripped = encoded
1617 .strip_suffix("<|end|>")
1618 .expect("should have end marker");
1619 let reconstructed = apply_marker_span_v0316(old, stripped).unwrap();
1620 assert_eq!(reconstructed, new);
1621 }
1622
1623 #[test]
1624 fn test_nearest_marker_number() {
1625 let offsets = vec![0, 10, 20, 30];
1626 assert_eq!(nearest_marker_number(Some(0), &offsets), 1);
1627 assert_eq!(nearest_marker_number(Some(9), &offsets), 2);
1628 assert_eq!(nearest_marker_number(Some(15), &offsets), 2);
1629 assert_eq!(nearest_marker_number(Some(25), &offsets), 3);
1630 assert_eq!(nearest_marker_number(Some(30), &offsets), 4);
1631 assert_eq!(nearest_marker_number(None, &offsets), 1);
1632 }
1633
1634 #[test]
1635 fn test_marker_tag_relative_formats_as_expected() {
1636 assert_eq!(marker_tag_relative(-2), "<|marker-2|>");
1637 assert_eq!(marker_tag_relative(-1), "<|marker-1|>");
1638 assert_eq!(marker_tag_relative(0), "<|marker-0|>");
1639 assert_eq!(marker_tag_relative(1), "<|marker+1|>");
1640 assert_eq!(marker_tag_relative(2), "<|marker+2|>");
1641 }
1642
1643 #[test]
1644 fn test_write_editable_with_markers_v0317_includes_relative_markers_and_cursor() {
1645 let editable = "aaa\nbbb\nccc\n";
1646 let mut output = String::new();
1647 write_editable_with_markers_v0317(&mut output, editable, 4, "<|user_cursor|>");
1648
1649 assert!(output.contains("<|marker-0|>"));
1650 assert!(output.contains("<|user_cursor|>"));
1651
1652 let stripped = output.replace("<|user_cursor|>", "");
1653 let stripped =
1654 collect_relative_marker_tags(&stripped)
1655 .iter()
1656 .fold(stripped.clone(), |acc, marker| {
1657 let tag = &stripped[marker.tag_start..marker.tag_end];
1658 acc.replace(tag, "")
1659 });
1660 assert_eq!(stripped, editable);
1661 }
1662
1663 #[test]
1664 fn test_apply_marker_span_v0317_basic() {
1665 let old = "aaa\nbbb\nccc\n";
1666 let output = "<|marker-0|>aaa\nBBB\nccc\n<|marker+1|>";
1667 let result = apply_marker_span_v0317(old, output, Some(0)).unwrap();
1668 assert_eq!(result, "aaa\nBBB\nccc\n");
1669 }
1670
1671 #[test]
1672 fn test_apply_marker_span_v0317_no_edit() {
1673 let old = "aaa\nbbb\nccc\n";
1674 let output = "<|marker-0|><|marker-0|>";
1675 let result = apply_marker_span_v0317(old, output, Some(0)).unwrap();
1676 assert_eq!(result, old);
1677 }
1678
1679 #[test]
1680 fn test_encode_v0317_no_edits() {
1681 let old = "aaa\nbbb\nccc\n";
1682 let result =
1683 encode_from_old_and_new_v0317(old, old, Some(5), "<|user_cursor|>", "<|end|>").unwrap();
1684 assert_eq!(result, "<|marker-0|><|marker-0|><|end|>");
1685 }
1686
1687 #[test]
1688 fn test_roundtrip_v0317() {
1689 let old = "line1\nline2\nline3\n\nline5\nline6\nline7\nline8\n";
1690 let new = "line1\nLINE2\nline3\n\nline5\nLINE6\nline7\nline8\n";
1691 let cursor = Some(6);
1692
1693 let encoded =
1694 encode_from_old_and_new_v0317(old, new, cursor, "<|user_cursor|>", "<|end|>").unwrap();
1695 let stripped = encoded
1696 .strip_suffix("<|end|>")
1697 .expect("should have end marker");
1698 let stripped = stripped.replace("<|user_cursor|>", "");
1699 let reconstructed = apply_marker_span_v0317(old, &stripped, cursor).unwrap();
1700 assert_eq!(reconstructed, new);
1701 }
1702
1703 #[test]
1704 fn test_roundtrip_v0317_with_cursor_marker() {
1705 let old = "aaa\nbbb\nccc\n";
1706 let new = "aaa\nBBB\nccc\n";
1707 let result =
1708 encode_from_old_and_new_v0317(old, new, Some(5), "<|user_cursor|>", "<|end|>").unwrap();
1709 assert!(result.contains("<|user_cursor|>"), "result: {result}");
1710 assert!(result.contains("<|marker-0|>"), "result: {result}");
1711 }
1712
1713 #[test]
1714 fn test_compute_marker_offsets_v0318_uses_larger_block_sizes() {
1715 let text = "l1\nl2\nl3\n\nl5\nl6\nl7\nl8\nl9\nl10\nl11\nl12\nl13\n";
1716 let v0316_offsets = compute_marker_offsets(text);
1717 let v0318_offsets = compute_marker_offsets_v0318(text);
1718
1719 assert!(v0318_offsets.len() < v0316_offsets.len());
1720 assert_eq!(v0316_offsets.first().copied(), Some(0));
1721 assert_eq!(v0318_offsets.first().copied(), Some(0));
1722 assert_eq!(v0316_offsets.last().copied(), Some(text.len()));
1723 assert_eq!(v0318_offsets.last().copied(), Some(text.len()));
1724 }
1725
1726 #[test]
1727 fn test_roundtrip_v0318() {
1728 let old = "line1\nline2\nline3\n\nline5\nline6\nline7\nline8\nline9\nline10\n";
1729 let new = "line1\nline2\nline3\n\nline5\nLINE6\nline7\nline8\nline9\nline10\n";
1730 let encoded =
1731 encode_from_old_and_new_v0318(old, new, None, "<|user_cursor|>", "<|end|>").unwrap();
1732 let stripped = encoded
1733 .strip_suffix("<|end|>")
1734 .expect("should have end marker");
1735 let reconstructed = apply_marker_span_v0318(old, stripped).unwrap();
1736 assert_eq!(reconstructed, new);
1737 }
1738
1739 #[test]
1740 fn test_roundtrip_v0318_append_at_end_of_editable_region() {
1741 let old = "line1\nline2\nline3\n";
1742 let new = "line1\nline2\nline3\nline4\n";
1743 let encoded =
1744 encode_from_old_and_new_v0318(old, new, None, "<|user_cursor|>", "<|end|>").unwrap();
1745
1746 assert_ne!(encoded, "<|marker_2|><|end|>");
1747
1748 let stripped = encoded
1749 .strip_suffix("<|end|>")
1750 .expect("should have end marker");
1751 let reconstructed = apply_marker_span_v0318(old, stripped).unwrap();
1752 assert_eq!(reconstructed, new);
1753 }
1754
1755 #[test]
1756 fn test_roundtrip_v0318_insert_at_internal_marker_boundary() {
1757 let old = "alpha\nbeta\n\ngamma\ndelta\n";
1758 let new = "alpha\nbeta\n\ninserted\ngamma\ndelta\n";
1759 let encoded =
1760 encode_from_old_and_new_v0318(old, new, None, "<|user_cursor|>", "<|end|>").unwrap();
1761
1762 let stripped = encoded
1763 .strip_suffix("<|end|>")
1764 .expect("should have end marker");
1765 let reconstructed = apply_marker_span_v0318(old, stripped).unwrap();
1766 assert_eq!(reconstructed, new);
1767 }
1768
1769 #[test]
1770 fn test_encode_v0317_markers_stay_on_line_boundaries() {
1771 let old = "\
1772\t\t\t\tcontinue outer;
1773\t\t\t}
1774\t\t}
1775\t}
1776
1777\tconst intersectionObserver = new IntersectionObserver((entries) => {
1778\t\tfor (const entry of entries) {
1779\t\t\tif (entry.isIntersecting) {
1780\t\t\t\tintersectionObserver.unobserve(entry.target);
1781\t\t\t\tanchorPreload(/** @type {HTMLAnchorElement} */ (entry.target));
1782\t\t\t}
1783\t\t}
1784\t});
1785
1786\tconst observer = new MutationObserver(() => {
1787\t\tconst links = /** @type {NodeListOf<HTMLAnchorElement>} */ (
1788\t\t\tdocument.querySelectorAll('a[data-preload]')
1789\t\t);
1790
1791\t\tfor (const link of links) {
1792\t\t\tif (linkSet.has(link)) continue;
1793\t\t\tlinkSet.add(link);
1794
1795\t\t\tswitch (link.dataset.preload) {
1796\t\t\t\tcase '':
1797\t\t\t\tcase 'true':
1798\t\t\t\tcase 'hover': {
1799\t\t\t\t\tlink.addEventListener('mouseenter', function callback() {
1800\t\t\t\t\t\tlink.removeEventListener('mouseenter', callback);
1801\t\t\t\t\t\tanchorPreload(link);
1802\t\t\t\t\t});
1803";
1804 let new = old.replacen(
1805 "\t\t\t\tcase 'true':\n",
1806 "\t\t\t\tcase 'TRUE':<|user_cursor|>\n",
1807 1,
1808 );
1809
1810 let cursor_offset = new.find("<|user_cursor|>").expect("cursor marker in new");
1811 let new_without_cursor = new.replace("<|user_cursor|>", "");
1812
1813 let encoded = encode_from_old_and_new_v0317(
1814 old,
1815 &new_without_cursor,
1816 Some(cursor_offset),
1817 "<|user_cursor|>",
1818 "<|end|>",
1819 )
1820 .unwrap();
1821
1822 let core = encoded.strip_suffix("<|end|>").unwrap_or(&encoded);
1823 for marker in collect_relative_marker_tags(core) {
1824 let tag_start = marker.tag_start;
1825 assert!(
1826 tag_start == 0 || core.as_bytes()[tag_start - 1] == b'\n',
1827 "marker not at line boundary: {} in output:\n{}",
1828 marker_tag_relative(marker.value),
1829 core
1830 );
1831 }
1832 }
1833}
1834