tenant.openagents/omega

1use serde::{Deserialize, Serialize};
2use std::collections::{BTreeMap, BTreeSet};
3use std::error::Error;
4use std::fmt;
5use std::path::Path;
6use std::sync::Arc;
7use zeta_prompt::udiff::{apply_diff_to_string, apply_diff_to_string_with_hunk_offset};
8
9use crate::reversal::compute_prediction_reversal_ratio_from_history;
10use crate::{
11    jumps::{
12        EditableContextCoverage, Excerpt, PatchLocationMatch, editable_context_coverage,
13        patch_location_match,
14    },
15    patch::{Hunk, Patch, PatchLine},
16    patch_metrics::{
17        ClassificationMetrics, DeltaChrFMetrics, braces_disbalance, count_patch_token_changes,
18        delta_chr_f, delta_chr_f_beta, exact_lines_match, has_isolated_whitespace_changes,
19        is_editable_region_correct,
20    },
21};
22
23#[derive(Clone, Debug, Serialize, Deserialize)]
24pub struct PredictionScore {
25    pub delta_chr_f: f32,
26    #[serde(default)]
27    pub delta_chr_f_true_positives: usize,
28    #[serde(default)]
29    pub delta_chr_f_false_positives: usize,
30    #[serde(default)]
31    pub delta_chr_f_false_negatives: usize,
32    #[serde(default)]
33    pub delta_chr_f_precision: f64,
34    #[serde(default)]
35    pub delta_chr_f_recall: f64,
36    #[serde(default)]
37    pub delta_chr_f_beta: f64,
38    pub braces_disbalance: usize,
39    #[serde(default)]
40    pub exact_lines_tp: usize,
41    #[serde(default)]
42    pub exact_lines_fp: usize,
43    #[serde(default)]
44    pub exact_lines_fn: usize,
45    #[serde(default)]
46    pub reversal_ratio: f32,
47    #[serde(default, skip_serializing_if = "Option::is_none")]
48    pub cursor_distance: Option<usize>,
49    #[serde(default, skip_serializing_if = "Option::is_none")]
50    pub cursor_exact_match: Option<bool>,
51    pub wrong_editable_region: Option<bool>,
52    #[serde(default)]
53    pub has_isolated_whitespace_changes: bool,
54    #[serde(default)]
55    pub inserted_tokens: usize,
56    #[serde(default)]
57    pub deleted_tokens: usize,
58    #[serde(default, skip_serializing_if = "Option::is_none")]
59    pub kept_rate: Option<f64>,
60    #[serde(default, skip_serializing_if = "Option::is_none")]
61    pub recall_rate: Option<f64>,
62    #[serde(default, skip_serializing_if = "Option::is_none")]
63    pub kept_chars: Option<usize>,
64    #[serde(default, skip_serializing_if = "Option::is_none")]
65    pub correctly_deleted_chars: Option<usize>,
66    #[serde(default, skip_serializing_if = "Option::is_none")]
67    pub discarded_chars: Option<usize>,
68    #[serde(default, skip_serializing_if = "Option::is_none")]
69    pub cumulative_logprob: Option<f64>,
70    #[serde(default, skip_serializing_if = "Option::is_none")]
71    pub avg_logprob: Option<f64>,
72    #[serde(default, skip_serializing_if = "Option::is_none")]
73    pub editable_context_coverage: Option<EditableContextCoverage>,
74    #[serde(default, skip_serializing_if = "Option::is_none")]
75    pub jump_location: Option<PatchLocationMatch>,
76}
77
78impl PredictionScore {
79    pub fn zero() -> Self {
80        Self {
81            delta_chr_f: 0.0,
82            delta_chr_f_true_positives: 0,
83            delta_chr_f_false_positives: 0,
84            delta_chr_f_false_negatives: 0,
85            delta_chr_f_precision: 0.0,
86            delta_chr_f_recall: 0.0,
87            delta_chr_f_beta: delta_chr_f_beta(),
88            braces_disbalance: 0,
89            exact_lines_tp: 0,
90            exact_lines_fp: 0,
91            exact_lines_fn: 0,
92            reversal_ratio: 0.0,
93            cursor_distance: None,
94            cursor_exact_match: None,
95            wrong_editable_region: None,
96            has_isolated_whitespace_changes: false,
97            inserted_tokens: 0,
98            deleted_tokens: 0,
99            kept_rate: None,
100            recall_rate: None,
101            kept_chars: None,
102            correctly_deleted_chars: None,
103            discarded_chars: None,
104            cumulative_logprob: None,
105            avg_logprob: None,
106            editable_context_coverage: None,
107            jump_location: None,
108        }
109    }
110
111    pub fn delta_chr_f_counts(&self) -> ClassificationMetrics {
112        ClassificationMetrics {
113            true_positives: self.delta_chr_f_true_positives,
114            false_positives: self.delta_chr_f_false_positives,
115            false_negatives: self.delta_chr_f_false_negatives,
116        }
117    }
118
119    pub fn exact_lines_counts(&self) -> ClassificationMetrics {
120        ClassificationMetrics {
121            true_positives: self.exact_lines_tp,
122            false_positives: self.exact_lines_fp,
123            false_negatives: self.exact_lines_fn,
124        }
125    }
126}
127
128impl Default for PredictionScore {
129    fn default() -> Self {
130        Self::zero()
131    }
132}
133
134#[derive(Clone, Debug)]
135pub struct PreparedExpectedPatch {
136    pub patch: String,
137    pub text: String,
138    pub cursor_editable_region_offset: Option<usize>,
139}
140
141#[derive(Clone, Debug)]
142pub struct PrepareExpectedPatchError {
143    message: String,
144}
145
146impl fmt::Display for PrepareExpectedPatchError {
147    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
148        self.message.fmt(formatter)
149    }
150}
151
152impl Error for PrepareExpectedPatchError {}
153
154pub fn prepare_expected_patches(
155    expected_patches_with_cursors: &[(String, Option<usize>)],
156    original_text: &str,
157    old_editable_region: Option<&str>,
158) -> Result<Vec<PreparedExpectedPatch>, PrepareExpectedPatchError> {
159    expected_patches_with_cursors
160        .iter()
161        .map(|(patch, cursor_in_patch)| {
162            let text = apply_diff_to_string(patch, original_text).map_err(|error| {
163                PrepareExpectedPatchError {
164                    message: error.to_string(),
165                }
166            })?;
167            let cursor_editable_region_offset =
168                if let (Some(editable_region), Some(cursor_in_patch)) =
169                    (old_editable_region, *cursor_in_patch)
170                {
171                    match apply_diff_to_string_with_hunk_offset(patch, editable_region) {
172                        Ok((_, hunk_offset)) => Some(hunk_offset.unwrap_or(0) + cursor_in_patch),
173                        Err(_) => None,
174                    }
175                } else {
176                    *cursor_in_patch
177                };
178
179            Ok(PreparedExpectedPatch {
180                patch: patch.clone(),
181                text,
182                cursor_editable_region_offset,
183            })
184        })
185        .collect()
186}
187
188#[derive(Clone, Copy, Debug)]
189pub struct ActualPredictionCursor {
190    pub row: u32,
191    pub editable_region_offset: Option<usize>,
192}
193
194#[derive(Clone, Copy, Debug)]
195pub struct PredictionReversalContext<'a> {
196    pub edit_history: &'a [Arc<zeta_prompt::Event>],
197    pub excerpt_start_row: Option<u32>,
198    pub cursor_path: &'a Path,
199}
200
201#[derive(Clone, Copy, Debug)]
202pub struct PredictionScoringInput<'a> {
203    pub original_text: &'a str,
204    pub expected_patches: &'a [PreparedExpectedPatch],
205    pub actual_patch: Option<&'a str>,
206    pub actual_cursor: Option<ActualPredictionCursor>,
207    pub reversal_context: Option<PredictionReversalContext<'a>>,
208    pub cumulative_logprob: Option<f64>,
209    pub avg_logprob: Option<f64>,
210    pub context: Option<&'a [Excerpt]>,
211}
212
213pub fn score_prediction(input: PredictionScoringInput<'_>) -> PredictionScore {
214    let editable_context_coverage = input.context.and_then(|context| {
215        input
216            .expected_patches
217            .iter()
218            .map(|expected| editable_context_coverage(&expected.patch, context))
219            .max_by(|left, right| {
220                left.lines_f1
221                    .total_cmp(&right.lines_f1)
222                    .then_with(|| left.files_f1.total_cmp(&right.files_f1))
223            })
224    });
225
226    let actual_patch = input.actual_patch.unwrap_or("");
227    let token_changes = count_patch_token_changes(actual_patch);
228
229    let mut best = input
230        .expected_patches
231        .iter()
232        .map(|expected| score_against_expected_patch(input, expected, actual_patch))
233        .max_by(|left, right| {
234            left.delta_chr_f_metrics
235                .score
236                .total_cmp(&right.delta_chr_f_metrics.score)
237                .then_with(|| left.exact_lines.f1().total_cmp(&right.exact_lines.f1()))
238                .then_with(|| {
239                    left.jump_location
240                        .lines_f1
241                        .total_cmp(&right.jump_location.lines_f1)
242                })
243        })
244        .unwrap_or_else(|| score_against_no_expected_patch(input, actual_patch));
245
246    let (cursor_distance, cursor_exact_match) =
247        compute_cursor_metrics(best.expected_cursor, input.actual_cursor);
248
249    let wrong_editable_region = input
250        .actual_patch
251        .map(|actual_patch| !is_editable_region_correct(actual_patch));
252    let has_isolated_whitespace_changes = input.actual_patch.is_some_and(|actual_patch| {
253        has_isolated_whitespace_changes(actual_patch, input.actual_cursor.map(|cursor| cursor.row))
254    });
255
256    best.score.cumulative_logprob = input.cumulative_logprob;
257    best.score.avg_logprob = input.avg_logprob;
258    best.score.editable_context_coverage = editable_context_coverage;
259    best.score.inserted_tokens = token_changes.inserted_tokens;
260    best.score.deleted_tokens = token_changes.deleted_tokens;
261    best.score.cursor_distance = cursor_distance;
262    best.score.cursor_exact_match = cursor_exact_match;
263    best.score.wrong_editable_region = wrong_editable_region;
264    best.score.has_isolated_whitespace_changes = has_isolated_whitespace_changes;
265    best.score
266}
267
268struct ExpectedPatchScore {
269    score: PredictionScore,
270    delta_chr_f_metrics: DeltaChrFMetrics,
271    exact_lines: ClassificationMetrics,
272    jump_location: PatchLocationMatch,
273    expected_cursor: Option<usize>,
274}
275
276struct ContentScore {
277    delta_chr_f_metrics: DeltaChrFMetrics,
278    braces_disbalance: usize,
279    reversal_ratio: f32,
280    kept_rate: Option<f64>,
281    recall_rate: Option<f64>,
282    kept_chars: Option<usize>,
283    correctly_deleted_chars: Option<usize>,
284    discarded_chars: Option<usize>,
285}
286
287fn score_against_expected_patch(
288    input: PredictionScoringInput<'_>,
289    expected: &PreparedExpectedPatch,
290    actual_patch: &str,
291) -> ExpectedPatchScore {
292    let exact_lines = exact_lines_match(&expected.patch, actual_patch);
293    let jump_location = patch_location_match(&expected.patch, actual_patch);
294    let content_score = if let Some(context) = input.context {
295        score_content_on_context(input, expected, actual_patch, context).unwrap_or_else(|| {
296            if expected.patch.trim().is_empty() && actual_patch.trim().is_empty() {
297                score_content_on_cursor_excerpt(input, expected, actual_patch)
298            } else {
299                zero_content_score()
300            }
301        })
302    } else {
303        score_content_on_cursor_excerpt(input, expected, actual_patch)
304    };
305    let delta_chr_f_metrics = content_score.delta_chr_f_metrics.clone();
306
307    let score = PredictionScore {
308        delta_chr_f: delta_chr_f_metrics.score as f32,
309        delta_chr_f_true_positives: delta_chr_f_metrics.counts.true_positives,
310        delta_chr_f_false_positives: delta_chr_f_metrics.counts.false_positives,
311        delta_chr_f_false_negatives: delta_chr_f_metrics.counts.false_negatives,
312        delta_chr_f_precision: delta_chr_f_metrics.precision,
313        delta_chr_f_recall: delta_chr_f_metrics.recall,
314        delta_chr_f_beta: delta_chr_f_metrics.beta,
315        braces_disbalance: content_score.braces_disbalance,
316        exact_lines_tp: exact_lines.true_positives,
317        exact_lines_fp: exact_lines.false_positives,
318        exact_lines_fn: exact_lines.false_negatives,
319        reversal_ratio: content_score.reversal_ratio,
320        cursor_distance: None,
321        cursor_exact_match: None,
322        wrong_editable_region: None,
323        has_isolated_whitespace_changes: false,
324        inserted_tokens: 0,
325        deleted_tokens: 0,
326        kept_rate: content_score.kept_rate,
327        recall_rate: content_score.recall_rate,
328        kept_chars: content_score.kept_chars,
329        correctly_deleted_chars: content_score.correctly_deleted_chars,
330        discarded_chars: content_score.discarded_chars,
331        cumulative_logprob: None,
332        avg_logprob: None,
333        editable_context_coverage: None,
334        jump_location: Some(jump_location.clone()),
335    };
336
337    ExpectedPatchScore {
338        score,
339        delta_chr_f_metrics,
340        exact_lines,
341        jump_location,
342        expected_cursor: expected.cursor_editable_region_offset,
343    }
344}
345
346fn score_against_no_expected_patch(
347    input: PredictionScoringInput<'_>,
348    actual_patch: &str,
349) -> ExpectedPatchScore {
350    let expected = PreparedExpectedPatch {
351        patch: String::new(),
352        text: input.original_text.to_string(),
353        cursor_editable_region_offset: None,
354    };
355    score_against_expected_patch(input, &expected, actual_patch)
356}
357
358fn zero_content_score() -> ContentScore {
359    ContentScore {
360        delta_chr_f_metrics: DeltaChrFMetrics {
361            beta: delta_chr_f_beta(),
362            ..DeltaChrFMetrics::default()
363        },
364        braces_disbalance: 0,
365        reversal_ratio: 0.0,
366        kept_rate: None,
367        recall_rate: None,
368        kept_chars: None,
369        correctly_deleted_chars: None,
370        discarded_chars: None,
371    }
372}
373
374fn score_content_on_cursor_excerpt(
375    input: PredictionScoringInput<'_>,
376    expected: &PreparedExpectedPatch,
377    actual_patch: &str,
378) -> ContentScore {
379    let actual_text = apply_diff_to_string(actual_patch, input.original_text)
380        .unwrap_or_else(|_| input.original_text.to_string());
381    let delta_chr_f_metrics = delta_chr_f(input.original_text, &expected.text, &actual_text);
382    let braces_disbalance =
383        braces_disbalance(&actual_text).saturating_sub(braces_disbalance(input.original_text));
384    let reversal_ratio = input
385        .reversal_context
386        .map(|context| {
387            compute_prediction_reversal_ratio_from_history(
388                input.original_text,
389                context.edit_history,
390                context.excerpt_start_row,
391                &actual_text,
392                context.cursor_path,
393            )
394        })
395        .unwrap_or(0.0);
396    let kept_rate =
397        crate::kept_rate::compute_kept_rate(input.original_text, &actual_text, &expected.text);
398
399    ContentScore {
400        delta_chr_f_metrics,
401        braces_disbalance,
402        reversal_ratio,
403        kept_rate: Some(kept_rate.kept_rate),
404        recall_rate: Some(kept_rate.recall_rate),
405        kept_chars: Some(kept_rate.kept_chars),
406        correctly_deleted_chars: Some(kept_rate.correctly_deleted_chars),
407        discarded_chars: Some(kept_rate.discarded_chars),
408    }
409}
410
411fn score_content_on_context(
412    input: PredictionScoringInput<'_>,
413    expected: &PreparedExpectedPatch,
414    actual_patch: &str,
415    context: &[Excerpt],
416) -> Option<ContentScore> {
417    let expected_documents = apply_patch_to_documents(&expected.patch, context);
418    let actual_documents = apply_patch_to_documents(actual_patch, context);
419    let document_indices = expected_documents
420        .keys()
421        .chain(actual_documents.keys())
422        .copied()
423        .collect::<BTreeSet<_>>();
424
425    if document_indices.is_empty() {
426        return None;
427    }
428
429    let mut original_text = String::new();
430    let mut expected_text = String::new();
431    let mut actual_text = String::new();
432    let mut braces_disbalance_before = 0;
433    let mut braces_disbalance_after = 0;
434    let mut cursor_actual_text = None;
435
436    for document_ix in document_indices {
437        let document = context.get(document_ix)?;
438        let expected_document_text = expected_documents
439            .get(&document_ix)
440            .map(String::as_str)
441            .unwrap_or(document.content.as_str());
442        let actual_document_text = actual_documents
443            .get(&document_ix)
444            .map(String::as_str)
445            .unwrap_or(document.content.as_str());
446
447        if !original_text.is_empty() {
448            original_text.push('\n');
449            expected_text.push('\n');
450            actual_text.push('\n');
451        }
452        original_text.push_str(&document.content);
453        expected_text.push_str(expected_document_text);
454        actual_text.push_str(actual_document_text);
455
456        braces_disbalance_before += braces_disbalance(&document.content);
457        braces_disbalance_after += braces_disbalance(actual_document_text);
458
459        if input.reversal_context.is_some_and(|reversal_context| {
460            path_matches(
461                &reversal_context.cursor_path.to_string_lossy(),
462                &document.path,
463            )
464        }) {
465            cursor_actual_text = Some(actual_document_text.to_string());
466        }
467    }
468
469    let delta_chr_f_metrics = delta_chr_f(&original_text, &expected_text, &actual_text);
470    let kept_rate =
471        crate::kept_rate::compute_kept_rate(&original_text, &actual_text, &expected_text);
472    let reversal_ratio = if let (Some(reversal_context), Some(cursor_actual_text)) =
473        (input.reversal_context, cursor_actual_text.as_deref())
474    {
475        compute_prediction_reversal_ratio_from_history(
476            input.original_text,
477            reversal_context.edit_history,
478            reversal_context.excerpt_start_row,
479            cursor_actual_text,
480            reversal_context.cursor_path,
481        )
482    } else {
483        0.0
484    };
485
486    Some(ContentScore {
487        delta_chr_f_metrics,
488        braces_disbalance: braces_disbalance_after.saturating_sub(braces_disbalance_before),
489        reversal_ratio,
490        kept_rate: Some(kept_rate.kept_rate),
491        recall_rate: Some(kept_rate.recall_rate),
492        kept_chars: Some(kept_rate.kept_chars),
493        correctly_deleted_chars: Some(kept_rate.correctly_deleted_chars),
494        discarded_chars: Some(kept_rate.discarded_chars),
495    })
496}
497
498fn apply_patch_to_documents(patch: &str, context: &[Excerpt]) -> BTreeMap<usize, String> {
499    let patch = Patch::parse_unified_diff(patch);
500    let mut hunks_by_document: BTreeMap<usize, Vec<Hunk>> = BTreeMap::new();
501
502    for hunk in patch.hunks.into_iter().filter(hunk_has_change) {
503        if let Some(document_ix) = find_hunk_document(&hunk, context) {
504            hunks_by_document.entry(document_ix).or_default().push(hunk);
505        }
506    }
507
508    hunks_by_document
509        .into_iter()
510        .filter_map(|(document_ix, hunks)| {
511            let document = context.get(document_ix)?;
512            let document_patch = diff_for_document_hunks(document, &hunks);
513            let text = apply_diff_to_string(&document_patch, &document.content).ok()?;
514            Some((document_ix, text))
515        })
516        .collect()
517}
518
519fn find_hunk_document(hunk: &Hunk, context: &[Excerpt]) -> Option<usize> {
520    context
521        .iter()
522        .enumerate()
523        .find_map(|(document_ix, document)| {
524            if !path_matches(&hunk.filename, &document.path) {
525                return None;
526            }
527
528            let document_patch = diff_for_document_hunks(document, std::slice::from_ref(hunk));
529            apply_diff_to_string(&document_patch, &document.content)
530                .is_ok()
531                .then_some(document_ix)
532        })
533}
534
535fn diff_for_document_hunks(document: &Excerpt, hunks: &[Hunk]) -> String {
536    let mut diff = String::new();
537    diff.push_str(&format!("--- a/{}\n", document.path));
538    diff.push_str(&format!("+++ b/{}\n", document.path));
539
540    for hunk in hunks {
541        let old_start = adjust_hunk_start(hunk.old_start, &document.row_range);
542        let new_start = adjust_hunk_start(hunk.new_start, &document.row_range);
543        let old_count = hunk
544            .lines
545            .iter()
546            .filter(|line| matches!(line, PatchLine::Context(_) | PatchLine::Deletion(_)))
547            .count();
548        let new_count = hunk
549            .lines
550            .iter()
551            .filter(|line| matches!(line, PatchLine::Context(_) | PatchLine::Addition(_)))
552            .count();
553        diff.push_str(&format!(
554            "@@ -{},{} +{},{} @@\n",
555            old_start, old_count, new_start, new_count
556        ));
557        for line in &hunk.lines {
558            match line {
559                PatchLine::Context(text) => {
560                    diff.push(' ');
561                    diff.push_str(text);
562                    diff.push('\n');
563                }
564                PatchLine::Addition(text) => {
565                    diff.push('+');
566                    diff.push_str(text);
567                    diff.push('\n');
568                }
569                PatchLine::Deletion(text) => {
570                    diff.push('-');
571                    diff.push_str(text);
572                    diff.push('\n');
573                }
574                PatchLine::Garbage(text) => {
575                    diff.push_str(text);
576                    diff.push('\n');
577                }
578            }
579        }
580    }
581
582    diff
583}
584
585fn adjust_hunk_start(start: isize, row_range: &std::ops::Range<u32>) -> isize {
586    let Ok(start_row) = u32::try_from(start.saturating_sub(1)) else {
587        return start;
588    };
589
590    if row_range.start <= start_row && start_row <= row_range.end {
591        start.saturating_sub(row_range.start as isize)
592    } else {
593        start
594    }
595}
596
597fn hunk_has_change(hunk: &Hunk) -> bool {
598    hunk.lines
599        .iter()
600        .any(|line| matches!(line, PatchLine::Addition(_) | PatchLine::Deletion(_)))
601}
602
603fn path_matches(patch_path: &str, document_path: &str) -> bool {
604    patch_path == document_path
605        || strip_first_path_component(patch_path).is_some_and(|stripped| stripped == document_path)
606}
607
608fn strip_first_path_component(path: &str) -> Option<&str> {
609    path.split_once('/')
610        .map(|(_, rest)| rest)
611        .filter(|rest| !rest.is_empty())
612}
613
614fn compute_cursor_metrics(
615    expected_cursor_editable_region_offset: Option<usize>,
616    actual_cursor: Option<ActualPredictionCursor>,
617) -> (Option<usize>, Option<bool>) {
618    match (expected_cursor_editable_region_offset, actual_cursor) {
619        (Some(expected), Some(actual)) => {
620            let distance = expected.abs_diff(actual.editable_region_offset.unwrap_or_default());
621            let exact_match = distance == 0;
622            (Some(distance), Some(exact_match))
623        }
624        (None, None) => (None, None),
625        (Some(_), None) | (None, Some(_)) => (None, Some(false)),
626    }
627}
628
629#[cfg(test)]
630mod tests {
631    use super::*;
632
633    #[test]
634    fn test_kept_rate_is_computed_when_best_delta_chr_f_score_is_zero() {
635        let original_text = "";
636        let actual_patch = "--- a/file.txt\n+++ b/file.txt\n@@ -0,0 +1 @@\n+bbbbbb\n";
637        let expected_patch = "--- a/file.txt\n+++ b/file.txt\n@@ -0,0 +1 @@\n+cccccc\n";
638        let expected_patches = [PreparedExpectedPatch {
639            patch: expected_patch.to_string(),
640            text: "cccccc".to_string(),
641            cursor_editable_region_offset: None,
642        }];
643
644        let score = score_prediction(PredictionScoringInput {
645            original_text,
646            expected_patches: &expected_patches,
647            actual_patch: Some(actual_patch),
648            actual_cursor: None,
649            reversal_context: None,
650            cumulative_logprob: None,
651            avg_logprob: None,
652            context: None,
653        });
654
655        assert_eq!(score.delta_chr_f, 0.0);
656        assert_eq!(score.kept_rate, Some(0.0));
657    }
658
659    #[test]
660    fn test_scores_related_file_patch_against_context_document() {
661        let original_text = "fn main() {}\n";
662        let expected_patch = "--- a/src/lib.rs\n+++ b/src/lib.rs\n@@ -11,3 +11,3 @@\n fn value() {\n-    1\n+    2\n }\n";
663        let actual_patch = "--- a/project/src/lib.rs\n+++ b/project/src/lib.rs\n@@ -11,3 +11,3 @@\n fn value() {\n-    1\n+    2\n }\n";
664        let expected_patches = [PreparedExpectedPatch {
665            patch: expected_patch.to_string(),
666            text: original_text.to_string(),
667            cursor_editable_region_offset: None,
668        }];
669        let context = [
670            Excerpt {
671                path: "src/main.rs".to_string(),
672                row_range: 0..1,
673                content: original_text.to_string(),
674            },
675            Excerpt {
676                path: "src/lib.rs".to_string(),
677                row_range: 10..13,
678                content: "fn value() {\n    1\n}\n".to_string(),
679            },
680        ];
681
682        let score = score_prediction(PredictionScoringInput {
683            original_text,
684            expected_patches: &expected_patches,
685            actual_patch: Some(actual_patch),
686            actual_cursor: None,
687            reversal_context: None,
688            cumulative_logprob: None,
689            avg_logprob: None,
690            context: Some(&context),
691        });
692
693        assert_eq!(score.delta_chr_f, 100.0);
694        assert_eq!(score.exact_lines_tp, 2);
695        assert_eq!(score.jump_location.unwrap().files_f1, 1.0);
696    }
697
698    #[test]
699    fn test_missing_related_file_prediction_counts_as_false_negative() {
700        let original_text = "fn main() {}\n";
701        let expected_patch = "--- a/src/lib.rs\n+++ b/src/lib.rs\n@@ -11,3 +11,3 @@\n fn value() {\n-    1\n+    2\n }\n";
702        let expected_patches = [PreparedExpectedPatch {
703            patch: expected_patch.to_string(),
704            text: original_text.to_string(),
705            cursor_editable_region_offset: None,
706        }];
707        let context = [Excerpt {
708            path: "src/lib.rs".to_string(),
709            row_range: 10..13,
710            content: "fn value() {\n    1\n}\n".to_string(),
711        }];
712
713        let score = score_prediction(PredictionScoringInput {
714            original_text,
715            expected_patches: &expected_patches,
716            actual_patch: None,
717            actual_cursor: None,
718            reversal_context: None,
719            cumulative_logprob: None,
720            avg_logprob: None,
721            context: Some(&context),
722        });
723
724        assert!(score.delta_chr_f < 100.0);
725        assert_eq!(score.exact_lines_fn, 2);
726        let location = score.jump_location.unwrap();
727        assert_eq!(location.files_fn, 1);
728        assert_eq!(location.lines_fn, 1);
729    }
730}
731