tenant.openagents/omega

1use serde::Serialize;
2
3use crate::jumps::LineFileClassification;
4use crate::patch_metrics::ClassificationMetrics;
5use crate::prediction_score::PredictionScore;
6
7#[derive(Clone, Copy, Debug, Default)]
8pub struct QaSummaryData {
9    pub reverts_edits: Option<bool>,
10    pub confidence: Option<u8>,
11}
12
13#[derive(Clone, Copy, Debug)]
14pub struct PredictionSummaryInput<'a> {
15    pub score: &'a PredictionScore,
16    pub qa: Option<QaSummaryData>,
17    pub retrieved_context_bytes: Option<usize>,
18}
19
20#[derive(Clone, Debug, Serialize)]
21pub struct SummaryJson {
22    pub total_examples: usize,
23    pub avg_delta_chr_f: f32,
24    pub delta_chr_f_beta: f64,
25    pub delta_chr_f_true_positives: usize,
26    pub delta_chr_f_false_positives: usize,
27    pub delta_chr_f_false_negatives: usize,
28    pub delta_chr_f_precision: f64,
29    pub delta_chr_f_recall: f64,
30    pub avg_braces_disbalance: f32,
31    pub exact_lines_true_positives: usize,
32    pub exact_lines_false_positives: usize,
33    pub exact_lines_false_negatives: usize,
34    pub exact_lines_precision: f64,
35    pub exact_lines_recall: f64,
36    pub exact_lines_f1: f64,
37    pub avg_reversal_ratio: f32,
38    #[serde(skip_serializing_if = "Option::is_none")]
39    pub qa_avg_reverts_edits: Option<f32>,
40    #[serde(skip_serializing_if = "Option::is_none")]
41    pub qa_avg_confidence: Option<f32>,
42    #[serde(skip_serializing_if = "Option::is_none")]
43    pub cursor_exact_match_rate: Option<f32>,
44    #[serde(skip_serializing_if = "Option::is_none")]
45    pub cursor_exact_matches: Option<usize>,
46    #[serde(skip_serializing_if = "Option::is_none")]
47    pub cursor_avg_distance: Option<f32>,
48    #[serde(skip_serializing_if = "Option::is_none")]
49    pub cursor_total_evaluated: Option<usize>,
50    #[serde(skip_serializing_if = "Option::is_none")]
51    pub wrong_editable_region_rate: Option<f32>,
52    pub isolated_whitespace_rate: Option<f32>,
53    #[serde(skip_serializing_if = "Option::is_none")]
54    pub isolated_whitespace_count: Option<usize>,
55    #[serde(skip_serializing_if = "Option::is_none")]
56    pub avg_kept_rate: Option<f64>,
57    #[serde(skip_serializing_if = "Option::is_none")]
58    pub kept_rate_examples: Option<usize>,
59    #[serde(skip_serializing_if = "Option::is_none")]
60    pub avg_recall_rate: Option<f64>,
61    #[serde(skip_serializing_if = "Option::is_none")]
62    pub recall_rate_examples: Option<usize>,
63    #[serde(skip_serializing_if = "Option::is_none")]
64    pub total_kept_chars: Option<usize>,
65    #[serde(skip_serializing_if = "Option::is_none")]
66    pub total_correctly_deleted_chars: Option<usize>,
67    #[serde(skip_serializing_if = "Option::is_none")]
68    pub total_discarded_chars: Option<usize>,
69    #[serde(skip_serializing_if = "Option::is_none")]
70    pub editable_context_examples: Option<usize>,
71    #[serde(skip_serializing_if = "Option::is_none")]
72    pub avg_editable_context_lines_precision: Option<f64>,
73    #[serde(skip_serializing_if = "Option::is_none")]
74    pub avg_editable_context_lines_recall: Option<f64>,
75    #[serde(skip_serializing_if = "Option::is_none")]
76    pub avg_editable_context_lines_f1: Option<f64>,
77    #[serde(skip_serializing_if = "Option::is_none")]
78    pub editable_context_lines_tp: Option<usize>,
79    #[serde(skip_serializing_if = "Option::is_none")]
80    pub editable_context_lines_fp: Option<usize>,
81    #[serde(skip_serializing_if = "Option::is_none")]
82    pub editable_context_lines_fn: Option<usize>,
83    #[serde(skip_serializing_if = "Option::is_none")]
84    pub avg_editable_context_files_precision: Option<f64>,
85    #[serde(skip_serializing_if = "Option::is_none")]
86    pub avg_editable_context_files_recall: Option<f64>,
87    #[serde(skip_serializing_if = "Option::is_none")]
88    pub avg_editable_context_files_f1: Option<f64>,
89    #[serde(skip_serializing_if = "Option::is_none")]
90    pub editable_context_files_tp: Option<usize>,
91    #[serde(skip_serializing_if = "Option::is_none")]
92    pub editable_context_files_fp: Option<usize>,
93    #[serde(skip_serializing_if = "Option::is_none")]
94    pub editable_context_files_fn: Option<usize>,
95    #[serde(skip_serializing_if = "Option::is_none")]
96    pub jump_location_examples: Option<usize>,
97    #[serde(skip_serializing_if = "Option::is_none")]
98    pub avg_jump_location_lines_precision: Option<f64>,
99    #[serde(skip_serializing_if = "Option::is_none")]
100    pub avg_jump_location_lines_recall: Option<f64>,
101    #[serde(skip_serializing_if = "Option::is_none")]
102    pub avg_jump_location_lines_f1: Option<f64>,
103    #[serde(skip_serializing_if = "Option::is_none")]
104    pub jump_location_lines_tp: Option<usize>,
105    #[serde(skip_serializing_if = "Option::is_none")]
106    pub jump_location_lines_fp: Option<usize>,
107    #[serde(skip_serializing_if = "Option::is_none")]
108    pub jump_location_lines_fn: Option<usize>,
109    #[serde(skip_serializing_if = "Option::is_none")]
110    pub avg_jump_location_files_precision: Option<f64>,
111    #[serde(skip_serializing_if = "Option::is_none")]
112    pub avg_jump_location_files_recall: Option<f64>,
113    #[serde(skip_serializing_if = "Option::is_none")]
114    pub avg_jump_location_files_f1: Option<f64>,
115    #[serde(skip_serializing_if = "Option::is_none")]
116    pub jump_location_files_tp: Option<usize>,
117    #[serde(skip_serializing_if = "Option::is_none")]
118    pub jump_location_files_fp: Option<usize>,
119    #[serde(skip_serializing_if = "Option::is_none")]
120    pub jump_location_files_fn: Option<usize>,
121    #[serde(skip_serializing_if = "Option::is_none")]
122    pub avg_retrieved_context_bytes: Option<f64>,
123    #[serde(skip_serializing_if = "Option::is_none")]
124    pub total_retrieved_context_bytes: Option<usize>,
125    #[serde(skip_serializing_if = "Option::is_none")]
126    pub retrieved_context_examples: Option<usize>,
127}
128
129/// Mean of observed values; `None` when nothing was observed.
130#[derive(Default)]
131struct MeanAggregate {
132    sum: f64,
133    count: usize,
134}
135
136impl MeanAggregate {
137    fn add(&mut self, value: f64) {
138        self.sum += value;
139        self.count += 1;
140    }
141
142    fn mean(&self) -> Option<f64> {
143        (self.count > 0).then(|| self.sum / self.count as f64)
144    }
145
146    fn mean_f32(&self) -> Option<f32> {
147        self.mean().map(|mean| mean as f32)
148    }
149
150    fn count(&self) -> Option<usize> {
151        (self.count > 0).then_some(self.count)
152    }
153}
154
155/// Fraction of observations that were hits; `None` when nothing was observed.
156#[derive(Default)]
157struct RateAggregate {
158    hits: usize,
159    total: usize,
160}
161
162impl RateAggregate {
163    fn add(&mut self, hit: bool) {
164        self.total += 1;
165        if hit {
166            self.hits += 1;
167        }
168    }
169
170    fn rate(&self) -> Option<f32> {
171        (self.total > 0).then(|| self.hits as f32 / self.total as f32)
172    }
173
174    fn hits(&self) -> Option<usize> {
175        (self.total > 0).then_some(self.hits)
176    }
177
178    fn total(&self) -> Option<usize> {
179        (self.total > 0).then_some(self.total)
180    }
181}
182
183/// Sum of observed values; `None` when nothing was observed.
184#[derive(Default)]
185struct SumAggregate {
186    sum: usize,
187    count: usize,
188}
189
190impl SumAggregate {
191    fn add(&mut self, value: usize) {
192        self.sum += value;
193        self.count += 1;
194    }
195
196    fn total(&self) -> Option<usize> {
197        (self.count > 0).then_some(self.sum)
198    }
199
200    fn mean(&self) -> Option<f64> {
201        (self.count > 0).then(|| self.sum as f64 / self.count as f64)
202    }
203
204    fn count(&self) -> Option<usize> {
205        (self.count > 0).then_some(self.count)
206    }
207}
208
209/// Macro-averaged precision/recall/F1 plus pooled (micro) TP/FP/FN counts.
210#[derive(Default)]
211struct PrfAggregate {
212    count: usize,
213    precision_sum: f64,
214    recall_sum: f64,
215    f1_sum: f64,
216    counts: ClassificationMetrics,
217}
218
219impl PrfAggregate {
220    fn add(&mut self, precision: f64, recall: f64, f1: f64, counts: &ClassificationMetrics) {
221        self.count += 1;
222        self.precision_sum += precision;
223        self.recall_sum += recall;
224        self.f1_sum += f1;
225        self.counts.accumulate(counts);
226    }
227
228    fn avg_precision(&self) -> Option<f64> {
229        (self.count > 0).then(|| self.precision_sum / self.count as f64)
230    }
231
232    fn avg_recall(&self) -> Option<f64> {
233        (self.count > 0).then(|| self.recall_sum / self.count as f64)
234    }
235
236    fn avg_f1(&self) -> Option<f64> {
237        (self.count > 0).then(|| self.f1_sum / self.count as f64)
238    }
239
240    fn true_positives(&self) -> Option<usize> {
241        (self.count > 0).then_some(self.counts.true_positives)
242    }
243
244    fn false_positives(&self) -> Option<usize> {
245        (self.count > 0).then_some(self.counts.false_positives)
246    }
247
248    fn false_negatives(&self) -> Option<usize> {
249        (self.count > 0).then_some(self.counts.false_negatives)
250    }
251
252    fn count(&self) -> Option<usize> {
253        (self.count > 0).then_some(self.count)
254    }
255}
256
257/// Aggregates the line- and file-level halves of a [`LineFileClassification`].
258#[derive(Default)]
259struct LineFileAggregate {
260    lines: PrfAggregate,
261    files: PrfAggregate,
262}
263
264impl LineFileAggregate {
265    fn add(&mut self, classification: &LineFileClassification) {
266        self.lines.add(
267            classification.lines_precision,
268            classification.lines_recall,
269            classification.lines_f1,
270            &classification.lines_counts(),
271        );
272        self.files.add(
273            classification.files_precision,
274            classification.files_recall,
275            classification.files_f1,
276            &classification.files_counts(),
277        );
278    }
279
280    fn count(&self) -> Option<usize> {
281        self.lines.count()
282    }
283}
284
285pub fn compute_summary<'a>(
286    predictions: impl IntoIterator<Item = PredictionSummaryInput<'a>>,
287) -> SummaryJson {
288    let mut total_scores: usize = 0;
289    let mut delta_chr_f_sum: f32 = 0.0;
290    let mut reversal_ratio_sum: f32 = 0.0;
291    let mut braces_disbalance_sum: usize = 0;
292    let mut total_delta_chr_f = ClassificationMetrics::default();
293    let mut delta_chr_f_precision_sum = 0.0;
294    let mut delta_chr_f_recall_sum = 0.0;
295    let mut delta_chr_f_beta = 0.0;
296    let mut total_exact_lines = ClassificationMetrics::default();
297    let mut qa_reverts = RateAggregate::default();
298    let mut qa_confidence = MeanAggregate::default();
299    let mut cursor_exact = RateAggregate::default();
300    let mut cursor_distance = MeanAggregate::default();
301    let mut wrong_editable_region = RateAggregate::default();
302    let mut isolated_whitespace = RateAggregate::default();
303    let mut kept_rate = MeanAggregate::default();
304    let mut recall_rate = MeanAggregate::default();
305    let mut kept_chars = SumAggregate::default();
306    let mut correctly_deleted_chars = SumAggregate::default();
307    let mut discarded_chars = SumAggregate::default();
308    let mut editable_context = LineFileAggregate::default();
309    let mut jump_location = LineFileAggregate::default();
310    let mut retrieved_context_bytes = SumAggregate::default();
311
312    for prediction in predictions {
313        let score = prediction.score;
314
315        total_scores += 1;
316        delta_chr_f_sum += score.delta_chr_f;
317        reversal_ratio_sum += score.reversal_ratio;
318        braces_disbalance_sum += score.braces_disbalance;
319        total_delta_chr_f.accumulate(&score.delta_chr_f_counts());
320        delta_chr_f_precision_sum += score.delta_chr_f_precision;
321        delta_chr_f_recall_sum += score.delta_chr_f_recall;
322        delta_chr_f_beta = score.delta_chr_f_beta;
323        total_exact_lines.accumulate(&score.exact_lines_counts());
324
325        if let Some(qa) = prediction.qa {
326            if let Some(reverts) = qa.reverts_edits {
327                qa_reverts.add(reverts);
328            }
329            if let Some(confidence) = qa.confidence {
330                qa_confidence.add(confidence as f64);
331            }
332        }
333
334        if let Some(wrong) = score.wrong_editable_region {
335            wrong_editable_region.add(wrong);
336        }
337        isolated_whitespace.add(score.has_isolated_whitespace_changes);
338
339        if let Some(value) = score.kept_rate {
340            kept_rate.add(value);
341        }
342        if let Some(value) = score.recall_rate {
343            recall_rate.add(value);
344        }
345        if let Some(value) = score.kept_chars {
346            kept_chars.add(value);
347        }
348        if let Some(value) = score.correctly_deleted_chars {
349            correctly_deleted_chars.add(value);
350        }
351        if let Some(value) = score.discarded_chars {
352            discarded_chars.add(value);
353        }
354        if let Some(value) = prediction.retrieved_context_bytes {
355            retrieved_context_bytes.add(value);
356        }
357
358        if let Some(coverage) = &score.editable_context_coverage {
359            editable_context.add(coverage);
360        }
361        if let Some(location) = &score.jump_location {
362            jump_location.add(location);
363        }
364
365        if let Some(exact_match) = score.cursor_exact_match {
366            cursor_exact.add(exact_match);
367        }
368        if let Some(distance) = score.cursor_distance {
369            cursor_distance.add(distance as f64);
370        }
371    }
372
373    SummaryJson {
374        total_examples: total_scores,
375        avg_delta_chr_f: if total_scores == 0 {
376            0.0
377        } else {
378            delta_chr_f_sum / total_scores as f32
379        },
380        delta_chr_f_beta,
381        delta_chr_f_true_positives: total_delta_chr_f.true_positives,
382        delta_chr_f_false_positives: total_delta_chr_f.false_positives,
383        delta_chr_f_false_negatives: total_delta_chr_f.false_negatives,
384        delta_chr_f_precision: if total_scores == 0 {
385            0.0
386        } else {
387            delta_chr_f_precision_sum / total_scores as f64
388        },
389        delta_chr_f_recall: if total_scores == 0 {
390            0.0
391        } else {
392            delta_chr_f_recall_sum / total_scores as f64
393        },
394        avg_braces_disbalance: if total_scores == 0 {
395            0.0
396        } else {
397            braces_disbalance_sum as f32 / total_scores as f32
398        },
399        exact_lines_true_positives: total_exact_lines.true_positives,
400        exact_lines_false_positives: total_exact_lines.false_positives,
401        exact_lines_false_negatives: total_exact_lines.false_negatives,
402        exact_lines_precision: total_exact_lines.precision(),
403        exact_lines_recall: total_exact_lines.recall(),
404        exact_lines_f1: total_exact_lines.f1(),
405        avg_reversal_ratio: if total_scores == 0 {
406            0.0
407        } else {
408            reversal_ratio_sum / total_scores as f32
409        },
410        qa_avg_reverts_edits: qa_reverts.rate(),
411        qa_avg_confidence: qa_confidence.mean_f32(),
412        cursor_exact_match_rate: cursor_exact.rate(),
413        cursor_exact_matches: cursor_exact.hits(),
414        cursor_avg_distance: cursor_distance.mean_f32(),
415        cursor_total_evaluated: cursor_exact.total(),
416        wrong_editable_region_rate: wrong_editable_region.rate(),
417        isolated_whitespace_rate: isolated_whitespace.rate(),
418        isolated_whitespace_count: isolated_whitespace.hits(),
419        avg_kept_rate: kept_rate.mean(),
420        kept_rate_examples: kept_rate.count(),
421        avg_recall_rate: recall_rate.mean(),
422        recall_rate_examples: recall_rate.count(),
423        total_kept_chars: kept_chars.total(),
424        total_correctly_deleted_chars: correctly_deleted_chars.total(),
425        total_discarded_chars: discarded_chars.total(),
426        editable_context_examples: editable_context.count(),
427        avg_editable_context_lines_precision: editable_context.lines.avg_precision(),
428        avg_editable_context_lines_recall: editable_context.lines.avg_recall(),
429        avg_editable_context_lines_f1: editable_context.lines.avg_f1(),
430        editable_context_lines_tp: editable_context.lines.true_positives(),
431        editable_context_lines_fp: editable_context.lines.false_positives(),
432        editable_context_lines_fn: editable_context.lines.false_negatives(),
433        avg_editable_context_files_precision: editable_context.files.avg_precision(),
434        avg_editable_context_files_recall: editable_context.files.avg_recall(),
435        avg_editable_context_files_f1: editable_context.files.avg_f1(),
436        editable_context_files_tp: editable_context.files.true_positives(),
437        editable_context_files_fp: editable_context.files.false_positives(),
438        editable_context_files_fn: editable_context.files.false_negatives(),
439        jump_location_examples: jump_location.count(),
440        avg_jump_location_lines_precision: jump_location.lines.avg_precision(),
441        avg_jump_location_lines_recall: jump_location.lines.avg_recall(),
442        avg_jump_location_lines_f1: jump_location.lines.avg_f1(),
443        jump_location_lines_tp: jump_location.lines.true_positives(),
444        jump_location_lines_fp: jump_location.lines.false_positives(),
445        jump_location_lines_fn: jump_location.lines.false_negatives(),
446        avg_jump_location_files_precision: jump_location.files.avg_precision(),
447        avg_jump_location_files_recall: jump_location.files.avg_recall(),
448        avg_jump_location_files_f1: jump_location.files.avg_f1(),
449        jump_location_files_tp: jump_location.files.true_positives(),
450        jump_location_files_fp: jump_location.files.false_positives(),
451        jump_location_files_fn: jump_location.files.false_negatives(),
452        avg_retrieved_context_bytes: retrieved_context_bytes.mean(),
453        total_retrieved_context_bytes: retrieved_context_bytes.total(),
454        retrieved_context_examples: retrieved_context_bytes.count(),
455    }
456}
457