tenant.openagents/omega

1use std::fmt::Display;
2use std::panic::Location;
3use std::thread::ThreadId;
4use std::time::{Duration, Instant};
5
6use collections::HashMap;
7use itertools::Itertools;
8use log::info;
9
10#[derive(Debug, Hash, Eq, PartialEq)]
11enum PerfIssue {
12    Foreground(&'static Location<'static>),
13    Background(&'static Location<'static>),
14    Action(&'static str),
15}
16
17pub struct Reporter {
18    monitor_interval: Duration,
19    forget_after: Duration,
20    history: HashMap<PerfIssue, Instant>,
21    report_longer_then: Duration,
22    foreground_thread: ThreadId,
23}
24
25type ReportMade = bool;
26impl Reporter {
27    pub fn new(
28        monitor_interval: Duration,
29        report_longer_then: Duration,
30        foreground_thread: ThreadId,
31    ) -> Self {
32        Self {
33            monitor_interval,
34            forget_after: Duration::from_mins(5),
35            history: HashMap::default(),
36            report_longer_then,
37            foreground_thread,
38        }
39    }
40    pub fn check_and_report(
41        &mut self,
42        task_stats: &[gpui::ThreadTaskStatistics],
43        action_stats: &gpui::ActionStatistics,
44    ) -> ReportMade {
45        let mut reported_task_hangs = false;
46        reported_task_hangs |= self.report_hanging_foreground(&task_stats);
47        reported_task_hangs |= self.report_hanging_background(&task_stats);
48
49        self.report_hanging_actions(action_stats);
50        reported_task_hangs
51    }
52
53    fn hold_report(&self, issue: PerfIssue) -> bool {
54        self.history
55            .get(&issue)
56            .map(Instant::elapsed)
57            .is_some_and(|since_report| since_report > self.monitor_interval)
58    }
59    fn update_reported(&mut self, new: impl Iterator<Item = PerfIssue>) {
60        let now = Instant::now();
61        for issue in new {
62            if self
63                .history
64                .get(&issue)
65                .is_none_or(|s| s.elapsed() > self.forget_after)
66            {
67                self.history.insert(issue, now);
68            }
69        }
70    }
71}
72
73impl Reporter {
74    fn report_hanging_foreground(
75        &mut self,
76        task_stats: &[gpui::ThreadTaskStatistics],
77    ) -> ReportMade {
78        let foreground = self.foreground_thread;
79        let Some(foreground) = task_stats.iter().find(|t| t.thread_id == foreground) else {
80            return false; // during startup the foreground may not yet have statistics
81        };
82
83        let hangs: Vec<_> = foreground
84            .stats
85            .longest_poll_times
86            .into_iter()
87            .filter(|task| task.poll_duration() > self.report_longer_then)
88            .filter(|task| !self.hold_report(PerfIssue::Foreground(task.location)))
89            .collect();
90        self.update_reported(
91            hangs
92                .iter()
93                .map(|task| PerfIssue::Foreground(task.location)),
94        );
95        if !hangs.is_empty() {
96            info!("New foreground hang detected:\n{}", DisplayTasks(&hangs));
97        }
98
99        !hangs.is_empty()
100    }
101
102    fn report_hanging_background(
103        &mut self,
104        task_stats: &[gpui::ThreadTaskStatistics],
105    ) -> ReportMade {
106        let foreground = self.foreground_thread;
107        let background = task_stats.iter().filter(move |t| t.thread_id != foreground);
108
109        let mut report_made = false;
110        for worker in background {
111            let hangs: Vec<_> = worker
112                .stats
113                .longest_poll_times
114                .into_iter()
115                .filter(|stat| stat.poll_duration() > self.report_longer_then)
116                .filter(|task| !self.hold_report(PerfIssue::Background(task.location)))
117                .collect();
118
119            if hangs.is_empty() {
120                continue;
121            }
122
123            self.update_reported(
124                hangs
125                    .iter()
126                    .map(|task| PerfIssue::Background(task.location)),
127            );
128
129            info!(
130                "Background hang detected on {}:\n{}",
131                worker.thread_name.as_deref().unwrap_or_else(|| "Unknown"),
132                DisplayTasks(&hangs)
133            );
134            report_made = true;
135        }
136        report_made
137    }
138
139    fn report_hanging_actions(&mut self, action_stats: &gpui::ActionStatistics) {
140        let hangs: Vec<_> = action_stats
141            .longest_runtimes(true)
142            .filter(|action| action.runtime() > self.report_longer_then)
143            .filter(|action| !self.hold_report(PerfIssue::Action(action.name)))
144            .collect();
145
146        self.update_reported(hangs.iter().map(|action| PerfIssue::Action(action.name)));
147        if !hangs.is_empty() {
148            info!("Action hang detected:\n{}", DisplayActions(hangs));
149        }
150    }
151}
152
153struct DisplayActions(Vec<gpui::profiler::ActionTiming>);
154struct DisplayTasks<'a>(&'a [gpui::TaskTiming]);
155
156impl Display for DisplayActions {
157    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
158        f.write_str("Actions(s) that ran too long\n")?;
159        for action in self.0.iter().sorted_by_key(|action| action.runtime()).rev() {
160            f.write_fmt(format_args!(
161                "{:<20} - {}",
162                format!("{:?}", action.runtime()), // impl debug does not support alignment
163                action.name
164            ))?;
165            writeln!(f)?;
166        }
167        Ok(())
168    }
169}
170
171impl<'a> Display for DisplayTasks<'a> {
172    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
173        f.write_str("Tasks(s) that ran too long\n")?;
174        for task in self
175            .0
176            .iter()
177            .sorted_by_key(|task| task.poll_duration())
178            .rev()
179        {
180            f.write_fmt(format_args!(
181                "{:<20} - {}",
182                format!("{:?}", task.poll_duration()), // impl debug does not support alignment
183                task.location
184            ))?;
185            writeln!(f)?;
186        }
187        Ok(())
188    }
189}
190