tenant.openagents/omega

1//! Headless CLI binary for running Omega Agent in evaluation/benchmark environments.
2//!
3//! Designed to work inside containerized environments (like Harbor/termbench) where:
4//! - The repository is already checked out at the working directory
5//! - The model API key is provided via environment variables
6//! - Results are written to an output directory (default: `/logs/agent/`)
7//!
8//! ## Usage
9//!
10//! ```text
11//! eval-cli --workdir /testbed --model anthropic/claude-sonnet-4-6-latest \
12//!          --instruction "Fix the bug described in..." --timeout 600
13//! ```
14//!
15//! ## Output
16//!
17//! Writes to `--output-dir` (default `/logs/agent/`):
18//!   - `result.json`  — structured result with status, timing, token usage,
19//!     step count, and tool-call counts (total and per tool)
20//!   - `thread.md`    — full conversation as markdown
21//!   - `thread.json`  — raw thread state as JSON
22//!
23//! ## Exit codes
24//!
25//! | Code | Meaning |
26//! |------|---------|
27//! | 0    | Agent finished |
28//! | 1    | Error (model/auth/runtime failure) |
29//! | 2    | Timeout |
30//! | 3    | Interrupted (SIGTERM/SIGINT) |
31
32mod headless;
33
34use std::path::PathBuf;
35use std::process;
36use std::rc::Rc;
37use std::str::FromStr;
38use std::sync::Arc;
39use std::sync::atomic::{AtomicBool, Ordering};
40use std::time::{Duration, Instant};
41
42use acp_thread::AgentConnection as _;
43use agent::{NativeAgent, NativeAgentConnection, Templates, ThreadStore};
44use agent_client_protocol::schema::v1 as acp;
45use anyhow::{Context, Result};
46use clap::Parser;
47use feature_flags::FeatureFlagAppExt as _;
48
49use futures::{FutureExt, select_biased};
50use gpui::{AppContext as _, AsyncApp, Entity, UpdateGlobal};
51use language_model::{
52    ANTHROPIC_PROVIDER_ID, LanguageModel, LanguageModelId, LanguageModelProviderId,
53    LanguageModelRegistry, SelectedModel,
54};
55use project::Project;
56use settings::SettingsStore;
57use util::path_list::PathList;
58
59use crate::headless::AgentCliAppState;
60
61#[derive(Parser, Debug)]
62#[command(
63    name = "eval-cli",
64    about = "Run Omega Agent headlessly in evaluation/benchmark environments"
65)]
66struct Args {
67    /// Output current environment variables as JSON to stdout.
68    /// Used internally by Omega's shell environment capture.
69    #[arg(long, hide = true)]
70    printenv: bool,
71
72    /// Path to the repository working directory. Defaults to the current directory.
73    #[arg(long, default_value = ".")]
74    workdir: PathBuf,
75
76    /// Instruction/prompt text. If omitted, read from stdin.
77    #[arg(long, allow_hyphen_values = true)]
78    instruction: Option<String>,
79
80    /// File containing additional instruction text appended after the task prompt.
81    #[arg(long)]
82    instruction_suffix_file: Option<PathBuf>,
83
84    /// Language model to use, in `provider/model` format.
85    #[arg(long, default_value = "anthropic/claude-sonnet-4-6-latest")]
86    model: String,
87
88    /// Maximum wall-clock time in seconds for the agent run.
89    #[arg(long)]
90    timeout: Option<u64>,
91
92    /// Directory for output artifacts (result.json, thread.md, thread.json).
93    #[arg(long, default_value = ".")]
94    output_dir: PathBuf,
95
96    /// Disable staff mode (staff mode is enabled by default).
97    #[arg(long)]
98    no_staff: bool,
99
100    /// Reasoning effort level for models that support thinking (low, medium, high).
101    /// Defaults to "high" for thinking-capable models.
102    #[arg(long)]
103    reasoning_effort: Option<String>,
104
105    /// Enable or disable extended thinking. Defaults to model auto-detection if omitted.
106    #[arg(long)]
107    thinking: Option<bool>,
108}
109
110enum AgentOutcome {
111    Completed,
112    Timeout { seconds: u64 },
113    Interrupted,
114}
115
116#[derive(serde::Serialize)]
117struct EvalResult {
118    status: String,
119    #[serde(skip_serializing_if = "Option::is_none")]
120    error: Option<String>,
121    duration_secs: f64,
122    #[serde(skip_serializing_if = "Option::is_none")]
123    timeout_secs: Option<u64>,
124    model: String,
125    #[serde(skip_serializing_if = "Option::is_none")]
126    input_tokens: Option<u64>,
127    #[serde(skip_serializing_if = "Option::is_none")]
128    output_tokens: Option<u64>,
129    #[serde(skip_serializing_if = "Option::is_none")]
130    cache_creation_input_tokens: Option<u64>,
131    #[serde(skip_serializing_if = "Option::is_none")]
132    cache_read_input_tokens: Option<u64>,
133    /// Number of agent (assistant) turns, i.e. model round-trips in the agentic
134    /// loop. Reported as "steps" by the eval harness.
135    #[serde(skip_serializing_if = "Option::is_none")]
136    step_count: Option<u64>,
137    /// Total number of tool calls across all steps.
138    #[serde(skip_serializing_if = "Option::is_none")]
139    tool_call_count: Option<u64>,
140    /// Tool calls broken down by tool name.
141    #[serde(skip_serializing_if = "Option::is_none")]
142    tool_calls: Option<std::collections::BTreeMap<String, u64>>,
143}
144
145/// Per-run statistics collected from the finished thread, written into
146/// `result.json` so the post-hoc report can compute success-conditioned metrics.
147#[derive(Default)]
148struct RunStats {
149    token_usage: Option<language_model::TokenUsage>,
150    step_count: Option<u64>,
151    tool_call_count: Option<u64>,
152    tool_calls: Option<std::collections::BTreeMap<String, u64>>,
153}
154
155const EXIT_OK: i32 = 0;
156const EXIT_ERROR: i32 = 1;
157const EXIT_TIMEOUT: i32 = 2;
158const EXIT_INTERRUPTED: i32 = 3;
159const MODEL_DISCOVERY_TIMEOUT: Duration = Duration::from_secs(30);
160const MODEL_DISCOVERY_POLL_INTERVAL: Duration = Duration::from_millis(100);
161
162static TERMINATED: AtomicBool = AtomicBool::new(false);
163
164fn main() {
165    let args = Args::parse();
166
167    if args.printenv {
168        util::shell_env::print_env();
169        return;
170    }
171
172    env_logger::init();
173
174    ctrlc::set_handler(|| {
175        TERMINATED.store(true, Ordering::SeqCst);
176    })
177    .expect("failed to set signal handler");
178
179    let instruction = read_instruction(&args).unwrap_or_else(|e| {
180        eprintln!("Error reading instruction: {e}");
181        process::exit(EXIT_ERROR);
182    });
183
184    let workdir = args.workdir.canonicalize().unwrap_or_else(|e| {
185        eprintln!("Invalid --workdir {:?}: {e}", args.workdir);
186        process::exit(EXIT_ERROR);
187    });
188
189    let output_dir = args.output_dir.clone();
190    if let Err(e) = std::fs::create_dir_all(&output_dir) {
191        eprintln!("Error creating output dir {}: {e}", output_dir.display());
192        process::exit(EXIT_ERROR);
193    }
194    let output_dir = output_dir.canonicalize().unwrap_or_else(|e| {
195        eprintln!("Invalid --output-dir {:?}: {e}", output_dir);
196        process::exit(EXIT_ERROR);
197    });
198
199    let http_client = Arc::new(reqwest_client::ReqwestClient::new());
200    let app = gpui_platform::headless().with_http_client(http_client);
201
202    app.run(move |cx| {
203        let app_state = headless::init(cx);
204        cx.set_staff(!args.no_staff);
205
206        // Eval hook: enable additional feature-flag-gated tools (e.g. the LSP
207        // navigation tools behind `lsp-tool` / `rename-tool`) so experiments can
208        // measure the agent with tools that aren't yet GA. Comma-separated flag
209        // names; unset in production.
210        if let Ok(raw_flags) = std::env::var("ZED_EVAL_ENABLE_FLAGS") {
211            let flags: Vec<String> = raw_flags
212                .split(',')
213                .map(|flag| flag.trim().to_string())
214                .filter(|flag| !flag.is_empty())
215                .collect();
216            if !flags.is_empty() {
217                cx.update_flags(!args.no_staff, flags);
218            }
219        }
220
221        let openai_compatible_providers_json = openai_compatible_providers_override();
222        let anthropic_available_models_json = anthropic_available_models_override();
223
224        let model_name = args.model.clone();
225        let timeout = args.timeout;
226        let thinking_override = args.thinking;
227        let reasoning_effort = args.reasoning_effort.clone();
228
229        cx.spawn(async move |cx| {
230            // Each settings change below is applied in its own `cx.update` call (rather than
231            // inline in the synchronous body of `app.run`) so that GPUI flushes the resulting
232            // `NotifyGlobalObservers` effect before we move on. Without this, the
233            // openai_compatible/anthropic provider registration (driven by an
234            // `observe_global::<SettingsStore>` callback in language_models::init) wouldn't
235            // have run yet by the time we collect `auth_tasks`, so a newly-added provider's
236            // `authenticate()` would never get called and it would be permanently stuck
237            // unauthenticated.
238            if let Some(providers_json) = &openai_compatible_providers_json {
239                let result = cx.update(|cx| apply_openai_compatible_providers(providers_json, cx));
240                if let Err(e) = result {
241                    eprintln!("Error applying {OPENAI_COMPATIBLE_PROVIDERS_ENV}: {e:#}");
242                    process::exit(EXIT_ERROR);
243                }
244            }
245
246            if let Some(models_json) = &anthropic_available_models_json {
247                let result = cx.update(|cx| apply_anthropic_available_models(models_json, cx));
248                if let Err(e) = result {
249                    eprintln!("Error applying {ANTHROPIC_AVAILABLE_MODELS_ENV}: {e:#}");
250                    process::exit(EXIT_ERROR);
251                }
252            }
253
254            let auth_tasks = cx.update(|cx| {
255                LanguageModelRegistry::global(cx).update(cx, |registry, cx| {
256                    registry
257                        .providers()
258                        .iter()
259                        .map(|p| p.authenticate(cx))
260                        .collect::<Vec<_>>()
261                })
262            });
263            futures::future::join_all(auth_tasks).await;
264
265            let start = Instant::now();
266
267            let (outcome, stats) = run_agent(
268                &app_state,
269                &workdir,
270                &instruction,
271                &model_name,
272                timeout,
273                thinking_override,
274                reasoning_effort.as_deref(),
275                Some(&output_dir),
276                openai_compatible_providers_json.as_deref(),
277                anthropic_available_models_json.as_deref(),
278                cx,
279            )
280            .await;
281
282            let duration = start.elapsed();
283
284            let (status, error, exit_code) = match &outcome {
285                Ok(AgentOutcome::Completed) => ("completed".to_string(), None, EXIT_OK),
286                Ok(AgentOutcome::Timeout { seconds }) => {
287                    eprintln!("Timeout: agent exceeded {seconds}s time limit");
288                    ("timeout".to_string(), None, EXIT_TIMEOUT)
289                }
290                Ok(AgentOutcome::Interrupted) => {
291                    eprintln!("Interrupted: received SIGTERM, saved partial output");
292                    ("interrupted".to_string(), None, EXIT_INTERRUPTED)
293                }
294                Err(e) => {
295                    eprintln!("Error: {e:#}");
296                    ("error".to_string(), Some(format!("{e:#}")), EXIT_ERROR)
297                }
298            };
299
300            let token_usage = stats.token_usage;
301            let result = EvalResult {
302                status,
303                error,
304                duration_secs: duration.as_secs_f64(),
305                timeout_secs: timeout,
306                model: model_name.clone(),
307                input_tokens: token_usage.as_ref().map(|u| u.input_tokens),
308                output_tokens: token_usage.as_ref().map(|u| u.output_tokens),
309                cache_creation_input_tokens: token_usage
310                    .as_ref()
311                    .filter(|u| u.cache_creation_input_tokens > 0)
312                    .map(|u| u.cache_creation_input_tokens),
313                cache_read_input_tokens: token_usage
314                    .as_ref()
315                    .filter(|u| u.cache_read_input_tokens > 0)
316                    .map(|u| u.cache_read_input_tokens),
317                step_count: stats.step_count,
318                tool_call_count: stats.tool_call_count,
319                tool_calls: stats.tool_calls,
320            };
321
322            match serde_json::to_string_pretty(&result) {
323                Ok(json) => {
324                    if let Err(e) = std::fs::write(output_dir.join("result.json"), &json) {
325                        eprintln!("Error writing result.json: {e:#}");
326                    }
327                    eprintln!("[eval-cli] result: {json}");
328                }
329                Err(e) => eprintln!("Error serializing result: {e:#}"),
330            }
331
332            cx.update(|cx| cx.quit());
333            process::exit(exit_code);
334        })
335        .detach();
336    });
337}
338
339/// Name of the env var carrying a JSON object to merge into
340/// `language_models.openai_compatible` user settings before model discovery, in
341/// the same shape as Omega's `openai_compatible` settings key (provider id ->
342/// `{ "api_url": ..., "available_models": [...] }`). Lets eval-cli route the
343/// agent itself through an OpenAI-compatible endpoint (e.g. Baseten) that isn't
344/// one of Omega's built-in providers, without hardcoding it into eval-cli.
345const OPENAI_COMPATIBLE_PROVIDERS_ENV: &str = "ZED_OPENAI_COMPATIBLE_PROVIDERS";
346
347fn openai_compatible_providers_override() -> Option<String> {
348    let raw = std::env::var(OPENAI_COMPATIBLE_PROVIDERS_ENV).ok()?;
349    if raw.trim().is_empty() {
350        return None;
351    }
352    Some(raw)
353}
354
355fn apply_openai_compatible_providers(providers_json: &str, cx: &mut gpui::App) -> Result<()> {
356    let settings = format!(r#"{{"language_models": {{"openai_compatible": {providers_json}}}}}"#);
357    SettingsStore::update_global(cx, |store, cx| {
358        store.set_user_settings(&settings, cx).result()
359    })
360    .context("applying openai_compatible provider settings")?;
361    Ok(())
362}
363
364/// Name of the env var carrying a JSON array to merge into
365/// `language_models.anthropic.available_models` user settings before model
366/// discovery, in the same shape as Omega's `anthropic.available_models` settings
367/// key (a list of `{ "name": ..., "max_tokens": ..., ... }` entries). Lets
368/// eval-cli run models that exist on the Anthropic API for the configured
369/// key (e.g. early-access-program models) but aren't returned by the live
370/// `/v1/models` listing, without hardcoding them into eval-cli.
371const ANTHROPIC_AVAILABLE_MODELS_ENV: &str = "ZED_ANTHROPIC_AVAILABLE_MODELS";
372
373fn anthropic_available_models_override() -> Option<String> {
374    let raw = std::env::var(ANTHROPIC_AVAILABLE_MODELS_ENV).ok()?;
375    if raw.trim().is_empty() {
376        return None;
377    }
378    Some(raw)
379}
380
381fn apply_anthropic_available_models(models_json: &str, cx: &mut gpui::App) -> Result<()> {
382    let settings =
383        format!(r#"{{"language_models": {{"anthropic": {{"available_models": {models_json}}}}}}}"#);
384    SettingsStore::update_global(cx, |store, cx| {
385        store.set_user_settings(&settings, cx).result()
386    })
387    .context("applying anthropic available_models settings")?;
388    Ok(())
389}
390
391fn read_instruction(args: &Args) -> Result<String> {
392    let mut text = if let Some(text) = &args.instruction {
393        text.clone()
394    } else {
395        use std::io::Read;
396        let mut buf = String::new();
397        std::io::stdin()
398            .read_to_string(&mut buf)
399            .context("reading instruction from stdin")?;
400        buf
401    };
402    anyhow::ensure!(!text.trim().is_empty(), "instruction is empty");
403
404    if let Some(path) = &args.instruction_suffix_file {
405        let suffix = read_instruction_suffix_file(path)?;
406        text.push_str("\n\n");
407        text.push_str(&suffix);
408    }
409    Ok(text)
410}
411
412fn read_instruction_suffix_file(path: &PathBuf) -> Result<String> {
413    let suffix = std::fs::read_to_string(path)
414        .with_context(|| format!("reading instruction suffix file {}", path.display()))?;
415    let suffix = suffix.trim().to_string();
416    anyhow::ensure!(!suffix.is_empty(), "instruction suffix file is empty");
417    Ok(suffix)
418}
419
420async fn wait_for_model(selected: &SelectedModel, cx: &mut AsyncApp) -> Result<()> {
421    let started_at = Instant::now();
422
423    loop {
424        let found = cx.update(|cx| find_available_model(selected, cx).is_some());
425        if found {
426            return Ok(());
427        }
428
429        cx.update(|cx| ensure_provider_authenticated(selected, cx))?;
430
431        let selected_provider_has_models = cx.update(|cx| {
432            LanguageModelRegistry::global(cx)
433                .read(cx)
434                .available_models(cx)
435                .any(|model| model.provider_id() == selected.provider)
436        });
437        let should_wait_for_discovery =
438            selected.provider == ANTHROPIC_PROVIDER_ID || !selected_provider_has_models;
439
440        if !should_wait_for_discovery || started_at.elapsed() >= MODEL_DISCOVERY_TIMEOUT {
441            return Err(cx.update(|cx| model_not_found_error(&selected_model_name(selected), cx)));
442        }
443
444        cx.background_executor()
445            .timer(MODEL_DISCOVERY_POLL_INTERVAL)
446            .await;
447    }
448}
449
450fn ensure_provider_authenticated(selected: &SelectedModel, cx: &gpui::App) -> Result<()> {
451    let registry = LanguageModelRegistry::global(cx);
452    let provider = registry
453        .read(cx)
454        .provider(&selected.provider)
455        .ok_or_else(|| anyhow::anyhow!("Provider {} not found", selected.provider.0))?;
456
457    anyhow::ensure!(
458        provider.is_authenticated(cx),
459        "Provider {} is not authenticated",
460        selected.provider.0
461    );
462
463    Ok(())
464}
465
466fn find_available_model(
467    selected: &SelectedModel,
468    cx: &gpui::App,
469) -> Option<Arc<dyn LanguageModel>> {
470    let registry = LanguageModelRegistry::global(cx);
471    let models = registry.read(cx).available_models(cx).collect::<Vec<_>>();
472
473    if let Some(model) = models
474        .iter()
475        .find(|model| model.provider_id() == selected.provider && model.id() == selected.model)
476    {
477        return Some(model.clone());
478    }
479
480    models
481        .into_iter()
482        .filter(|model| {
483            model.provider_id() == selected.provider
484                && model_id_matches_selected(&model.provider_id(), &model.id(), &selected.model)
485        })
486        .max_by(|left, right| left.id().0.to_string().cmp(&right.id().0.to_string()))
487}
488
489fn model_id_matches_selected(
490    provider_id: &LanguageModelProviderId,
491    available: &LanguageModelId,
492    selected: &LanguageModelId,
493) -> bool {
494    if available == selected {
495        return true;
496    }
497
498    if provider_id != &ANTHROPIC_PROVIDER_ID {
499        return false;
500    }
501
502    anthropic_model_ids_match(available.0.as_ref(), selected.0.as_ref())
503}
504
505fn anthropic_model_ids_match(available: &str, selected: &str) -> bool {
506    let available = anthropic_model_alias_base(available);
507    let selected = anthropic_model_alias_base(selected);
508
509    available == selected || anthropic_dated_model_id_matches_base(available, selected)
510}
511
512fn anthropic_model_alias_base(mut model_id: &str) -> &str {
513    if let Some(stripped) = model_id.strip_suffix("-latest") {
514        model_id = stripped;
515    }
516    if let Some(stripped) = model_id.strip_suffix("-thinking") {
517        model_id = stripped;
518    }
519    if let Some(stripped) = model_id.strip_suffix("-1m-context") {
520        model_id = stripped;
521    }
522    model_id
523}
524
525fn anthropic_dated_model_id_matches_base(available: &str, selected: &str) -> bool {
526    let Some(suffix) = available.strip_prefix(selected) else {
527        return false;
528    };
529    let Some(date) = suffix.strip_prefix('-') else {
530        return false;
531    };
532
533    date.len() == 8 && date.chars().all(|character| character.is_ascii_digit())
534}
535
536fn selected_model_name(selected: &SelectedModel) -> String {
537    format!("{}/{}", selected.provider.0, selected.model.0)
538}
539
540fn model_not_found_error(model_name: &str, cx: &gpui::App) -> anyhow::Error {
541    let available = LanguageModelRegistry::global(cx)
542        .read(cx)
543        .available_models(cx)
544        .map(|model| format!("{}/{}", model.provider_id().0, model.id().0))
545        .collect::<Vec<_>>();
546    let available = if available.is_empty() {
547        "(none)".to_string()
548    } else {
549        available.join(", ")
550    };
551
552    anyhow::anyhow!("Model {model_name} not found. Available: {available}")
553}
554
555#[cfg(test)]
556mod tests {
557    use super::*;
558
559    #[test]
560    fn anthropic_latest_alias_matches_listed_base_model() {
561        assert!(model_id_matches_selected(
562            &ANTHROPIC_PROVIDER_ID,
563            &LanguageModelId("claude-sonnet-4-6".into()),
564            &LanguageModelId("claude-sonnet-4-6-latest".into()),
565        ));
566    }
567
568    #[test]
569    fn anthropic_thinking_alias_matches_listed_base_model() {
570        assert!(model_id_matches_selected(
571            &ANTHROPIC_PROVIDER_ID,
572            &LanguageModelId("claude-sonnet-4-6".into()),
573            &LanguageModelId("claude-sonnet-4-6-1m-context-thinking-latest".into()),
574        ));
575    }
576
577    #[test]
578    fn anthropic_latest_alias_matches_listed_dated_model() {
579        assert!(model_id_matches_selected(
580            &ANTHROPIC_PROVIDER_ID,
581            &LanguageModelId("claude-sonnet-4-6-20260518".into()),
582            &LanguageModelId("claude-sonnet-4-6-latest".into()),
583        ));
584    }
585
586    #[test]
587    fn non_anthropic_models_require_exact_ids() {
588        assert!(!model_id_matches_selected(
589            &LanguageModelProviderId("other".into()),
590            &LanguageModelId("claude-sonnet-4-6".into()),
591            &LanguageModelId("claude-sonnet-4-6-latest".into()),
592        ));
593    }
594}
595
596async fn run_agent(
597    app_state: &Arc<AgentCliAppState>,
598    workdir: &std::path::Path,
599    instruction: &str,
600    model_name: &str,
601    timeout: Option<u64>,
602    thinking_override: Option<bool>,
603    reasoning_effort: Option<&str>,
604    output_dir: Option<&std::path::Path>,
605    openai_compatible_providers_json: Option<&str>,
606    anthropic_available_models_json: Option<&str>,
607    cx: &mut AsyncApp,
608) -> (Result<AgentOutcome>, RunStats) {
609    let selected = match SelectedModel::from_str(model_name).map_err(|e| anyhow::anyhow!("{e}")) {
610        Ok(selected) => selected,
611        Err(e) => return (Err(e), RunStats::default()),
612    };
613
614    if let Err(e) = wait_for_model(&selected, cx).await {
615        return (Err(e), RunStats::default());
616    }
617
618    let setup_result: Result<()> = cx.update(|cx| {
619        let registry = LanguageModelRegistry::global(cx);
620        let model = find_available_model(&selected, cx)
621            .ok_or_else(|| model_not_found_error(model_name, cx))?;
622        let provider = registry
623            .read(cx)
624            .provider(&model.provider_id())
625            .context("Provider not found")?;
626
627        let supports_thinking = model.supports_thinking();
628        let model_id = model.id().0.to_string();
629
630        registry.update(cx, |registry, cx| {
631            registry.set_default_model(
632                Some(language_model::ConfiguredModel { provider, model }),
633                cx,
634            );
635        });
636
637        let enable_thinking = thinking_override.unwrap_or(supports_thinking);
638        let effort = if enable_thinking {
639            match reasoning_effort {
640                Some(level) => format!("\"{level}\""),
641                None => "\"high\"".to_string(),
642            }
643        } else {
644            "null".to_string()
645        };
646        let provider_id = selected.provider.0.to_string();
647        // set_user_settings replaces the whole user settings buffer, so the
648        // openai_compatible/anthropic blocks applied earlier (before model
649        // discovery) have to be folded back in here, or they would be dropped
650        // by this call.
651        let mut language_models_fields = Vec::new();
652        if let Some(providers_json) = openai_compatible_providers_json {
653            language_models_fields.push(format!(r#""openai_compatible": {providers_json}"#));
654        }
655        if let Some(models_json) = anthropic_available_models_json {
656            language_models_fields.push(format!(
657                r#""anthropic": {{"available_models": {models_json}}}"#
658            ));
659        }
660        let language_models_settings = format!("{{{}}}", language_models_fields.join(","));
661        // Disable specific tools (e.g. `fetch`/`search_web` on air-gapped
662        // benchmarks) the canonical way: via the agent profile. The model only
663        // sees a built-in tool when the active profile enables it (see
664        // Thread::enabled_tools), so we define a dedicated "eval" profile that
665        // mirrors the built-in "write" profile minus the disabled tools, and make
666        // it the default profile. A fresh profile key is NOT deep-merged against
667        // the defaults, so it can't inherit "write"'s tools — the full set is
668        // listed explicitly here. Keep WRITE_TOOLS in sync with the "write"
669        // profile in assets/settings/default.json.
670        let profile_field = {
671            let raw = std::env::var("ZED_EVAL_DISABLE_TOOLS").unwrap_or_default();
672            let disabled = raw
673                .split(',')
674                .map(|name| name.trim().to_string())
675                .filter(|name| !name.is_empty())
676                .collect::<std::collections::HashSet<String>>();
677            if disabled.is_empty() {
678                String::new()
679            } else {
680                const WRITE_TOOLS: &[&str] = &[
681                    "copy_path",
682                    "create_directory",
683                    "create_thread",
684                    "delete_path",
685                    "diagnostics",
686                    "apply_code_action",
687                    "edit_file",
688                    "write_file",
689                    "fetch",
690                    "find_path",
691                    "find_references",
692                    "get_code_actions",
693                    "go_to_definition",
694                    "list_agents_and_models",
695                    "list_directory",
696                    "move_path",
697                    "rename_symbol",
698                    "read_file",
699                    "grep",
700                    "skill",
701                    "spawn_agent",
702                    "terminal",
703                    "search_web",
704                ];
705                let tools = WRITE_TOOLS
706                    .iter()
707                    .filter(|tool| !disabled.contains(**tool))
708                    .map(|tool| format!(r#""{tool}": true"#))
709                    .collect::<Vec<_>>()
710                    .join(", ");
711                format!(
712                    r#","default_profile": "eval", "profiles": {{"eval": {{"name": "Eval", "enable_all_context_servers": true, "tools": {{{tools}}}}}}}"#
713                )
714            }
715        };
716        SettingsStore::update_global(cx, |store, cx| {
717            let settings = format!(
718                r#"{{
719                    "language_models": {language_models_settings},
720                    "agent": {{
721                        "tool_permissions": {{"default": "allow"}},
722                        "default_model": {{
723                            "provider": "{provider_id}",
724                            "model": "{model_id}",
725                            "enable_thinking": {enable_thinking},
726                            "effort": {effort}
727                        }}{profile_field}
728                    }},
729                    "autosave": "off",
730                    "format_on_save": "off"
731                }}"
732                "#
733            );
734            store.set_user_settings(&settings, cx).result()
735        })
736        .context("updating agent settings")?;
737
738        anyhow::Ok(())
739    });
740
741    if let Err(e) = setup_result {
742        return (Err(e), RunStats::default());
743    }
744
745    let project = cx.update(|cx| {
746        Project::local(
747            app_state.client.clone(),
748            app_state.node_runtime.clone(),
749            app_state.user_store.clone(),
750            app_state.languages.clone(),
751            app_state.fs.clone(),
752            None,
753            project::LocalProjectFlags {
754                init_worktree_trust: false,
755                ..Default::default()
756            },
757            cx,
758        )
759    });
760
761    let worktree = project.update(cx, |project, cx| project.create_worktree(workdir, true, cx));
762    let worktree = match worktree.await {
763        Ok(w) => w,
764        Err(e) => return (Err(e).context("creating worktree"), RunStats::default()),
765    };
766
767    let scan_result = worktree.update(cx, |tree, _cx| {
768        tree.as_local()
769            .context("expected local worktree")
770            .map(|local| local.scan_complete())
771    });
772    match scan_result {
773        Ok(future) => future.await,
774        Err(e) => return (Err(e), RunStats::default()),
775    };
776
777    let output_worktree = match output_dir {
778        Some(output_dir) if !output_dir.starts_with(workdir) => {
779            let output_worktree = project.update(cx, |project, cx| {
780                project.create_worktree(output_dir, true, cx)
781            });
782            match output_worktree.await {
783                Ok(worktree) => Some(worktree),
784                Err(e) => {
785                    return (
786                        Err(e).context("creating output worktree"),
787                        RunStats::default(),
788                    );
789                }
790            }
791        }
792        _ => None,
793    };
794
795    if let Some(output_worktree) = output_worktree {
796        let scan_result = output_worktree.update(cx, |tree, _cx| {
797            tree.as_local()
798                .context("expected local output worktree")
799                .map(|local| local.scan_complete())
800        });
801        match scan_result {
802            Ok(future) => future.await,
803            Err(e) => return (Err(e), RunStats::default()),
804        };
805    }
806
807    let agent = cx.update(|cx| {
808        let thread_store = cx.new(|cx| ThreadStore::new(cx));
809        NativeAgent::new(thread_store, Templates::new(), app_state.fs.clone(), cx)
810    });
811
812    let connection = Rc::new(NativeAgentConnection(agent.clone()));
813    let acp_thread = match cx
814        .update(|cx| {
815            connection
816                .clone()
817                .new_session(project, PathList::new(&[workdir]), cx)
818        })
819        .await
820    {
821        Ok(t) => t,
822        Err(e) => return (Err(e).context("creating ACP session"), RunStats::default()),
823    };
824
825    let _subscription = cx.subscribe(&acp_thread, |acp_thread, event, cx| {
826        log_acp_thread_event(&acp_thread, event, cx);
827    });
828
829    let message = vec![acp::ContentBlock::Text(acp::TextContent::new(
830        instruction.to_string(),
831    ))];
832
833    let send_future = acp_thread.update(cx, |acp_thread: &mut acp_thread::AcpThread, cx| {
834        acp_thread.send(message, cx)
835    });
836
837    let timeout_future = if let Some(timeout_secs) = timeout {
838        futures::future::Either::Left(
839            cx.background_executor()
840                .timer(Duration::from_secs(timeout_secs)),
841        )
842    } else {
843        futures::future::Either::Right(futures::future::pending::<()>())
844    };
845
846    let sigterm_future = {
847        let executor = cx.background_executor().clone();
848        async move {
849            while !TERMINATED.load(Ordering::Relaxed) {
850                executor.timer(Duration::from_millis(100)).await;
851            }
852        }
853    };
854
855    let outcome = select_biased! {
856        result = send_future.fuse() => match result {
857            Ok(Some(response)) => {
858                eprintln!("[eval-cli] stopped: {:?}", response.stop_reason);
859                if response.stop_reason == acp::StopReason::MaxTokens {
860                    Err(anyhow::anyhow!("Model hit maximum token limit"))
861                } else {
862                    Ok(AgentOutcome::Completed)
863                }
864            }
865            Ok(None) => {
866                eprintln!("[eval-cli] completed (no response)");
867                Ok(AgentOutcome::Completed)
868            }
869            Err(e) => Err(e).context("agent run failed"),
870        },
871        _ = sigterm_future.fuse() => {
872            eprintln!("[eval-cli] received SIGTERM, cancelling...");
873            acp_thread.update(cx, |t: &mut acp_thread::AcpThread, cx| t.cancel(cx)).await;
874            Ok(AgentOutcome::Interrupted)
875        },
876        _ = timeout_future.fuse() => {
877            acp_thread.update(cx, |t: &mut acp_thread::AcpThread, cx| t.cancel(cx)).await;
878            Ok(AgentOutcome::Timeout { seconds: timeout.unwrap_or(0) })
879        }
880    };
881
882    let thread = cx.update(|cx| {
883        let session_id = acp_thread.read(cx).session_id().clone();
884        connection.thread(&session_id, cx)
885    });
886
887    let mut step_count = None;
888    let mut tool_call_count = None;
889    let mut tool_calls = None;
890    let cumulative_usage = if let Some(thread) = &thread {
891        let db_thread = thread.read_with(cx, |thread, cx| thread.to_db(cx));
892        let db_thread = db_thread.await;
893        let mut counts = std::collections::BTreeMap::<String, u64>::new();
894        let mut agent_turn_count = 0;
895        for message in &db_thread.messages {
896            let Some(agent_message) = message.as_agent_message() else {
897                continue;
898            };
899            agent_turn_count += 1;
900            for request_message in agent_message.to_request() {
901                for content in request_message.content {
902                    if let language_model::MessageContent::ToolUse(tool_use) = content {
903                        *counts.entry(tool_use.name.to_string()).or_default() += 1;
904                    }
905                }
906            }
907        }
908        step_count = Some(agent_turn_count);
909        tool_call_count = Some(counts.values().sum());
910        tool_calls = Some(counts);
911        let usage = db_thread.cumulative_token_usage;
912        if usage.input_tokens > 0 || usage.output_tokens > 0 {
913            Some(usage)
914        } else {
915            None
916        }
917    } else {
918        None
919    };
920
921    let acp_usage = cx.update(|cx| {
922        acp_thread
923            .read(cx)
924            .token_usage()
925            .map(|usage| language_model::TokenUsage {
926                input_tokens: usage.input_tokens,
927                output_tokens: usage.output_tokens,
928                ..Default::default()
929            })
930    });
931
932    let final_usage = cumulative_usage.or(acp_usage);
933
934    if let (Some(thread), Some(dir)) = (&thread, output_dir) {
935        let markdown = thread.read_with(cx, |thread, _cx| thread.to_markdown());
936        if let Err(e) = std::fs::write(dir.join("thread.md"), markdown) {
937            eprintln!("Error writing thread.md: {e:#}");
938        }
939
940        let db_thread = thread.read_with(cx, |thread, cx| thread.to_db(cx));
941        let db_thread = db_thread.await;
942        match serde_json::to_string_pretty(&db_thread) {
943            Ok(json) => {
944                if let Err(e) = std::fs::write(dir.join("thread.json"), json) {
945                    eprintln!("Error writing thread.json: {e:#}");
946                }
947            }
948            Err(e) => eprintln!("Error serializing thread.json: {e:#}"),
949        }
950    }
951
952    (
953        outcome,
954        RunStats {
955            token_usage: final_usage,
956            step_count,
957            tool_call_count,
958            tool_calls,
959        },
960    )
961}
962
963fn log_acp_thread_event(
964    acp_thread: &Entity<acp_thread::AcpThread>,
965    event: &acp_thread::AcpThreadEvent,
966    cx: &mut gpui::App,
967) {
968    match event {
969        acp_thread::AcpThreadEvent::NewEntry => {
970            let entries = acp_thread.read(cx).entries();
971            if let Some(acp_thread::AgentThreadEntry::AssistantMessage(message)) = entries.last() {
972                for chunk in &message.chunks {
973                    if let acp_thread::AssistantMessageChunk::Message { id: _, block } = chunk {
974                        if let acp_thread::ContentBlock::Markdown { markdown } = block {
975                            let text = markdown.read(cx).source().to_string();
976                            if !text.is_empty() {
977                                eprint!("{text}");
978                            }
979                        }
980                    }
981                }
982            }
983        }
984        acp_thread::AcpThreadEvent::EntryUpdated(index) => {
985            let entries = acp_thread.read(cx).entries();
986            if let Some(acp_thread::AgentThreadEntry::ToolCall(tool_call)) = entries.get(*index) {
987                if let Some(name) = &tool_call.tool_name {
988                    match &tool_call.status {
989                        acp_thread::ToolCallStatus::Completed => {
990                            eprintln!("[tool] {name} ✓");
991                        }
992                        acp_thread::ToolCallStatus::Failed => {
993                            eprintln!("[tool] {name} ✗");
994                        }
995                        acp_thread::ToolCallStatus::Rejected => {
996                            eprintln!("[tool] {name} rejected");
997                        }
998                        acp_thread::ToolCallStatus::Canceled => {
999                            eprintln!("[tool] {name} canceled");
1000                        }
1001                        _ => {}
1002                    }
1003                }
1004            }
1005        }
1006        acp_thread::AcpThreadEvent::Stopped(reason) => {
1007            eprintln!("\n[eval-cli] stopped: {reason:?}");
1008        }
1009        acp_thread::AcpThreadEvent::Error => {
1010            eprintln!("[eval-cli] error event");
1011        }
1012        acp_thread::AcpThreadEvent::Retry(status) => {
1013            eprintln!("[eval-cli] retry: {status:?}");
1014        }
1015        acp_thread::AcpThreadEvent::SubagentSpawned(session_id) => {
1016            eprintln!("[eval-cli] subagent spawned: {session_id}");
1017        }
1018        _ => {}
1019    }
1020}
1021