Skip to repository content1488 lines · 56.1 KB · rust
tenant.openagents/omega
No repository description is available.
OpenAgents Git authority 2026-07-28T05:40:39.071Z Public web read
NIP-34 coordinate
30617:7649603503856e5148d571eac2766b288a8ff1e9e35d380337a1d2b0015b4f92:omegaMaintainersHidden in public view
References2 branches · 1 tag
Read-only clone
git clone https://openagents.com/git/tenant.openagents/omega.gitBrowse files
edit_file.rs
1use crate::tools::edit_file_tool::*;
2use crate::{
3 AgentTool, ContextServerRegistry, EditFileTool, GrepTool, GrepToolInput, ReadFileTool,
4 ReadFileToolInput, Template, Templates, Thread, ToolCallEventStream, ToolInput,
5};
6use Role::*;
7use anyhow::{Context as _, Result};
8use client::{Client, RefreshLlmTokenListener, UserStore};
9use fs::FakeFs;
10use futures::{FutureExt, StreamExt, future::LocalBoxFuture};
11use gpui::{AppContext as _, AsyncApp, Entity, TestAppContext, UpdateGlobal as _};
12use http_client::StatusCode;
13use language::language_settings::FormatOnSave;
14use language_model::{
15 LanguageModel, LanguageModelCompletionError, LanguageModelCompletionEvent,
16 LanguageModelRegistry, LanguageModelRequest, LanguageModelRequestMessage,
17 LanguageModelToolResult, LanguageModelToolResultContent, LanguageModelToolUse,
18 LanguageModelToolUseId, MessageContent, Role, SelectedModel,
19};
20use project::Project;
21use prompt_store::{ProjectContext, WorktreeContext};
22use rand::prelude::*;
23use reqwest_client::ReqwestClient;
24use serde::Serialize;
25use serde_json::json;
26use settings::SettingsStore;
27use std::{
28 fmt::{self, Display},
29 path::{Path, PathBuf},
30 str::FromStr,
31 sync::Arc,
32 time::Duration,
33};
34use util::path;
35
36#[derive(Serialize)]
37struct DiffJudgeTemplate {
38 diff: String,
39 assertions: &'static str,
40}
41
42impl Template for DiffJudgeTemplate {
43 const TEMPLATE_NAME: &'static str = "diff_judge.hbs";
44}
45
46#[derive(Clone)]
47struct EvalInput {
48 conversation: Vec<LanguageModelRequestMessage>,
49 input_file_path: PathBuf,
50 input_content: Option<String>,
51 assertion: EvalAssertion,
52}
53
54impl EvalInput {
55 fn new(
56 conversation: Vec<LanguageModelRequestMessage>,
57 input_file_path: impl Into<PathBuf>,
58 input_content: Option<String>,
59 assertion: EvalAssertion,
60 ) -> Self {
61 EvalInput {
62 conversation,
63 input_file_path: input_file_path.into(),
64 input_content,
65 assertion,
66 }
67 }
68}
69
70#[derive(Clone)]
71struct EvalSample {
72 text_before: String,
73 text_after: String,
74 tool_input: EditFileToolInput,
75 diff: String,
76}
77
78trait AssertionFn: 'static + Send + Sync {
79 fn assert<'a>(
80 &'a self,
81 sample: &'a EvalSample,
82 judge_model: Arc<dyn LanguageModel>,
83 cx: &'a mut TestAppContext,
84 ) -> LocalBoxFuture<'a, Result<EvalAssertionOutcome>>;
85}
86
87impl<F> AssertionFn for F
88where
89 F: 'static
90 + Send
91 + Sync
92 + AsyncFn(
93 &EvalSample,
94 Arc<dyn LanguageModel>,
95 &mut TestAppContext,
96 ) -> Result<EvalAssertionOutcome>,
97{
98 fn assert<'a>(
99 &'a self,
100 sample: &'a EvalSample,
101 judge_model: Arc<dyn LanguageModel>,
102 cx: &'a mut TestAppContext,
103 ) -> LocalBoxFuture<'a, Result<EvalAssertionOutcome>> {
104 (self)(sample, judge_model, cx).boxed_local()
105 }
106}
107
108#[derive(Clone)]
109struct EvalAssertion(Arc<dyn AssertionFn>);
110
111impl EvalAssertion {
112 fn new<F>(f: F) -> Self
113 where
114 F: 'static
115 + Send
116 + Sync
117 + AsyncFn(
118 &EvalSample,
119 Arc<dyn LanguageModel>,
120 &mut TestAppContext,
121 ) -> Result<EvalAssertionOutcome>,
122 {
123 EvalAssertion(Arc::new(f))
124 }
125
126 fn assert_diff_any(expected_diffs: Vec<impl Into<String>>) -> Self {
127 let expected_diffs: Vec<String> = expected_diffs.into_iter().map(Into::into).collect();
128 Self::new(async move |sample, _judge, _cx| {
129 let matches = expected_diffs.iter().any(|possible_diff| {
130 language::apply_diff_patch(&sample.text_before, possible_diff)
131 .map(|expected| {
132 strip_empty_lines(&expected) == strip_empty_lines(&sample.text_after)
133 })
134 .unwrap_or(false)
135 });
136
137 Ok(EvalAssertionOutcome {
138 score: if matches { 100 } else { 0 },
139 message: None,
140 })
141 })
142 }
143
144 fn judge_diff(assertions: &'static str) -> Self {
145 Self::new(async move |sample, judge, cx| {
146 let prompt = DiffJudgeTemplate {
147 diff: sample.diff.clone(),
148 assertions,
149 }
150 .render(&Templates::new())
151 .context("Failed to render diff judge template")?;
152
153 let request = LanguageModelRequest {
154 messages: vec![LanguageModelRequestMessage {
155 role: Role::User,
156 content: vec![prompt.into()],
157 cache: false,
158 reasoning_details: None,
159 }],
160 thinking_allowed: true,
161 thinking_effort: judge
162 .default_effort_level()
163 .map(|effort_level| effort_level.value.to_string()),
164 ..Default::default()
165 };
166 let mut response = retry_on_rate_limit(async || {
167 Ok(judge
168 .stream_completion_text(request.clone(), &cx.to_async())
169 .await?)
170 })
171 .await?;
172 let mut output = String::new();
173 while let Some(chunk) = response.stream.next().await {
174 let chunk = chunk?;
175 output.push_str(&chunk);
176 }
177
178 let re = regex::Regex::new(r"<score>(\d+)</score>")
179 .context("Failed to compile score regex")?;
180 if let Some(captures) = re.captures(&output)
181 && let Some(score_match) = captures.get(1)
182 {
183 let score = score_match.as_str().parse().unwrap_or(0);
184 return Ok(EvalAssertionOutcome {
185 score,
186 message: Some(output),
187 });
188 }
189
190 anyhow::bail!("No score found in response. Raw output: {output}");
191 })
192 }
193
194 async fn run(
195 &self,
196 input: &EvalSample,
197 judge_model: Arc<dyn LanguageModel>,
198 cx: &mut TestAppContext,
199 ) -> Result<EvalAssertionOutcome> {
200 self.0.assert(input, judge_model, cx).await
201 }
202}
203
204#[derive(Clone)]
205struct EditEvalOutput {
206 sample: EvalSample,
207 assertion: EvalAssertionOutcome,
208}
209
210impl Display for EditEvalOutput {
211 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
212 writeln!(f, "Score: {:?}", self.assertion.score)?;
213 if let Some(message) = self.assertion.message.as_ref() {
214 writeln!(f, "Message: {}", message)?;
215 }
216 writeln!(f, "Diff:\n{}", self.sample.diff)?;
217 writeln!(f, "Tool Input:\n{:#?}", self.sample.tool_input)?;
218 Ok(())
219 }
220}
221
222#[derive(Clone, Debug, Eq, PartialEq, Hash)]
223struct EvalAssertionOutcome {
224 score: usize,
225 message: Option<String>,
226}
227
228struct EditToolTest {
229 fs: Arc<FakeFs>,
230 project: Entity<Project>,
231 model: Arc<dyn LanguageModel>,
232 judge_model: Arc<dyn LanguageModel>,
233 model_thinking_effort: Option<String>,
234}
235
236impl EditToolTest {
237 async fn new(cx: &mut TestAppContext) -> Self {
238 cx.executor().allow_parking();
239
240 let fs = FakeFs::new(cx.executor());
241 cx.update(|cx| {
242 let settings_store = SettingsStore::test(cx);
243 cx.set_global(settings_store);
244 SettingsStore::update_global(cx, |store: &mut SettingsStore, cx| {
245 store.update_user_settings(cx, |settings| {
246 settings
247 .project
248 .all_languages
249 .defaults
250 .ensure_final_newline_on_save = Some(false);
251 settings.project.all_languages.defaults.format_on_save =
252 Some(FormatOnSave::Off);
253 });
254 });
255
256 gpui_tokio::init(cx);
257 let http_client = Arc::new(ReqwestClient::user_agent("agent tests").unwrap());
258 cx.set_http_client(http_client);
259 let client = Client::production(cx);
260 let user_store = cx.new(|cx| UserStore::new(client.clone(), cx));
261 language_model::init(cx);
262 RefreshLlmTokenListener::register(client.clone(), user_store.clone(), cx);
263 language_models::init(user_store, client, cx);
264 });
265
266 fs.insert_tree("/root", json!({})).await;
267 let project = Project::test(fs.clone(), [path!("/root").as_ref()], cx).await;
268 let agent_model = SelectedModel::from_str(
269 &std::env::var("ZED_AGENT_MODEL")
270 .unwrap_or("anthropic/claude-sonnet-4-6-latest".into()),
271 )
272 .unwrap();
273 let judge_model = SelectedModel::from_str(
274 &std::env::var("ZED_JUDGE_MODEL")
275 .unwrap_or("anthropic/claude-sonnet-4-6-latest".into()),
276 )
277 .unwrap();
278
279 let authenticate_provider_tasks = cx.update(|cx| {
280 LanguageModelRegistry::global(cx).update(cx, |registry, cx| {
281 registry
282 .providers()
283 .iter()
284 .map(|p| p.authenticate(cx))
285 .collect::<Vec<_>>()
286 })
287 });
288 let (model, judge_model) = cx
289 .update(|cx| {
290 cx.spawn(async move |cx| {
291 futures::future::join_all(authenticate_provider_tasks).await;
292 let model = Self::load_model(&agent_model, cx).await;
293 let judge_model = Self::load_model(&judge_model, cx).await;
294 (model.unwrap(), judge_model.unwrap())
295 })
296 })
297 .await;
298
299 let model_thinking_effort = model
300 .default_effort_level()
301 .map(|effort_level| effort_level.value.to_string());
302
303 Self {
304 fs,
305 project,
306 model,
307 judge_model,
308 model_thinking_effort,
309 }
310 }
311
312 async fn load_model(
313 selected_model: &SelectedModel,
314 cx: &mut AsyncApp,
315 ) -> Result<Arc<dyn LanguageModel>> {
316 cx.update(|cx| {
317 let registry = LanguageModelRegistry::read_global(cx);
318 let provider = registry
319 .provider(&selected_model.provider)
320 .expect("Provider not found");
321 provider.authenticate(cx)
322 })
323 .await?;
324 Ok(cx.update(|cx| {
325 let models = LanguageModelRegistry::read_global(cx);
326 models
327 .available_models(cx)
328 .find(|model| {
329 model.provider_id() == selected_model.provider
330 && model.id() == selected_model.model
331 })
332 .unwrap_or_else(|| panic!("Model {} not found", selected_model.model.0))
333 }))
334 }
335
336 async fn eval(&self, mut eval: EvalInput, cx: &mut TestAppContext) -> Result<EditEvalOutput> {
337 eval.conversation
338 .last_mut()
339 .context("Conversation must not be empty")?
340 .cache = true;
341
342 // Populate the FakeFs so `resolve_path` / `entry_for_path` can find
343 // the file in the worktree.
344 if let Some(input_content) = eval.input_content.as_deref() {
345 let abs_path = Path::new("/root").join(
346 eval.input_file_path
347 .strip_prefix("root")
348 .unwrap_or(&eval.input_file_path),
349 );
350 self.fs.insert_file(&abs_path, input_content.into()).await;
351
352 // Wait for the worktree to pick up the new file.
353 cx.run_until_parked();
354 }
355
356 let tools = crate::built_in_tools().collect::<Vec<_>>();
357
358 let system_prompt = {
359 let worktrees = vec![WorktreeContext {
360 root_name: "root".to_string(),
361 abs_path: Path::new("/path/to/root").into(),
362 rules_file: None,
363 }];
364 let project_context = ProjectContext::new(worktrees);
365 let tool_names = tools
366 .iter()
367 .map(|tool| tool.name.clone().into())
368 .collect::<Vec<_>>();
369 let template = crate::SystemPromptTemplate {
370 project: &project_context,
371 available_tools: tool_names,
372 model_name: None,
373 date: chrono::Local::now().format("%Y-%m-%d").to_string(),
374 user_agents_md: None,
375 sandboxing: false,
376 is_linux: cfg!(target_os = "linux"),
377 is_windows: cfg!(target_os = "windows"),
378 };
379 let templates = Templates::new();
380 template.render(&templates)?
381 };
382
383 let has_system_prompt = eval
384 .conversation
385 .first()
386 .is_some_and(|msg| msg.role == Role::System);
387 let messages = if has_system_prompt {
388 eval.conversation
389 } else {
390 [LanguageModelRequestMessage {
391 role: Role::System,
392 content: vec![MessageContent::Text(system_prompt)],
393 cache: true,
394 reasoning_details: None,
395 }]
396 .into_iter()
397 .chain(eval.conversation)
398 .collect::<Vec<_>>()
399 };
400
401 let request = LanguageModelRequest {
402 messages,
403 tools,
404 thinking_allowed: true,
405 thinking_effort: self.model_thinking_effort.clone(),
406 ..Default::default()
407 };
408
409 // The model will call the tool as "edit_file" (the production-visible
410 // name), but the schema is from EditFileTool.
411 let tool_input =
412 retry_on_rate_limit(async || self.extract_tool_use(request.clone(), cx).await).await?;
413
414 let language_registry = self
415 .project
416 .read_with(cx, |project, _cx| project.languages().clone());
417
418 let context_server_registry = cx
419 .new(|cx| ContextServerRegistry::new(self.project.read(cx).context_server_store(), cx));
420 let thread = cx.new(|cx| {
421 Thread::new(
422 self.project.clone(),
423 cx.new(|_cx| ProjectContext::default()),
424 context_server_registry,
425 Templates::new(),
426 Some(self.model.clone()),
427 cx,
428 )
429 });
430 let action_log = thread.read_with(cx, |thread, _| thread.action_log().clone());
431
432 let tool = Arc::new(EditFileTool::new(
433 self.project.clone(),
434 thread.downgrade(),
435 action_log,
436 language_registry,
437 ));
438
439 let result = cx
440 .update(|cx| {
441 tool.clone().run(
442 ToolInput::resolved(tool_input.clone()),
443 ToolCallEventStream::test().0,
444 cx,
445 )
446 })
447 .await;
448
449 let output = match result {
450 Ok(output) => output,
451 Err(output) => {
452 anyhow::bail!("Tool returned error: {}", output);
453 }
454 };
455
456 let EditFileToolOutput::Success { new_text, .. } = &output else {
457 anyhow::bail!("Tool returned error output: {}", output);
458 };
459
460 let sample = EvalSample {
461 tool_input,
462 diff: language::unified_diff(
463 eval.input_content.as_deref().unwrap_or_default(),
464 new_text,
465 ),
466 text_before: eval.input_content.unwrap_or_default(),
467 text_after: new_text.clone(),
468 };
469
470 let assertion = eval
471 .assertion
472 .run(&sample, self.judge_model.clone(), cx)
473 .await?;
474
475 Ok(EditEvalOutput { assertion, sample })
476 }
477
478 /// Stream the model completion and extract the first complete tool use
479 /// whose name matches `EditFileTool::NAME`, parsed as `EditFileToolInput`.
480 async fn extract_tool_use(
481 &self,
482 request: LanguageModelRequest,
483 cx: &mut TestAppContext,
484 ) -> Result<EditFileToolInput> {
485 let model = self.model.clone();
486 let events = cx
487 .update(|cx| {
488 let async_cx = cx.to_async();
489 cx.foreground_executor()
490 .spawn(async move { model.stream_completion(request, &async_cx).await })
491 })
492 .await
493 .map_err(|err| anyhow::anyhow!("completion error: {}", err))?;
494
495 let mut streamed_text = String::new();
496 let mut stop_reason = None;
497 let mut parse_errors = Vec::new();
498
499 let mut events = events.fuse();
500 while let Some(event) = events.next().await {
501 match event {
502 Ok(LanguageModelCompletionEvent::ToolUse(tool_use))
503 if tool_use.is_input_complete
504 && tool_use.name.as_ref() == EditFileTool::NAME =>
505 {
506 let input: EditFileToolInput = tool_use
507 .input
508 .parse()
509 .context("Failed to parse tool input as EditFileToolInput")?;
510 return Ok(input);
511 }
512 Ok(LanguageModelCompletionEvent::Text(text)) => {
513 if streamed_text.len() < 2_000 {
514 streamed_text.push_str(&text);
515 }
516 }
517 Ok(LanguageModelCompletionEvent::Stop(reason)) => {
518 stop_reason = Some(reason);
519 }
520 Ok(LanguageModelCompletionEvent::ToolUseJsonParseError {
521 tool_name,
522 raw_input,
523 json_parse_error,
524 ..
525 }) if tool_name.as_ref() == EditFileTool::NAME => {
526 parse_errors.push(format!("{json_parse_error}\nRaw input:\n{raw_input:?}"));
527 }
528 Err(err) => {
529 return Err(anyhow::anyhow!("completion error: {}", err));
530 }
531 _ => {}
532 }
533 }
534
535 let streamed_text = streamed_text.trim();
536 let streamed_text_suffix = if streamed_text.is_empty() {
537 String::new()
538 } else {
539 format!("\nStreamed text:\n{streamed_text}")
540 };
541 let stop_reason_suffix = stop_reason
542 .map(|reason| format!("\nStop reason: {reason:?}"))
543 .unwrap_or_default();
544 let parse_errors_suffix = if parse_errors.is_empty() {
545 String::new()
546 } else {
547 format!("\nTool parse errors:\n{}", parse_errors.join("\n"))
548 };
549
550 anyhow::bail!(
551 "Stream ended without an edit_file tool use{stop_reason_suffix}{parse_errors_suffix}{streamed_text_suffix}"
552 )
553 }
554}
555
556fn run_eval(eval: EvalInput) -> eval_utils::EvalOutput<()> {
557 super::run_gpui_eval(
558 |cx| {
559 async move {
560 let test = EditToolTest::new(cx).await;
561 let result = test.eval(eval, cx).await;
562 drop(test);
563 cx.run_until_parked();
564 result
565 }
566 .boxed_local()
567 },
568 |output| {
569 if output.assertion.score < 80 {
570 eval_utils::OutcomeKind::Failed
571 } else {
572 eval_utils::OutcomeKind::Passed
573 }
574 },
575 )
576}
577
578fn message(
579 role: Role,
580 contents: impl IntoIterator<Item = MessageContent>,
581) -> LanguageModelRequestMessage {
582 LanguageModelRequestMessage {
583 role,
584 content: contents.into_iter().collect(),
585 cache: false,
586 reasoning_details: None,
587 }
588}
589
590fn text(text: impl Into<String>) -> MessageContent {
591 MessageContent::Text(text.into())
592}
593
594fn lines(input: &str, range: std::ops::Range<usize>) -> String {
595 input
596 .lines()
597 .skip(range.start)
598 .take(range.len())
599 .collect::<Vec<_>>()
600 .join("\n")
601}
602
603fn tool_use(
604 id: impl Into<Arc<str>>,
605 name: impl Into<Arc<str>>,
606 input: impl Serialize,
607) -> MessageContent {
608 MessageContent::ToolUse(LanguageModelToolUse {
609 id: LanguageModelToolUseId::from(id.into()),
610 name: name.into(),
611 raw_input: serde_json::to_string_pretty(&input).unwrap(),
612 input: language_model::LanguageModelToolUseInput::Json(
613 serde_json::to_value(input).unwrap(),
614 ),
615 is_input_complete: true,
616 thought_signature: None,
617 })
618}
619
620fn tool_result(
621 id: impl Into<Arc<str>>,
622 name: impl Into<Arc<str>>,
623 result: impl Into<Arc<str>>,
624) -> MessageContent {
625 MessageContent::ToolResult(LanguageModelToolResult {
626 tool_use_id: LanguageModelToolUseId::from(id.into()),
627 tool_name: name.into(),
628 is_error: false,
629 content: vec![LanguageModelToolResultContent::Text(result.into())],
630 output: None,
631 })
632}
633
634fn strip_empty_lines(text: &str) -> String {
635 text.lines()
636 .filter(|line| !line.trim().is_empty())
637 .collect::<Vec<_>>()
638 .join("\n")
639}
640
641async fn retry_on_rate_limit<R>(mut request: impl AsyncFnMut() -> Result<R>) -> Result<R> {
642 const MAX_RETRIES: usize = 20;
643 let mut attempt = 0;
644
645 loop {
646 attempt += 1;
647 let response = request().await;
648
649 if attempt >= MAX_RETRIES {
650 return response;
651 }
652
653 let retry_delay = match &response {
654 Ok(_) => None,
655 Err(err) => match err.downcast_ref::<LanguageModelCompletionError>() {
656 Some(err) => match &err {
657 LanguageModelCompletionError::RateLimitExceeded { retry_after, .. }
658 | LanguageModelCompletionError::ServerOverloaded { retry_after, .. } => {
659 Some(retry_after.unwrap_or(Duration::from_secs(5)))
660 }
661 LanguageModelCompletionError::UpstreamProviderError {
662 status,
663 retry_after,
664 ..
665 } => {
666 let should_retry = matches!(
667 *status,
668 StatusCode::TOO_MANY_REQUESTS | StatusCode::SERVICE_UNAVAILABLE
669 ) || status.as_u16() == 529;
670
671 if should_retry {
672 Some(retry_after.unwrap_or(Duration::from_secs(5)))
673 } else {
674 None
675 }
676 }
677 LanguageModelCompletionError::ApiReadResponseError { .. }
678 | LanguageModelCompletionError::ApiInternalServerError { .. }
679 | LanguageModelCompletionError::HttpSend { .. } => {
680 Some(Duration::from_secs(2_u64.pow((attempt - 1) as u32).min(30)))
681 }
682 _ => None,
683 },
684 _ => None,
685 },
686 };
687
688 if let Some(retry_after) = retry_delay {
689 let jitter = retry_after.mul_f64(rand::rng().random_range(0.0..1.0));
690 eprintln!("Attempt #{attempt}: Retry after {retry_after:?} + jitter of {jitter:?}");
691 #[allow(clippy::disallowed_methods)]
692 async_io::Timer::after(retry_after + jitter).await;
693 } else {
694 return response;
695 }
696 }
697}
698
699#[test]
700#[cfg_attr(not(feature = "unit-eval"), ignore)]
701fn eval_delete_function() {
702 let input_file_path = "root/blame.rs";
703 let input_file_content = include_str!("fixtures/delete_run_git_blame/before.rs");
704 let output_file_content = include_str!("fixtures/delete_run_git_blame/after.rs");
705 let possible_diffs = vec![
706 language::unified_diff(input_file_content, output_file_content),
707 language::unified_diff(
708 input_file_content,
709 &output_file_content
710 .replace(
711 "const GIT_BLAME_NO_COMMIT_ERROR: &str = \"fatal: no such ref: HEAD\";\n",
712 "",
713 )
714 .replace(
715 "const GIT_BLAME_NO_PATH: &str = \"fatal: no such path\";\n",
716 "",
717 ),
718 ),
719 ];
720
721 eval_utils::eval(100, 0.95, eval_utils::NoProcessor, move || {
722 run_eval(EvalInput::new(
723 vec![
724 message(
725 User,
726 [text(indoc::formatdoc! {"
727 Read the `{input_file_path}` file and delete `run_git_blame`. Just that
728 one function, not its usages.
729 "})],
730 ),
731 message(
732 Assistant,
733 [tool_use(
734 "tool_1",
735 ReadFileTool::NAME,
736 ReadFileToolInput {
737 path: input_file_path.into(),
738 start_line: None,
739 end_line: None,
740 },
741 )],
742 ),
743 message(
744 User,
745 [tool_result(
746 "tool_1",
747 ReadFileTool::NAME,
748 input_file_content,
749 )],
750 ),
751 ],
752 input_file_path,
753 Some(input_file_content.into()),
754 EvalAssertion::assert_diff_any(possible_diffs.clone()),
755 ))
756 });
757}
758
759#[test]
760#[cfg_attr(not(feature = "unit-eval"), ignore)]
761fn eval_extract_handle_command_output() {
762 let input_file_path = "root/blame.rs";
763 let input_file_content = include_str!("fixtures/extract_handle_command_output/before.rs");
764 let possible_diffs = vec![
765 include_str!("fixtures/extract_handle_command_output/possible-01.diff"),
766 include_str!("fixtures/extract_handle_command_output/possible-02.diff"),
767 include_str!("fixtures/extract_handle_command_output/possible-03.diff"),
768 include_str!("fixtures/extract_handle_command_output/possible-04.diff"),
769 include_str!("fixtures/extract_handle_command_output/possible-05.diff"),
770 include_str!("fixtures/extract_handle_command_output/possible-06.diff"),
771 include_str!("fixtures/extract_handle_command_output/possible-07.diff"),
772 include_str!("fixtures/extract_handle_command_output/possible-08.diff"),
773 include_str!("fixtures/extract_handle_command_output/possible-09.diff"),
774 ];
775
776 eval_utils::eval(100, 0.95, eval_utils::NoProcessor, move || {
777 run_eval(EvalInput::new(
778 vec![
779 message(
780 User,
781 [text(indoc::formatdoc! {"
782 Read the `{input_file_path}` file and extract a method in
783 the final stanza of `run_git_blame` to deal with command failures,
784 call it `handle_command_output` and take the std::process::Output as the only parameter.
785 Do not document the method and do not add any comments.
786
787 Add it right next to `run_git_blame` and copy it verbatim from `run_git_blame`.
788 "})],
789 ),
790 message(
791 Assistant,
792 [tool_use(
793 "tool_1",
794 ReadFileTool::NAME,
795 ReadFileToolInput {
796 path: input_file_path.into(),
797 start_line: None,
798 end_line: None,
799 },
800 )],
801 ),
802 message(
803 User,
804 [tool_result(
805 "tool_1",
806 ReadFileTool::NAME,
807 input_file_content,
808 )],
809 ),
810 ],
811 input_file_path,
812 Some(input_file_content.into()),
813 EvalAssertion::assert_diff_any(possible_diffs.clone()),
814 ))
815 });
816}
817
818#[test]
819#[cfg_attr(not(feature = "unit-eval"), ignore)]
820fn eval_translate_doc_comments() {
821 let input_file_path = "root/canvas.rs";
822 let input_file_content = include_str!("fixtures/translate_doc_comments/before.rs");
823
824 eval_utils::eval(200, 1., eval_utils::NoProcessor, move || {
825 run_eval(EvalInput::new(
826 vec![
827 message(
828 User,
829 [text(indoc::formatdoc! {"
830 Read the `{input_file_path}` file and edit it (without overwriting it),
831 translating all the doc comments to italian.
832 "})],
833 ),
834 message(
835 Assistant,
836 [tool_use(
837 "tool_1",
838 ReadFileTool::NAME,
839 ReadFileToolInput {
840 path: input_file_path.into(),
841 start_line: None,
842 end_line: None,
843 },
844 )],
845 ),
846 message(
847 User,
848 [tool_result(
849 "tool_1",
850 ReadFileTool::NAME,
851 input_file_content,
852 )],
853 ),
854 ],
855 input_file_path,
856 Some(input_file_content.into()),
857 EvalAssertion::judge_diff("Doc comments were translated to Italian"),
858 ))
859 });
860}
861
862#[test]
863#[cfg_attr(not(feature = "unit-eval"), ignore)]
864fn eval_use_wasi_sdk_in_compile_parser_to_wasm() {
865 let input_file_path = "root/lib.rs";
866 let input_file_content =
867 include_str!("fixtures/use_wasi_sdk_in_compile_parser_to_wasm/before.rs");
868
869 eval_utils::eval(100, 0.95, eval_utils::NoProcessor, move || {
870 run_eval(EvalInput::new(
871 vec![
872 message(
873 User,
874 [text(indoc::formatdoc! {"
875 Read the `{input_file_path}` file and change `compile_parser_to_wasm` to use `wasi-sdk` instead of emscripten.
876 Use `ureq` to download the SDK for the current platform and architecture.
877 Extract the archive into a sibling of `lib` inside the `tree-sitter` directory in the cache_dir.
878 Compile the parser to wasm using the `bin/clang` executable (or `bin/clang.exe` on windows)
879 that's inside of the archive.
880 Don't re-download the SDK if that executable already exists.
881
882 Use these clang flags: -fPIC -shared -Os -Wl,--export=tree_sitter_{{language_name}}
883
884 Here are the available wasi-sdk assets:
885 - wasi-sdk-25.0-x86_64-macos.tar.gz
886 - wasi-sdk-25.0-arm64-macos.tar.gz
887 - wasi-sdk-25.0-x86_64-linux.tar.gz
888 - wasi-sdk-25.0-arm64-linux.tar.gz
889 - wasi-sdk-25.0-x86_64-linux.tar.gz
890 - wasi-sdk-25.0-arm64-linux.tar.gz
891 - wasi-sdk-25.0-x86_64-windows.tar.gz
892 "})],
893 ),
894 message(
895 Assistant,
896 [tool_use(
897 "tool_1",
898 ReadFileTool::NAME,
899 ReadFileToolInput {
900 path: input_file_path.into(),
901 start_line: Some(971),
902 end_line: Some(1050),
903 },
904 )],
905 ),
906 message(
907 User,
908 [tool_result(
909 "tool_1",
910 ReadFileTool::NAME,
911 lines(input_file_content, 971..1050),
912 )],
913 ),
914 message(
915 Assistant,
916 [tool_use(
917 "tool_2",
918 ReadFileTool::NAME,
919 ReadFileToolInput {
920 path: input_file_path.into(),
921 start_line: Some(1050),
922 end_line: Some(1100),
923 },
924 )],
925 ),
926 message(
927 User,
928 [tool_result(
929 "tool_2",
930 ReadFileTool::NAME,
931 lines(input_file_content, 1050..1100),
932 )],
933 ),
934 message(
935 Assistant,
936 [tool_use(
937 "tool_3",
938 ReadFileTool::NAME,
939 ReadFileToolInput {
940 path: input_file_path.into(),
941 start_line: Some(1100),
942 end_line: Some(1150),
943 },
944 )],
945 ),
946 message(
947 User,
948 [tool_result(
949 "tool_3",
950 ReadFileTool::NAME,
951 lines(input_file_content, 1100..1150),
952 )],
953 ),
954 ],
955 input_file_path,
956 Some(input_file_content.into()),
957 EvalAssertion::judge_diff(indoc::indoc! {"
958 - The compile_parser_to_wasm method has been changed to use wasi-sdk
959 - ureq is used to download the SDK for current platform and architecture
960 "}),
961 ))
962 });
963}
964
965#[test]
966#[cfg_attr(not(feature = "unit-eval"), ignore)]
967fn eval_disable_cursor_blinking() {
968 let input_file_path = "root/editor.rs";
969 let input_file_content = include_str!("fixtures/disable_cursor_blinking/before.rs");
970 let possible_diffs = vec![
971 include_str!("fixtures/disable_cursor_blinking/possible-01.diff"),
972 include_str!("fixtures/disable_cursor_blinking/possible-02.diff"),
973 include_str!("fixtures/disable_cursor_blinking/possible-03.diff"),
974 include_str!("fixtures/disable_cursor_blinking/possible-04.diff"),
975 ];
976
977 eval_utils::eval(100, 0.51, eval_utils::NoProcessor, move || {
978 run_eval(EvalInput::new(
979 vec![
980 message(User, [text("Let's research how to cursor blinking works.")]),
981 message(
982 Assistant,
983 [tool_use(
984 "tool_1",
985 GrepTool::NAME,
986 GrepToolInput {
987 regex: "blink".into(),
988 include_pattern: None,
989 offset: 0,
990 case_sensitive: false,
991 },
992 )],
993 ),
994 message(
995 User,
996 [tool_result(
997 "tool_1",
998 GrepTool::NAME,
999 [
1000 lines(input_file_content, 100..400),
1001 lines(input_file_content, 800..1300),
1002 lines(input_file_content, 1600..2000),
1003 lines(input_file_content, 5000..5500),
1004 lines(input_file_content, 8000..9000),
1005 lines(input_file_content, 18455..18470),
1006 lines(input_file_content, 20000..20500),
1007 lines(input_file_content, 21000..21300),
1008 ]
1009 .join("Match found:\n\n"),
1010 )],
1011 ),
1012 message(
1013 User,
1014 [text(indoc::indoc! {"
1015 Comment out the lines that interact with the BlinkManager.
1016 Keep the outer `update` blocks, but comments everything that's inside (including if statements).
1017 Don't add additional comments.
1018 "})],
1019 ),
1020 ],
1021 input_file_path,
1022 Some(input_file_content.into()),
1023 EvalAssertion::assert_diff_any(possible_diffs.clone()),
1024 ))
1025 });
1026}
1027
1028#[test]
1029#[cfg_attr(not(feature = "unit-eval"), ignore)]
1030fn eval_from_pixels_constructor() {
1031 let input_file_path = "root/canvas.rs";
1032 let input_file_content = include_str!("fixtures/from_pixels_constructor/before.rs");
1033
1034 eval_utils::eval(100, 0.95, eval_utils::NoProcessor, move || {
1035 run_eval(EvalInput::new(
1036 vec![
1037 message(
1038 User,
1039 [text(indoc::indoc! {"
1040 Introduce a new `from_pixels` constructor in Canvas and
1041 also add tests for it in the same file.
1042 "})],
1043 ),
1044 message(
1045 Assistant,
1046 [tool_use(
1047 "tool_1",
1048 ReadFileTool::NAME,
1049 ReadFileToolInput {
1050 path: input_file_path.into(),
1051 start_line: None,
1052 end_line: None,
1053 },
1054 )],
1055 ),
1056 message(
1057 User,
1058 [tool_result(
1059 "tool_1",
1060 ReadFileTool::NAME,
1061 input_file_content,
1062 )],
1063 ),
1064 message(
1065 Assistant,
1066 [tool_use(
1067 "tool_2",
1068 GrepTool::NAME,
1069 GrepToolInput {
1070 regex: "mod\\s+tests".into(),
1071 include_pattern: Some("font-kit/src/canvas.rs".into()),
1072 offset: 0,
1073 case_sensitive: false,
1074 },
1075 )],
1076 ),
1077 message(
1078 User,
1079 [tool_result("tool_2", GrepTool::NAME, "No matches found")],
1080 ),
1081 message(
1082 Assistant,
1083 [tool_use(
1084 "tool_3",
1085 GrepTool::NAME,
1086 GrepToolInput {
1087 regex: "mod\\s+tests".into(),
1088 include_pattern: Some("font-kit/src/**/*.rs".into()),
1089 offset: 0,
1090 case_sensitive: false,
1091 },
1092 )],
1093 ),
1094 message(
1095 User,
1096 [tool_result("tool_3", GrepTool::NAME, "No matches found")],
1097 ),
1098 message(
1099 Assistant,
1100 [tool_use(
1101 "tool_4",
1102 GrepTool::NAME,
1103 GrepToolInput {
1104 regex: "#\\[test\\]".into(),
1105 include_pattern: Some("font-kit/src/**/*.rs".into()),
1106 offset: 0,
1107 case_sensitive: false,
1108 },
1109 )],
1110 ),
1111 message(
1112 User,
1113 [tool_result(
1114 "tool_4",
1115 GrepTool::NAME,
1116 indoc::indoc! {"
1117 Found 6 matches:
1118
1119 ## Matches in font-kit/src/loaders/core_text.rs
1120
1121 ### mod test › L926-936
1122 ```
1123 mod test {
1124 use super::Font;
1125 use crate::properties::{Stretch, Weight};
1126
1127 #[cfg(feature = \"source\")]
1128 use crate::source::SystemSource;
1129
1130 static TEST_FONT_POSTSCRIPT_NAME: &'static str = \"ArialMT\";
1131
1132 #[cfg(feature = \"source\")]
1133 #[test]
1134 ```
1135
1136 55 lines remaining in ancestor node. Read the file to see all.
1137
1138 ### mod test › L947-951
1139 ```
1140 }
1141
1142 #[test]
1143 fn test_core_text_to_css_font_weight() {
1144 // Exact matches
1145 ```
1146
1147 ### mod test › L959-963
1148 ```
1149 }
1150
1151 #[test]
1152 fn test_core_text_to_css_font_stretch() {
1153 // Exact matches
1154 ```
1155
1156 ## Matches in font-kit/src/loaders/freetype.rs
1157
1158 ### mod test › L1238-1248
1159 ```
1160 mod test {
1161 use crate::loaders::freetype::Font;
1162
1163 static PCF_FONT_PATH: &str = \"resources/tests/times-roman-pcf/timR12.pcf\";
1164 static PCF_FONT_POSTSCRIPT_NAME: &str = \"Times-Roman\";
1165
1166 #[test]
1167 fn get_pcf_postscript_name() {
1168 let font = Font::from_path(PCF_FONT_PATH, 0).unwrap();
1169 assert_eq!(font.postscript_name().unwrap(), PCF_FONT_POSTSCRIPT_NAME);
1170 }
1171 ```
1172
1173 1 lines remaining in ancestor node. Read the file to see all.
1174
1175 ## Matches in font-kit/src/sources/core_text.rs
1176
1177 ### mod test › L265-275
1178 ```
1179 mod test {
1180 use crate::properties::{Stretch, Weight};
1181
1182 #[test]
1183 fn test_css_to_core_text_font_weight() {
1184 // Exact matches
1185 assert_eq!(super::css_to_core_text_font_weight(Weight(100.0)), -0.7);
1186 assert_eq!(super::css_to_core_text_font_weight(Weight(400.0)), 0.0);
1187 assert_eq!(super::css_to_core_text_font_weight(Weight(700.0)), 0.4);
1188 assert_eq!(super::css_to_core_text_font_weight(Weight(900.0)), 0.8);
1189
1190 ```
1191
1192 27 lines remaining in ancestor node. Read the file to see all.
1193
1194 ### mod test › L278-282
1195 ```
1196 }
1197
1198 #[test]
1199 fn test_css_to_core_text_font_stretch() {
1200 // Exact matches
1201 ```
1202 "},
1203 )],
1204 ),
1205 ],
1206 input_file_path,
1207 Some(input_file_content.into()),
1208 EvalAssertion::judge_diff(indoc::indoc! {"
1209 - The diff contains a new `from_pixels` constructor
1210 - The diff contains new tests for the `from_pixels` constructor
1211 "}),
1212 ))
1213 });
1214}
1215
1216#[test]
1217#[cfg_attr(not(feature = "unit-eval"), ignore)]
1218fn eval_zode() {
1219 let input_file_path = "root/zode.py";
1220 let input_content = None;
1221
1222 eval_utils::eval(50, 1., eval_utils::NoProcessor, move || {
1223 run_eval(EvalInput::new(
1224 vec![
1225 message(User, [text(include_str!("fixtures/zode/prompt.md"))]),
1226 message(
1227 Assistant,
1228 [
1229 tool_use(
1230 "tool_1",
1231 ReadFileTool::NAME,
1232 ReadFileToolInput {
1233 path: "root/eval/react.py".into(),
1234 start_line: None,
1235 end_line: None,
1236 },
1237 ),
1238 tool_use(
1239 "tool_2",
1240 ReadFileTool::NAME,
1241 ReadFileToolInput {
1242 path: "root/eval/react_test.py".into(),
1243 start_line: None,
1244 end_line: None,
1245 },
1246 ),
1247 ],
1248 ),
1249 message(
1250 User,
1251 [
1252 tool_result(
1253 "tool_1",
1254 ReadFileTool::NAME,
1255 include_str!("fixtures/zode/react.py"),
1256 ),
1257 tool_result(
1258 "tool_2",
1259 ReadFileTool::NAME,
1260 include_str!("fixtures/zode/react_test.py"),
1261 ),
1262 ],
1263 ),
1264 ],
1265 input_file_path,
1266 input_content.clone(),
1267 EvalAssertion::new(async move |sample, _, _cx| {
1268 let invalid_starts = [' ', '`', '\n'];
1269 let mut message = String::new();
1270 for start in invalid_starts {
1271 if sample.text_after.starts_with(start) {
1272 message.push_str(&format!("The sample starts with a {:?}\n", start));
1273 break;
1274 }
1275 }
1276 message.pop();
1277
1278 if message.is_empty() {
1279 Ok(EvalAssertionOutcome {
1280 score: 100,
1281 message: None,
1282 })
1283 } else {
1284 Ok(EvalAssertionOutcome {
1285 score: 0,
1286 message: Some(message),
1287 })
1288 }
1289 }),
1290 ))
1291 });
1292}
1293
1294#[test]
1295#[cfg_attr(not(feature = "unit-eval"), ignore)]
1296fn eval_add_overwrite_test() {
1297 let input_file_path = "root/action_log.rs";
1298 let input_file_content = include_str!("fixtures/add_overwrite_test/before.rs");
1299
1300 eval_utils::eval(200, 0.5, eval_utils::NoProcessor, move || {
1301 run_eval(EvalInput::new(
1302 vec![
1303 message(
1304 User,
1305 [text(indoc::indoc! {"
1306 Introduce a new test in `action_log.rs` to test overwriting a file.
1307 That is, a file already exists, but we call `buffer_created` as if the file were new.
1308 Take inspiration from all the other tests in the file.
1309 "})],
1310 ),
1311 message(
1312 Assistant,
1313 [tool_use(
1314 "tool_1",
1315 ReadFileTool::NAME,
1316 ReadFileToolInput {
1317 path: input_file_path.into(),
1318 start_line: None,
1319 end_line: None,
1320 },
1321 )],
1322 ),
1323 message(
1324 User,
1325 [tool_result(
1326 "tool_1",
1327 ReadFileTool::NAME,
1328 indoc::indoc! {"
1329 pub struct ActionLog [L13-20]
1330 tracked_buffers [L15]
1331 edited_since_project_diagnostics_check [L17]
1332 project [L19]
1333 impl ActionLog [L22-498]
1334 pub fn new [L24-30]
1335 pub fn project [L32-34]
1336 pub fn checked_project_diagnostics [L37-39]
1337 pub fn has_edited_files_since_project_diagnostics_check [L42-44]
1338 fn track_buffer_internal [L46-101]
1339 fn handle_buffer_event [L103-116]
1340 fn handle_buffer_edited [L118-123]
1341 fn handle_buffer_file_changed [L125-158]
1342 async fn maintain_diff [L160-264]
1343 pub fn buffer_read [L267-269]
1344 pub fn buffer_created [L272-276]
1345 pub fn buffer_edited [L279-287]
1346 pub fn will_delete_buffer [L289-304]
1347 pub fn keep_edits_in_range [L306-364]
1348 pub fn reject_edits_in_ranges [L366-459]
1349 pub fn keep_all_edits [L461-473]
1350 pub fn changed_buffers [L476-482]
1351 pub fn stale_buffers [L485-497]
1352 fn apply_non_conflicting_edits [L500-561]
1353 fn diff_snapshots [L563-585]
1354 fn point_to_row_edit [L587-614]
1355 enum ChangeAuthor [L617-620]
1356 User [L618]
1357 Agent [L619]
1358 enum TrackedBufferStatus [L623-627]
1359 Created [L624]
1360 Modified [L625]
1361 Deleted [L626]
1362 struct TrackedBuffer [L629-641]
1363 buffer [L630]
1364 base_text [L631]
1365 unreviewed_changes [L632]
1366 status [L633]
1367 version [L634]
1368 diff [L635]
1369 snapshot [L636]
1370 diff_update [L637]
1371 _open_lsp_handle [L638]
1372 _maintain_diff [L639]
1373 _subscription [L640]
1374 impl TrackedBuffer [L643-657]
1375 fn has_changes [L644-650]
1376 fn schedule_diff_update [L652-656]
1377 pub struct ChangedBuffer [L659-661]
1378 pub diff [L660]
1379 mod tests [L664-1574]
1380 fn init_logger [L678-682]
1381 fn init_test [L684-691]
1382 async fn test_keep_edits [L694-769]
1383 async fn test_deletions [L772-854]
1384 async fn test_overlapping_user_edits [L857-951]
1385 async fn test_creating_files [L954-1010]
1386 async fn test_deleting_files [L1013-1120]
1387 async fn test_reject_edits [L1123-1255]
1388 async fn test_reject_multiple_edits [L1258-1331]
1389 async fn test_reject_deleted_file [L1334-1388]
1390 async fn test_reject_created_file [L1391-1443]
1391 async fn test_random_diffs [L1446-1535]
1392 fn quiesce [L1510-1534]
1393 struct HunkStatus [L1538-1542]
1394 range [L1539]
1395 diff_status [L1540]
1396 old_text [L1541]
1397 fn unreviewed_hunks [L1544-1573]
1398
1399 Showing symbols 1-69 (total symbols: 69)
1400
1401 Using the line numbers in this outline, you can call this tool again while specifying
1402 the start_line and end_line fields to see the implementations of symbols in the outline.
1403 "},
1404 )],
1405 ),
1406 message(
1407 Assistant,
1408 [
1409 text(
1410 "Let's look at the existing test cases that are related to file creation to understand the pattern:",
1411 ),
1412 tool_use(
1413 "tool_2",
1414 ReadFileTool::NAME,
1415 ReadFileToolInput {
1416 path: input_file_path.into(),
1417 start_line: Some(953),
1418 end_line: Some(1010),
1419 },
1420 ),
1421 ],
1422 ),
1423 message(
1424 User,
1425 [tool_result(
1426 "tool_2",
1427 ReadFileTool::NAME,
1428 lines(input_file_content, 953..1010),
1429 )],
1430 ),
1431 message(
1432 Assistant,
1433 [
1434 text(
1435 "Now let's look at another relevant test to understand how they're structured:",
1436 ),
1437 tool_use(
1438 "tool_3",
1439 ReadFileTool::NAME,
1440 ReadFileToolInput {
1441 path: input_file_path.into(),
1442 start_line: Some(1012),
1443 end_line: Some(1120),
1444 },
1445 ),
1446 ],
1447 ),
1448 message(
1449 User,
1450 [tool_result(
1451 "tool_3",
1452 ReadFileTool::NAME,
1453 lines(input_file_content, 1012..1120),
1454 )],
1455 ),
1456 message(
1457 Assistant,
1458 [
1459 text("Now let's look at how `buffer_created` is implemented:"),
1460 tool_use(
1461 "tool_4",
1462 ReadFileTool::NAME,
1463 ReadFileToolInput {
1464 path: input_file_path.into(),
1465 start_line: Some(271),
1466 end_line: Some(276),
1467 },
1468 ),
1469 ],
1470 ),
1471 message(
1472 User,
1473 [tool_result(
1474 "tool_4",
1475 ReadFileTool::NAME,
1476 lines(input_file_content, 271..276),
1477 )],
1478 ),
1479 ],
1480 input_file_path,
1481 Some(input_file_content.into()),
1482 EvalAssertion::judge_diff(
1483 "A new test for overwritten files was created, without changing any previous test",
1484 ),
1485 ))
1486 });
1487}
1488