Skip to repository content2121 lines · 84.4 KB · rust
tenant.openagents/omega
No repository description is available.
OpenAgents Git authority 2026-07-28T02:33:34.104Z Public web read
NIP-34 coordinate
30617:7649603503856e5148d571eac2766b288a8ff1e9e35d380337a1d2b0015b4f92:omegaMaintainersHidden in public view
References2 branches · 1 tag
Read-only clone
git clone https://openagents.com/git/tenant.openagents/omega.gitBrowse files
llama_cpp.rs
1use anyhow::Result;
2use collections::{HashMap, HashSet};
3use credentials_provider::CredentialsProvider;
4use fs::Fs;
5use futures::Stream;
6use futures::{FutureExt, StreamExt, future::BoxFuture, stream::BoxStream};
7use gpui::{App, AsyncApp, Context, Entity, Task, TaskExt};
8use http_client::{CustomHeaders, HttpClient};
9use language_model::util::parse_tool_arguments;
10use language_model::{
11 ApiKeyState, AuthenticateError, EnvVar, IconOrSvg, InlineDescription, LanguageModel,
12 LanguageModelCompletionError, LanguageModelCompletionEvent, LanguageModelId, LanguageModelName,
13 LanguageModelProvider, LanguageModelProviderId, LanguageModelProviderName,
14 LanguageModelProviderState, LanguageModelRequest, LanguageModelToolChoice,
15 LanguageModelToolResultContent, LanguageModelToolUse, MessageContent, ProviderSettingsView,
16 RateLimiter, Role, StopReason, SubPageProviderSettings, TokenUsage, env_var,
17};
18use llama_cpp::{
19 LLAMA_CPP_API_URL, ModelEntry, Props, get_models, get_props, stream_chat_completion,
20 stream_model_events,
21};
22pub use settings::LlamaCppAvailableModel as AvailableModel;
23use settings::{Settings, SettingsStore, update_settings_file};
24use std::pin::Pin;
25use std::sync::LazyLock;
26use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard};
27use std::time::Duration;
28use ui::{
29 ButtonLike, ButtonLink, ConfiguredApiCard, Divider, List, ListBulletItem, Tooltip, prelude::*,
30};
31use ui_input::InputField;
32use util::ResultExt;
33
34use crate::AllLanguageModelSettings;
35
36const LLAMA_CPP_DOWNLOAD_URL: &str = "https://llama.app";
37const LLAMA_CPP_MODELS_URL: &str = "https://huggingface.co/models?library=gguf&sort=trending";
38
39const PROVIDER_ID: LanguageModelProviderId = LanguageModelProviderId::new("llama.cpp");
40const PROVIDER_NAME: LanguageModelProviderName = LanguageModelProviderName::new("llama.cpp");
41
42const API_KEY_ENV_VAR_NAME: &str = "LLAMACPP_API_KEY";
43static API_KEY_ENV_VAR: LazyLock<EnvVar> = env_var!(API_KEY_ENV_VAR_NAME);
44
45/// How long to wait before reconnecting to `/models/sse` after the stream ends.
46const MODEL_EVENT_RECONNECT_INTERVAL: Duration = Duration::from_secs(5);
47
48/// Context length assumed for an unloaded router model (it can't be probed
49/// without loading it). Generous so early messages work; re-discovery refines
50/// it once the model loads.
51const ASSUMED_UNLOADED_CONTEXT: u64 = 131_072;
52
53#[derive(Default, Debug, Clone, PartialEq)]
54pub struct LlamaCppSettings {
55 pub api_url: String,
56 pub auto_discover: bool,
57 pub available_models: Vec<AvailableModel>,
58 pub context_window: Option<u64>,
59 pub custom_headers: CustomHeaders,
60}
61
62pub struct LlamaCppLanguageModelProvider {
63 http_client: Arc<dyn HttpClient>,
64 state: Entity<State>,
65 /// Live capabilities shared with the agent's models (see [`LiveCapabilities`]).
66 capability_cells: CapabilityCells,
67 /// Live model-load progress shared with the models (see [`LoadingProgress`]).
68 loading_progress: LoadingProgress,
69}
70
71pub struct State {
72 api_key_state: ApiKeyState,
73 credentials_provider: Arc<dyn CredentialsProvider>,
74 http_client: Arc<dyn HttpClient>,
75 fetched_models: Vec<llama_cpp::Model>,
76 fetch_model_task: Option<Task<Result<()>>>,
77 /// Router-mode task on `/models/sse`; re-runs discovery as models load/unload.
78 model_event_task: Option<Task<()>>,
79 /// Same `Arc` as the provider's; re-discovery keeps these cells in sync.
80 capability_cells: CapabilityCells,
81 /// Same `Arc` as the provider's; the event stream updates it as a model loads.
82 loading_progress: LoadingProgress,
83}
84
85impl State {
86 fn is_authenticated(&self) -> bool {
87 !self.fetched_models.is_empty()
88 }
89
90 fn set_api_key(&mut self, api_key: Option<String>, cx: &mut Context<Self>) -> Task<Result<()>> {
91 let credentials_provider = self.credentials_provider.clone();
92 let api_url = LlamaCppLanguageModelProvider::api_url(cx);
93 let task = self.api_key_state.store(
94 api_url,
95 api_key,
96 |this| &mut this.api_key_state,
97 credentials_provider,
98 cx,
99 );
100
101 self.fetched_models.clear();
102 // Drop the event stream so it reconnects with the new key (re-fetch
103 // below restarts it).
104 self.model_event_task = None;
105 write_recover(&self.loading_progress).clear();
106 cx.spawn(async move |this, cx| {
107 let result = task.await;
108 this.update(cx, |this, cx| this.restart_fetch_models_task(cx))
109 .ok();
110 result
111 })
112 }
113
114 fn authenticate(&mut self, cx: &mut Context<Self>) -> Task<Result<(), AuthenticateError>> {
115 let credentials_provider = self.credentials_provider.clone();
116 let api_url = LlamaCppLanguageModelProvider::api_url(cx);
117 let load_key_task = self.api_key_state.load_if_needed(
118 api_url,
119 |this| &mut this.api_key_state,
120 credentials_provider,
121 cx,
122 );
123
124 if self.is_authenticated() {
125 return Task::ready(Ok(()));
126 }
127
128 cx.spawn(async move |this, cx| {
129 match load_key_task.await {
130 Ok(()) | Err(AuthenticateError::CredentialsNotFound) => {}
131 Err(error) => {
132 log::warn!("failed to load llama.cpp API key: {error}");
133 }
134 }
135 let fetch_models_task = this.update(cx, |this, cx| this.fetch_models(cx))?;
136 match fetch_models_task.await {
137 Ok(()) => Ok(()),
138 Err(err) => {
139 // A refused connection means the server isn't running yet, not an error.
140 let connection_refused = err.chain().any(|cause| {
141 cause
142 .downcast_ref::<std::io::Error>()
143 .is_some_and(|io_err| {
144 io_err.kind() == std::io::ErrorKind::ConnectionRefused
145 })
146 });
147 if connection_refused {
148 Err(AuthenticateError::ConnectionRefused)
149 } else {
150 Err(AuthenticateError::Other(err))
151 }
152 }
153 }
154 })
155 }
156
157 fn fetch_models(&mut self, cx: &mut Context<Self>) -> Task<Result<()>> {
158 let http_client = Arc::clone(&self.http_client);
159 let settings = LlamaCppLanguageModelProvider::settings(cx);
160 let api_url = LlamaCppLanguageModelProvider::api_url(cx);
161 let api_key = self.api_key_state.key(&api_url);
162 let extra_headers = settings.custom_headers.clone();
163
164 cx.spawn(async move |this, cx| {
165 let entries = get_models(
166 http_client.as_ref(),
167 &api_url,
168 api_key.as_deref(),
169 &extra_headers,
170 )
171 .await?;
172
173 let is_router = entries.iter().any(ModelEntry::is_router_entry);
174
175 // Models the server reports as loading, used below to prune stale
176 // progress labels by reconciling against the live listing (a preempted
177 // load or missed SSE event can skip the terminal event).
178 let loading_ids: HashSet<String> = entries
179 .iter()
180 .filter(|entry| entry.is_loading())
181 .map(|entry| entry.id.clone())
182 .collect();
183
184 let models: Vec<llama_cpp::Model> = if is_router {
185 // Router mode: metadata comes from `/v1/models`. We probe
186 // `/props` only for loaded models so listing never triggers a
187 // load; unloaded models use the listing's hints and overrides.
188 let tasks = entries.into_iter().map(|entry| {
189 let http_client = Arc::clone(&http_client);
190 let api_url = api_url.clone();
191 let api_key = api_key.clone();
192 let extra_headers = extra_headers.clone();
193 async move {
194 let props = if entry.is_loaded() {
195 get_props(
196 http_client.as_ref(),
197 &api_url,
198 api_key.as_deref(),
199 Some(&entry.id),
200 &extra_headers,
201 )
202 .await
203 .log_err()
204 } else {
205 None
206 };
207 model_from_entry(&entry, props.as_ref())
208 }
209 });
210 futures::stream::iter(tasks)
211 .buffer_unordered(5)
212 .collect()
213 .await
214 } else {
215 // Single-model mode: one `/props` call describes the loaded model.
216 let props = get_props(
217 http_client.as_ref(),
218 &api_url,
219 api_key.as_deref(),
220 None,
221 &extra_headers,
222 )
223 .await
224 .log_err();
225 entries
226 .iter()
227 .map(|entry| model_from_entry(entry, props.as_ref()))
228 .collect()
229 };
230
231 this.update(cx, |this, cx| {
232 this.fetched_models = models;
233 let effective = compute_effective_models(
234 &this.fetched_models,
235 LlamaCppLanguageModelProvider::settings(cx),
236 );
237 sync_capability_cells(&this.capability_cells, &effective);
238 // Drop progress labels for models no longer loading, so a stale
239 // "Loading …" can't stick after a preempted load or missed event.
240 write_recover(&this.loading_progress).retain(|id, _| loading_ids.contains(id));
241 // Router mode loads models on demand: subscribe so capabilities
242 // self-correct as they load/unload. Start it once (events trigger
243 // re-discovery, not a re-spawn); single-model mode needs no stream.
244 if is_router {
245 if this.model_event_task.is_none() {
246 this.start_model_event_stream(cx);
247 }
248 } else {
249 this.model_event_task = None;
250 }
251 cx.notify();
252 })
253 })
254 }
255
256 /// Subscribes to `/models/sse` and re-runs discovery as models load, unload,
257 /// or the list changes, so capabilities stay current. Reconnects if the stream
258 /// drops; on builds without `/models/sse` the refresh is simply skipped.
259 fn start_model_event_stream(&mut self, cx: &mut Context<Self>) {
260 let http_client = Arc::clone(&self.http_client);
261 let api_url = LlamaCppLanguageModelProvider::api_url(cx);
262 let api_key = self.api_key_state.key(&api_url);
263 let extra_headers = LlamaCppLanguageModelProvider::settings(cx)
264 .custom_headers
265 .clone();
266
267 self.model_event_task = Some(cx.spawn(async move |this, cx| {
268 loop {
269 match stream_model_events(
270 http_client.as_ref(),
271 &api_url,
272 api_key.as_deref(),
273 &extra_headers,
274 )
275 .await
276 {
277 Ok(mut events) => {
278 while let Some(event) = events.next().await {
279 let Some(event) = event.log_err() else {
280 continue;
281 };
282 if let Some(exit_code) = event.load_failure() {
283 log::error!(
284 "llama.cpp model {} failed to load (exit code {exit_code})",
285 event.model
286 );
287 }
288 // Loading-progress tick: record it for the selector (no
289 // re-discovery). `cx.notify()` drives `ProviderStateChanged`.
290 if let Some(progress) = event.load_progress() {
291 let label = SharedString::from(progress.progress_label());
292 if this
293 .update(cx, |this, cx| {
294 write_recover(&this.loading_progress)
295 .insert(event.model.clone(), label);
296 cx.notify();
297 })
298 .is_err()
299 {
300 return;
301 }
302 continue;
303 }
304 if !event.changes_model_state() {
305 continue;
306 }
307 // Terminal load/unload (or list change): drop the
308 // progress label and re-discover to refresh capabilities.
309 if this
310 .update(cx, |this, cx| {
311 write_recover(&this.loading_progress).remove(&event.model);
312 this.restart_fetch_models_task(cx);
313 })
314 .is_err()
315 {
316 return;
317 }
318 }
319 }
320 // Endpoint missing (older build) or connection failed; retry after a backoff.
321 Err(error) => {
322 log::warn!("llama.cpp model event stream unavailable: {error:#}");
323 }
324 }
325
326 cx.background_executor()
327 .timer(MODEL_EVENT_RECONNECT_INTERVAL)
328 .await;
329 if this.update(cx, |_, _| ()).is_err() {
330 return;
331 }
332 }
333 }));
334 }
335
336 fn restart_fetch_models_task(&mut self, cx: &mut Context<Self>) {
337 let task = self.fetch_models(cx);
338 self.fetch_model_task.replace(task);
339 }
340}
341
342/// Capabilities that only become known once a router model loads. `LanguageModel`'s
343/// methods take no `cx`, yet the agent reads them live each turn, so we share
344/// them through a map that re-discovery updates — an already-selected model
345/// picks up real values without re-selection. Image support is available from
346/// `/v1/models` hints, so it stays a field.
347#[derive(Clone, Copy, Debug, PartialEq)]
348struct LiveCapabilities {
349 max_tokens: u64,
350 supports_tools: bool,
351 supports_thinking: bool,
352}
353
354impl LiveCapabilities {
355 fn of(model: &llama_cpp::Model) -> Self {
356 Self {
357 max_tokens: model.max_tokens,
358 supports_tools: model.supports_tools,
359 supports_thinking: model.supports_thinking,
360 }
361 }
362}
363
364/// Live capabilities keyed by model name, shared by the provider and its models.
365type CapabilityCells = Arc<RwLock<HashMap<String, LiveCapabilities>>>;
366
367/// Model name → load-status label (e.g. `"Loading weights 42%"`) while a router
368/// model loads, shared so the model selector can show progress. Absent once loaded.
369type LoadingProgress = Arc<RwLock<HashMap<String, SharedString>>>;
370
371/// Locks for reading, recovering instead of panicking on a poisoned lock. The
372/// critical sections are infallible map ops, so poisoning is unreachable anyway.
373fn read_recover<T>(lock: &RwLock<T>) -> RwLockReadGuard<'_, T> {
374 lock.read().unwrap_or_else(|poisoned| poisoned.into_inner())
375}
376
377/// Locks for writing; see [`read_recover`].
378fn write_recover<T>(lock: &RwLock<T>) -> RwLockWriteGuard<'_, T> {
379 lock.write()
380 .unwrap_or_else(|poisoned| poisoned.into_inner())
381}
382
383/// The exact set of models `provided_models` exposes: discovery merged with the
384/// `available_models` and `context_window` overrides. Shared with re-discovery.
385fn compute_effective_models(
386 fetched_models: &[llama_cpp::Model],
387 settings: &LlamaCppSettings,
388) -> HashMap<String, llama_cpp::Model> {
389 let mut models: HashMap<String, llama_cpp::Model> = HashMap::default();
390 if settings.auto_discover {
391 for model in fetched_models {
392 let mut model = model.clone();
393 if let Some(context_window) = settings.context_window {
394 model.max_tokens = context_window;
395 }
396 models.insert(model.name.clone(), model);
397 }
398 }
399 merge_settings_into_models(
400 &mut models,
401 &settings.available_models,
402 settings.context_window,
403 );
404 models
405}
406
407/// Updates the shared capability map from the effective models, so a model held
408/// by an open conversation observes the new values (it reads the map by name).
409fn sync_capability_cells(cells: &CapabilityCells, effective: &HashMap<String, llama_cpp::Model>) {
410 let mut cells = write_recover(cells);
411 for model in effective.values() {
412 cells.insert(model.name.clone(), LiveCapabilities::of(model));
413 }
414}
415
416/// Builds a model from a `/v1/models` entry, refined by `/props` when the model
417/// is loaded. An unloaded router model can't be probed, so we assume optimistic
418/// capabilities and let re-discovery refine them on load.
419fn model_from_entry(entry: &ModelEntry, props: Option<&Props>) -> llama_cpp::Model {
420 let max_tokens = props
421 .and_then(Props::context_length)
422 .or_else(|| entry.meta.as_ref().and_then(|meta| meta.n_ctx))
423 .or_else(|| entry.meta.as_ref().and_then(|meta| meta.n_ctx_train))
424 .unwrap_or(ASSUMED_UNLOADED_CONTEXT);
425 // Trust `/props` when present. Without it, assume tools for an unloaded model
426 // (re-discovery corrects on load) but not for a loaded model whose probe failed.
427 let supports_tools = match props {
428 Some(props) => props.supports_tools(),
429 None => !entry.is_loaded(),
430 };
431 let supports_images = props.is_some_and(Props::supports_images) || entry.supports_images_hint();
432 let supports_thinking = props.is_some_and(Props::supports_thinking);
433
434 llama_cpp::Model::new(
435 &entry.id,
436 Some(&display_name_for(&entry.id)),
437 Some(max_tokens),
438 supports_tools,
439 supports_images,
440 supports_thinking,
441 )
442}
443
444/// Friendly display name from a model id, which is often a `.gguf` file path.
445fn display_name_for(id: &str) -> String {
446 let base = id.rsplit(['/', '\\']).next().unwrap_or(id);
447 base.strip_suffix(".gguf").unwrap_or(base).to_string()
448}
449
450fn telemetry_id_for(id: &str) -> String {
451 format!("{PROVIDER_ID}/{}", display_name_for(id))
452}
453
454impl LlamaCppLanguageModelProvider {
455 pub fn new(
456 http_client: Arc<dyn HttpClient>,
457 credentials_provider: Arc<dyn CredentialsProvider>,
458 cx: &mut App,
459 ) -> Self {
460 let capability_cells: CapabilityCells = Arc::new(RwLock::new(HashMap::default()));
461 let loading_progress: LoadingProgress = Arc::new(RwLock::new(HashMap::default()));
462 let this = Self {
463 http_client: http_client.clone(),
464 capability_cells: capability_cells.clone(),
465 loading_progress: loading_progress.clone(),
466 state: cx.new(|cx| {
467 cx.observe_global::<SettingsStore>({
468 let mut last_settings = LlamaCppLanguageModelProvider::settings(cx).clone();
469 move |this: &mut State, cx| {
470 let current_settings = LlamaCppLanguageModelProvider::settings(cx);
471 let settings_changed = current_settings != &last_settings;
472 if settings_changed {
473 let url_changed = last_settings.api_url != current_settings.api_url;
474 last_settings = current_settings.clone();
475 if url_changed {
476 let credentials_provider = this.credentials_provider.clone();
477 let api_url = Self::api_url(cx);
478 this.api_key_state.handle_url_change(
479 api_url,
480 |this| &mut this.api_key_state,
481 credentials_provider,
482 cx,
483 );
484 this.fetched_models.clear();
485 // Drop the event stream so it reconnects against
486 // the new URL (re-auth below restarts it).
487 this.model_event_task = None;
488 write_recover(&this.loading_progress).clear();
489 this.authenticate(cx).detach();
490 }
491 cx.notify();
492 }
493 }
494 })
495 .detach();
496
497 State {
498 http_client,
499 fetched_models: Default::default(),
500 fetch_model_task: None,
501 model_event_task: None,
502 capability_cells,
503 loading_progress,
504 api_key_state: ApiKeyState::new(Self::api_url(cx), (*API_KEY_ENV_VAR).clone()),
505 credentials_provider,
506 }
507 }),
508 };
509 // Discover eagerly so a running server is picked up without opening settings.
510 this.state
511 .update(cx, |state, cx| state.restart_fetch_models_task(cx));
512 this
513 }
514
515 fn settings(cx: &App) -> &LlamaCppSettings {
516 &AllLanguageModelSettings::get_global(cx).llama_cpp
517 }
518
519 fn api_url(cx: &App) -> SharedString {
520 let api_url = &Self::settings(cx).api_url;
521 if api_url.is_empty() {
522 LLAMA_CPP_API_URL.into()
523 } else {
524 SharedString::new(api_url.as_str())
525 }
526 }
527
528 fn has_custom_url(cx: &App) -> bool {
529 Self::settings(cx).api_url != LLAMA_CPP_API_URL
530 }
531}
532
533impl LanguageModelProviderState for LlamaCppLanguageModelProvider {
534 type ObservableEntity = State;
535
536 fn observable_entity(&self) -> Option<Entity<Self::ObservableEntity>> {
537 Some(self.state.clone())
538 }
539}
540
541impl LanguageModelProvider for LlamaCppLanguageModelProvider {
542 fn id(&self) -> LanguageModelProviderId {
543 PROVIDER_ID
544 }
545
546 fn name(&self) -> LanguageModelProviderName {
547 PROVIDER_NAME
548 }
549
550 fn icon(&self) -> IconOrSvg {
551 IconOrSvg::Icon(IconName::AiLlamaCpp)
552 }
553
554 fn default_model(&self, _: &App) -> Option<Arc<dyn LanguageModel>> {
555 // No default model: in router mode it could trigger an expensive load of
556 // an unloaded model on a constrained machine.
557 None
558 }
559
560 fn default_fast_model(&self, _: &App) -> Option<Arc<dyn LanguageModel>> {
561 // See explanation for default_model.
562 None
563 }
564
565 fn provided_models(&self, cx: &App) -> Vec<Arc<dyn LanguageModel>> {
566 let settings = LlamaCppLanguageModelProvider::settings(cx);
567 let effective = compute_effective_models(&self.state.read(cx).fetched_models, settings);
568
569 // Refresh the shared capability map so open conversations pick up settings changes.
570 sync_capability_cells(&self.capability_cells, &effective);
571 let mut models = effective
572 .into_values()
573 .map(|model| {
574 Arc::new(LlamaCppLanguageModel {
575 id: LanguageModelId::from(model.name.clone()),
576 name: model.name.clone(),
577 display_name: model.display_name().to_string(),
578 fallback_capabilities: LiveCapabilities::of(&model),
579 supports_images: model.supports_images,
580 capability_cells: self.capability_cells.clone(),
581 loading_progress: self.loading_progress.clone(),
582 http_client: self.http_client.clone(),
583 request_limiter: RateLimiter::new(4),
584 state: self.state.clone(),
585 }) as Arc<dyn LanguageModel>
586 })
587 .collect::<Vec<_>>();
588 models.sort_by_key(|model| model.name());
589 models
590 }
591
592 fn is_authenticated(&self, cx: &App) -> bool {
593 self.state.read(cx).is_authenticated()
594 }
595
596 fn authenticate(&self, cx: &mut App) -> Task<Result<(), AuthenticateError>> {
597 self.state.update(cx, |state, cx| state.authenticate(cx))
598 }
599
600 fn settings_view(&self, _cx: &mut App) -> Option<ProviderSettingsView> {
601 let state = self.state.clone();
602 Some(ProviderSettingsView::SubPage(
603 SubPageProviderSettings::new(move |window, cx| {
604 cx.new(|cx| ConfigurationView::new(state.clone(), window, cx))
605 .into()
606 })
607 .description(InlineDescription::Text(
608 "Run local models on your machine with LlamaCpp.".into(),
609 )),
610 ))
611 }
612}
613
614pub struct LlamaCppLanguageModel {
615 id: LanguageModelId,
616 /// The model id sent to the server.
617 name: String,
618 display_name: String,
619 /// Live capabilities shared with the provider, read fresh on each access so an
620 /// open conversation reflects the model's real values once it has loaded.
621 capability_cells: CapabilityCells,
622 /// Used when `capability_cells` has no entry (e.g. model removed mid-conversation).
623 fallback_capabilities: LiveCapabilities,
624 /// Available from `/v1/models` hints, so captured at build time.
625 supports_images: bool,
626 /// Shared with the provider; this model's load progress, read by `name` so the
627 /// selector can show a loading indicator.
628 loading_progress: LoadingProgress,
629 http_client: Arc<dyn HttpClient>,
630 request_limiter: RateLimiter,
631 state: Entity<State>,
632}
633
634impl LlamaCppLanguageModel {
635 /// The model's live capabilities, or the build-time fallback if the map lacks it.
636 fn capabilities(&self) -> LiveCapabilities {
637 read_recover(&self.capability_cells)
638 .get(&self.name)
639 .copied()
640 .unwrap_or(self.fallback_capabilities)
641 }
642
643 /// This model's load-status label while loading, read live from the shared map.
644 fn loading_label(&self) -> Option<SharedString> {
645 read_recover(&self.loading_progress)
646 .get(&self.name)
647 .cloned()
648 }
649
650 fn to_llama_cpp_request(
651 &self,
652 request: LanguageModelRequest,
653 ) -> Result<llama_cpp::ChatCompletionRequest> {
654 build_llama_cpp_request(
655 &self.name,
656 self.supports_images,
657 self.capabilities(),
658 request,
659 )
660 }
661
662 fn stream_completion(
663 &self,
664 request: llama_cpp::ChatCompletionRequest,
665 cx: &AsyncApp,
666 ) -> BoxFuture<
667 'static,
668 Result<futures::stream::BoxStream<'static, Result<llama_cpp::ResponseStreamEvent>>>,
669 > {
670 let http_client = self.http_client.clone();
671 let (api_key, api_url, extra_headers) = self.state.read_with(cx, |state, cx| {
672 let api_url = LlamaCppLanguageModelProvider::api_url(cx);
673 let extra_headers = LlamaCppLanguageModelProvider::settings(cx)
674 .custom_headers
675 .clone();
676 (state.api_key_state.key(&api_url), api_url, extra_headers)
677 });
678
679 let future = self.request_limiter.stream(async move {
680 let stream = stream_chat_completion(
681 http_client.as_ref(),
682 &api_url,
683 api_key.as_deref(),
684 request,
685 &extra_headers,
686 )
687 .await?;
688 Ok(stream)
689 });
690
691 async move { Ok(future.await?.boxed()) }.boxed()
692 }
693}
694
695fn build_llama_cpp_request(
696 model_name: &str,
697 supports_images: bool,
698 capabilities: LiveCapabilities,
699 request: LanguageModelRequest,
700) -> Result<llama_cpp::ChatCompletionRequest> {
701 if request.contains_custom_tool_input() {
702 anyhow::bail!("llama.cpp does not support custom tools");
703 }
704
705 let supports_tools = capabilities.supports_tools;
706 let supports_thinking = capabilities.supports_thinking;
707 let mut messages = Vec::new();
708
709 for message in request.messages {
710 let mut reasoning_content: Option<String> = None;
711 for content in message.content {
712 match content {
713 MessageContent::Text(text) => add_message_content_part(
714 llama_cpp::MessagePart::Text { text },
715 message.role,
716 &mut messages,
717 if supports_thinking && message.role == Role::Assistant {
718 reasoning_content.take()
719 } else {
720 None
721 },
722 ),
723 MessageContent::Thinking { text, .. } => {
724 if supports_thinking && message.role == Role::Assistant && !text.is_empty() {
725 reasoning_content.get_or_insert_default().push_str(&text);
726 }
727 }
728 MessageContent::RedactedThinking(_) => {}
729 MessageContent::Compaction(_) => {}
730 MessageContent::Image(image) => {
731 if supports_images {
732 add_message_content_part(
733 llama_cpp::MessagePart::Image {
734 image_url: llama_cpp::ImageUrl {
735 url: image.to_base64_url(),
736 detail: None,
737 },
738 },
739 message.role,
740 &mut messages,
741 if supports_thinking && message.role == Role::Assistant {
742 reasoning_content.take()
743 } else {
744 None
745 },
746 );
747 }
748 }
749 MessageContent::ToolUse(tool_use) => {
750 let input = tool_use.input.as_json().ok_or_else(|| {
751 anyhow::anyhow!("llama.cpp does not support custom tool calls")
752 })?;
753 let tool_call = llama_cpp::ToolCall {
754 id: tool_use.id.to_string(),
755 content: llama_cpp::ToolCallContent::Function {
756 function: llama_cpp::FunctionContent {
757 name: tool_use.name.to_string(),
758 arguments: serde_json::to_string(input).unwrap_or_default(),
759 },
760 },
761 };
762
763 if let Some(llama_cpp::ChatMessage::Assistant {
764 tool_calls,
765 reasoning_content: message_reasoning_content,
766 ..
767 }) = messages.last_mut()
768 {
769 append_reasoning_content(
770 message_reasoning_content,
771 reasoning_content.take(),
772 );
773 tool_calls.push(tool_call);
774 } else {
775 messages.push(llama_cpp::ChatMessage::Assistant {
776 content: None,
777 reasoning_content: reasoning_content.take(),
778 tool_calls: vec![tool_call],
779 });
780 }
781 }
782 MessageContent::ToolResult(tool_result) => {
783 let content: Vec<llama_cpp::MessagePart> = tool_result
784 .content
785 .iter()
786 .filter_map(|part| match part {
787 LanguageModelToolResultContent::Text(text) => {
788 Some(llama_cpp::MessagePart::Text {
789 text: text.to_string(),
790 })
791 }
792 LanguageModelToolResultContent::Image(image) => {
793 if supports_images {
794 Some(llama_cpp::MessagePart::Image {
795 image_url: llama_cpp::ImageUrl {
796 url: image.to_base64_url(),
797 detail: None,
798 },
799 })
800 } else {
801 None
802 }
803 }
804 })
805 .collect();
806
807 messages.push(llama_cpp::ChatMessage::Tool {
808 content: content.into(),
809 tool_call_id: tool_result.tool_use_id.to_string(),
810 });
811 }
812 }
813 }
814 }
815
816 let tools: Vec<llama_cpp::ToolDefinition> = if supports_tools {
817 request
818 .tools
819 .into_iter()
820 .map(|tool| {
821 let input_schema = match tool.input {
822 language_model::LanguageModelRequestToolInput::Function {
823 input_schema,
824 ..
825 } => input_schema,
826 language_model::LanguageModelRequestToolInput::Custom { .. } => {
827 return Err(anyhow::anyhow!("llama.cpp does not support custom tools"));
828 }
829 };
830 Ok(llama_cpp::ToolDefinition::Function {
831 function: llama_cpp::FunctionDefinition {
832 name: tool.name,
833 description: Some(tool.description),
834 parameters: Some(input_schema),
835 },
836 })
837 })
838 .collect::<Result<_>>()?
839 } else {
840 Vec::new()
841 };
842 // Only send `tool_choice` with actual tools; some OpenAI-compatible servers
843 // reject it otherwise.
844 let tool_choice = if tools.is_empty() {
845 None
846 } else {
847 request.tool_choice.map(|choice| match choice {
848 LanguageModelToolChoice::Auto => llama_cpp::ToolChoice::Auto,
849 LanguageModelToolChoice::Any => llama_cpp::ToolChoice::Required,
850 LanguageModelToolChoice::None => llama_cpp::ToolChoice::None,
851 })
852 };
853
854 Ok(llama_cpp::ChatCompletionRequest {
855 model: model_name.to_string(),
856 messages,
857 stream: true,
858 // Let the server decide the output length (its `n_predict` default).
859 max_tokens: None,
860 stop: if request.stop.is_empty() {
861 None
862 } else {
863 Some(request.stop)
864 },
865 // llama.cpp models often ship recommended sampler settings, so override
866 // temperature only when the request sets one.
867 temperature: request.temperature,
868 tools,
869 tool_choice,
870 stream_options: Some(llama_cpp::StreamOptions {
871 include_usage: true,
872 }),
873 })
874}
875
876impl LanguageModel for LlamaCppLanguageModel {
877 fn id(&self) -> LanguageModelId {
878 self.id.clone()
879 }
880
881 fn name(&self) -> LanguageModelName {
882 match self.loading_label() {
883 // Surface load progress in the display name so it shows wherever the
884 // model is named, without provider-agnostic UI changes. The agent
885 // rebuilds the name on `ProviderStateChanged`, which our ticks emit.
886 Some(label) => LanguageModelName::from(format!("{} · {}", self.display_name, label)),
887 None => LanguageModelName::from(self.display_name.clone()),
888 }
889 }
890
891 fn provider_id(&self) -> LanguageModelProviderId {
892 PROVIDER_ID
893 }
894
895 fn provider_name(&self) -> LanguageModelProviderName {
896 PROVIDER_NAME
897 }
898
899 fn supports_tools(&self) -> bool {
900 self.capabilities().supports_tools
901 }
902
903 fn supports_tool_choice(&self, choice: LanguageModelToolChoice) -> bool {
904 self.supports_tools()
905 && match choice {
906 LanguageModelToolChoice::Auto => true,
907 LanguageModelToolChoice::Any => true,
908 LanguageModelToolChoice::None => true,
909 }
910 }
911
912 fn supports_images(&self) -> bool {
913 self.supports_images
914 }
915
916 fn supports_thinking(&self) -> bool {
917 self.capabilities().supports_thinking
918 }
919
920 fn telemetry_id(&self) -> String {
921 telemetry_id_for(&self.name)
922 }
923
924 fn max_token_count(&self) -> u64 {
925 self.capabilities().max_tokens
926 }
927
928 fn stream_completion(
929 &self,
930 request: LanguageModelRequest,
931 cx: &AsyncApp,
932 ) -> BoxFuture<
933 'static,
934 Result<
935 BoxStream<'static, Result<LanguageModelCompletionEvent, LanguageModelCompletionError>>,
936 LanguageModelCompletionError,
937 >,
938 > {
939 let request = match self.to_llama_cpp_request(request) {
940 Ok(request) => request,
941 Err(error) => return async move { Err(error.into()) }.boxed(),
942 };
943 let completions = self.stream_completion(request, cx);
944 async move {
945 let mapper = LlamaCppEventMapper::new();
946 Ok(mapper.map_stream(completions.await?).boxed())
947 }
948 .boxed()
949 }
950}
951
952struct LlamaCppEventMapper {
953 tool_calls_by_index: HashMap<usize, RawToolCall>,
954}
955
956impl LlamaCppEventMapper {
957 fn new() -> Self {
958 Self {
959 tool_calls_by_index: HashMap::default(),
960 }
961 }
962
963 pub fn map_stream(
964 mut self,
965 events: Pin<Box<dyn Send + Stream<Item = Result<llama_cpp::ResponseStreamEvent>>>>,
966 ) -> impl Stream<Item = Result<LanguageModelCompletionEvent, LanguageModelCompletionError>>
967 {
968 events.flat_map(move |event| {
969 futures::stream::iter(match event {
970 Ok(event) => self.map_event(event),
971 Err(error) => vec![Err(LanguageModelCompletionError::from(error))],
972 })
973 })
974 }
975
976 pub fn map_event(
977 &mut self,
978 event: llama_cpp::ResponseStreamEvent,
979 ) -> Vec<Result<LanguageModelCompletionEvent, LanguageModelCompletionError>> {
980 let mut events = Vec::new();
981
982 if let Some(usage) = event.usage {
983 events.push(Ok(LanguageModelCompletionEvent::UsageUpdate(TokenUsage {
984 input_tokens: usage.prompt_tokens,
985 output_tokens: usage.completion_tokens,
986 cache_creation_input_tokens: 0,
987 cache_read_input_tokens: 0,
988 })));
989 }
990
991 if let Some(choice) = event.choices.into_iter().next() {
992 if let Some(reasoning_content) = choice.delta.reasoning_content {
993 events.push(Ok(LanguageModelCompletionEvent::Thinking {
994 text: reasoning_content,
995 signature: None,
996 }));
997 }
998
999 if let Some(content) = choice.delta.content {
1000 if !content.is_empty() {
1001 events.push(Ok(LanguageModelCompletionEvent::Text(content)));
1002 }
1003 }
1004
1005 if let Some(tool_calls) = choice.delta.tool_calls {
1006 for tool_call in tool_calls {
1007 let entry = self.tool_calls_by_index.entry(tool_call.index).or_default();
1008
1009 if let Some(tool_id) = tool_call.id {
1010 entry.id = tool_id;
1011 }
1012
1013 if let Some(function) = tool_call.function {
1014 if let Some(name) = function.name {
1015 // Only the first chunk carries the function name;
1016 // later chunks send an empty name with arguments.
1017 if !name.is_empty() {
1018 entry.name = name;
1019 }
1020 }
1021
1022 if let Some(arguments) = function.arguments {
1023 entry.arguments.push_str(&arguments);
1024 }
1025 }
1026 }
1027 }
1028
1029 if let Some(finish_reason) = choice.finish_reason.as_deref() {
1030 match finish_reason {
1031 "stop" => {
1032 events.push(Ok(LanguageModelCompletionEvent::Stop(StopReason::EndTurn)));
1033 }
1034 "tool_calls" => {
1035 events.extend(self.tool_calls_by_index.drain().map(|(_, tool_call)| {
1036 match parse_tool_arguments(&tool_call.arguments) {
1037 Ok(input) => Ok(LanguageModelCompletionEvent::ToolUse(
1038 LanguageModelToolUse {
1039 id: tool_call.id.into(),
1040 name: tool_call.name.into(),
1041 is_input_complete: true,
1042 input: language_model::LanguageModelToolUseInput::Json(
1043 input,
1044 ),
1045 raw_input: tool_call.arguments,
1046 thought_signature: None,
1047 },
1048 )),
1049 Err(error) => {
1050 Ok(LanguageModelCompletionEvent::ToolUseJsonParseError {
1051 id: tool_call.id.into(),
1052 tool_name: tool_call.name.into(),
1053 raw_input: tool_call.arguments.into(),
1054 json_parse_error: error.to_string(),
1055 })
1056 }
1057 }
1058 }));
1059
1060 events.push(Ok(LanguageModelCompletionEvent::Stop(StopReason::ToolUse)));
1061 }
1062 "length" => {
1063 events.push(Ok(LanguageModelCompletionEvent::Stop(
1064 StopReason::MaxTokens,
1065 )));
1066 }
1067 unexpected => {
1068 log::warn!("Unexpected llama.cpp finish_reason: {unexpected:?}");
1069 events.push(Ok(LanguageModelCompletionEvent::Stop(StopReason::EndTurn)));
1070 }
1071 }
1072 }
1073 }
1074
1075 events
1076 }
1077}
1078
1079#[derive(Default)]
1080struct RawToolCall {
1081 id: String,
1082 name: String,
1083 arguments: String,
1084}
1085
1086fn add_message_content_part(
1087 new_part: llama_cpp::MessagePart,
1088 role: Role,
1089 messages: &mut Vec<llama_cpp::ChatMessage>,
1090 reasoning_content: Option<String>,
1091) {
1092 match (role, messages.last_mut()) {
1093 (Role::User, Some(llama_cpp::ChatMessage::User { content }))
1094 | (Role::System, Some(llama_cpp::ChatMessage::System { content })) => {
1095 content.push_part(new_part);
1096 }
1097 (
1098 Role::Assistant,
1099 Some(llama_cpp::ChatMessage::Assistant {
1100 content: Some(content),
1101 reasoning_content: message_reasoning_content,
1102 ..
1103 }),
1104 ) => {
1105 append_reasoning_content(message_reasoning_content, reasoning_content);
1106 content.push_part(new_part);
1107 }
1108 _ => {
1109 messages.push(match role {
1110 Role::User => llama_cpp::ChatMessage::User {
1111 content: llama_cpp::MessageContent::from(vec![new_part]),
1112 },
1113 Role::Assistant => llama_cpp::ChatMessage::Assistant {
1114 content: Some(llama_cpp::MessageContent::from(vec![new_part])),
1115 reasoning_content,
1116 tool_calls: Vec::new(),
1117 },
1118 Role::System => llama_cpp::ChatMessage::System {
1119 content: llama_cpp::MessageContent::from(vec![new_part]),
1120 },
1121 });
1122 }
1123 }
1124}
1125
1126fn append_reasoning_content(target: &mut Option<String>, content: Option<String>) {
1127 let Some(content) = content else {
1128 return;
1129 };
1130 if content.is_empty() {
1131 return;
1132 }
1133 target.get_or_insert_default().push_str(&content);
1134}
1135
1136fn merge_settings_into_models(
1137 models: &mut HashMap<String, llama_cpp::Model>,
1138 available_models: &[AvailableModel],
1139 context_window: Option<u64>,
1140) {
1141 for setting_model in available_models {
1142 if let Some(model) = models.get_mut(&setting_model.name) {
1143 if context_window.is_none() {
1144 model.max_tokens = setting_model.max_tokens;
1145 }
1146 if setting_model.display_name.is_some() {
1147 model.display_name = setting_model.display_name.clone();
1148 }
1149 if let Some(supports_tools) = setting_model.supports_tools {
1150 model.supports_tools = supports_tools;
1151 }
1152 if let Some(supports_images) = setting_model.supports_images {
1153 model.supports_images = supports_images;
1154 }
1155 if let Some(supports_thinking) = setting_model.supports_thinking {
1156 model.supports_thinking = supports_thinking;
1157 }
1158 } else {
1159 models.insert(
1160 setting_model.name.clone(),
1161 llama_cpp::Model {
1162 name: setting_model.name.clone(),
1163 display_name: setting_model.display_name.clone(),
1164 max_tokens: context_window.unwrap_or(setting_model.max_tokens),
1165 supports_tools: setting_model.supports_tools.unwrap_or(false),
1166 supports_images: setting_model.supports_images.unwrap_or(false),
1167 supports_thinking: setting_model.supports_thinking.unwrap_or(false),
1168 },
1169 );
1170 }
1171 }
1172}
1173
1174struct ConfigurationView {
1175 api_key_editor: Entity<InputField>,
1176 api_url_editor: Entity<InputField>,
1177 context_window_editor: Entity<InputField>,
1178 state: Entity<State>,
1179}
1180
1181impl ConfigurationView {
1182 pub fn new(state: Entity<State>, window: &mut Window, cx: &mut Context<Self>) -> Self {
1183 let api_key_editor = cx.new(|cx| InputField::new(window, cx, "sk-...").label("API key"));
1184
1185 let api_url_editor = cx.new(|cx| {
1186 let input = InputField::new(window, cx, LLAMA_CPP_API_URL).label("API URL");
1187 input.set_text(&LlamaCppLanguageModelProvider::api_url(cx), window, cx);
1188 input
1189 });
1190
1191 let context_window_editor = cx.new(|cx| {
1192 let input = InputField::new(window, cx, "8192").label("Context Window");
1193 if let Some(context_window) = LlamaCppLanguageModelProvider::settings(cx).context_window
1194 {
1195 input.set_text(&context_window.to_string(), window, cx);
1196 }
1197 input
1198 });
1199
1200 cx.observe(&state, |_, _, cx| {
1201 cx.notify();
1202 })
1203 .detach();
1204
1205 Self {
1206 api_key_editor,
1207 api_url_editor,
1208 context_window_editor,
1209 state,
1210 }
1211 }
1212
1213 fn retry_connection(&mut self, window: &mut Window, cx: &mut Context<Self>) {
1214 let has_api_url = LlamaCppLanguageModelProvider::has_custom_url(cx);
1215 let has_api_key = self
1216 .state
1217 .read_with(cx, |state, _| state.api_key_state.has_key());
1218 if !has_api_url {
1219 self.save_api_url(cx);
1220 }
1221 if !has_api_key {
1222 self.save_api_key(&Default::default(), window, cx);
1223 }
1224
1225 self.state.update(cx, |state, cx| {
1226 state.restart_fetch_models_task(cx);
1227 });
1228 }
1229
1230 fn save_api_key(&mut self, _: &menu::Confirm, window: &mut Window, cx: &mut Context<Self>) {
1231 let api_key = self.api_key_editor.read(cx).text(cx).trim().to_string();
1232 if api_key.is_empty() {
1233 return;
1234 }
1235
1236 // A URL change can cause the editor to be shown again.
1237 self.api_key_editor
1238 .update(cx, |input, cx| input.set_text("", window, cx));
1239
1240 let state = self.state.clone();
1241 cx.spawn_in(window, async move |_, cx| {
1242 state
1243 .update(cx, |state, cx| state.set_api_key(Some(api_key), cx))
1244 .await
1245 })
1246 .detach_and_log_err(cx);
1247 }
1248
1249 fn reset_api_key(&mut self, window: &mut Window, cx: &mut Context<Self>) {
1250 self.api_key_editor
1251 .update(cx, |input, cx| input.set_text("", window, cx));
1252
1253 let state = self.state.clone();
1254 cx.spawn_in(window, async move |_, cx| {
1255 state
1256 .update(cx, |state, cx| state.set_api_key(None, cx))
1257 .await
1258 })
1259 .detach_and_log_err(cx);
1260
1261 cx.notify();
1262 }
1263
1264 fn save_api_url(&self, cx: &mut Context<Self>) {
1265 let api_url = self.api_url_editor.read(cx).text(cx).trim().to_string();
1266 let current_url = LlamaCppLanguageModelProvider::api_url(cx);
1267 if !api_url.is_empty() && &api_url != ¤t_url {
1268 let fs = <dyn Fs>::global(cx);
1269 update_settings_file(fs, cx, move |settings, _| {
1270 settings
1271 .language_models
1272 .get_or_insert_default()
1273 .llama_cpp
1274 .get_or_insert_default()
1275 .api_url = Some(api_url);
1276 });
1277 }
1278 }
1279
1280 fn reset_api_url(&mut self, window: &mut Window, cx: &mut Context<Self>) {
1281 self.api_url_editor
1282 .update(cx, |input, cx| input.set_text("", window, cx));
1283 let fs = <dyn Fs>::global(cx);
1284 update_settings_file(fs, cx, |settings, _cx| {
1285 if let Some(settings) = settings
1286 .language_models
1287 .as_mut()
1288 .and_then(|models| models.llama_cpp.as_mut())
1289 {
1290 settings.api_url = Some(LLAMA_CPP_API_URL.into());
1291 }
1292 });
1293 cx.notify();
1294 }
1295
1296 fn save_context_window(&mut self, cx: &mut Context<Self>) {
1297 let context_window_str = self
1298 .context_window_editor
1299 .read(cx)
1300 .text(cx)
1301 .trim()
1302 .to_string();
1303 let current_context_window = LlamaCppLanguageModelProvider::settings(cx).context_window;
1304
1305 if let Ok(context_window) = context_window_str.parse::<u64>() {
1306 if Some(context_window) != current_context_window {
1307 let fs = <dyn Fs>::global(cx);
1308 update_settings_file(fs, cx, move |settings, _| {
1309 settings
1310 .language_models
1311 .get_or_insert_default()
1312 .llama_cpp
1313 .get_or_insert_default()
1314 .context_window = Some(context_window);
1315 });
1316 }
1317 } else if context_window_str.is_empty() && current_context_window.is_some() {
1318 let fs = <dyn Fs>::global(cx);
1319 update_settings_file(fs, cx, move |settings, _| {
1320 settings
1321 .language_models
1322 .get_or_insert_default()
1323 .llama_cpp
1324 .get_or_insert_default()
1325 .context_window = None;
1326 });
1327 }
1328 }
1329
1330 fn reset_context_window(&mut self, window: &mut Window, cx: &mut Context<Self>) {
1331 self.context_window_editor
1332 .update(cx, |input, cx| input.set_text("", window, cx));
1333 let fs = <dyn Fs>::global(cx);
1334 update_settings_file(fs, cx, |settings, _cx| {
1335 if let Some(settings) = settings
1336 .language_models
1337 .as_mut()
1338 .and_then(|models| models.llama_cpp.as_mut())
1339 {
1340 settings.context_window = None;
1341 }
1342 });
1343 cx.notify();
1344 }
1345
1346 fn render_instructions(cx: &App) -> Div {
1347 v_flex()
1348 .gap_2()
1349 .child(
1350 Label::new(
1351 "Run open models locally with llama.cpp's built-in server, or connect to a \
1352 remote llama.cpp server.",
1353 )
1354 .color(Color::Muted),
1355 )
1356 .child(Label::new("To use a local llama.cpp server:").color(Color::Muted))
1357 .child(
1358 List::new()
1359 .child(
1360 ListBulletItem::new("")
1361 .child(Label::new("Install llama.cpp from").color(Color::Muted))
1362 .child(ButtonLink::new("llama.app", LLAMA_CPP_DOWNLOAD_URL)),
1363 )
1364 .child(
1365 ListBulletItem::new("")
1366 .child(
1367 Label::new("Start the server in router mode:").color(Color::Muted),
1368 )
1369 .child(Label::new("llama serve").inline_code(cx)),
1370 )
1371 .child(
1372 ListBulletItem::new(
1373 "Click 'Connect' below to start using llama.cpp in Omega",
1374 )
1375 .label_color(Color::Muted),
1376 ),
1377 )
1378 .child(
1379 Label::new(
1380 "Alternatively, you can connect to a remote llama.cpp server by specifying its \
1381 URL and API key (set with --api-key, may not be required):",
1382 )
1383 .color(Color::Muted),
1384 )
1385 }
1386
1387 fn render_api_key_editor(&self, cx: &Context<Self>) -> impl IntoElement {
1388 let state = self.state.read(cx);
1389 let env_var_set = state.api_key_state.is_from_env_var();
1390 let configured_card_label = if env_var_set {
1391 format!("API key set in {API_KEY_ENV_VAR_NAME} environment variable.")
1392 } else {
1393 "API key configured".to_string()
1394 };
1395
1396 let api_key_control = if !state.api_key_state.has_key() {
1397 self.api_key_editor.clone().into_any_element()
1398 } else {
1399 ConfiguredApiCard::new("llama-cpp-reset-key", configured_card_label)
1400 .disabled(env_var_set)
1401 .on_click(cx.listener(|this, _, window, cx| this.reset_api_key(window, cx)))
1402 .when(env_var_set, |this| {
1403 this.tooltip_label(format!(
1404 "To reset your API key, unset the {API_KEY_ENV_VAR_NAME} environment variable."
1405 ))
1406 })
1407 .into_any_element()
1408 };
1409
1410 v_flex()
1411 .on_action(cx.listener(Self::save_api_key))
1412 .child(api_key_control)
1413 .gap_1p5()
1414 .mb_2()
1415 .child(
1416 Label::new(format!(
1417 "You can also set the {API_KEY_ENV_VAR_NAME} environment variable and restart Omega."
1418 ))
1419 .size(LabelSize::Small)
1420 .color(Color::Muted),
1421 )
1422 }
1423
1424 fn render_context_window_editor(&self, cx: &Context<Self>) -> Div {
1425 let settings = LlamaCppLanguageModelProvider::settings(cx);
1426 let custom_context_window_set = settings.context_window.is_some();
1427
1428 if custom_context_window_set {
1429 h_flex()
1430 .p_1()
1431 .justify_between()
1432 .rounded_md()
1433 .border_1()
1434 .border_color(cx.theme().colors().border_variant)
1435 .bg(cx.theme().colors().background.opacity(0.5))
1436 .child(
1437 h_flex()
1438 .gap_1()
1439 .child(Icon::new(IconName::Check).color(Color::Success))
1440 .child(Label::new(format!(
1441 "Context Window: {}",
1442 settings.context_window.unwrap_or_default()
1443 ))),
1444 )
1445 .child(
1446 Button::new("reset-context-window", "Reset")
1447 .style(ButtonStyle::Outlined)
1448 .label_size(LabelSize::Small)
1449 .start_icon(Icon::new(IconName::Undo).size(IconSize::Small))
1450 .on_click(
1451 cx.listener(|this, _, window, cx| {
1452 this.reset_context_window(window, cx)
1453 }),
1454 ),
1455 )
1456 } else {
1457 v_flex()
1458 .on_action(
1459 cx.listener(|this, _: &menu::Confirm, _window, cx| {
1460 this.save_context_window(cx)
1461 }),
1462 )
1463 .child(self.context_window_editor.clone())
1464 .gap_1p5()
1465 .child(
1466 Label::new("Default: Discovered from the server")
1467 .size(LabelSize::Small)
1468 .color(Color::Muted),
1469 )
1470 }
1471 }
1472
1473 fn render_api_url_editor(&self, cx: &Context<Self>) -> Div {
1474 let api_url = LlamaCppLanguageModelProvider::api_url(cx);
1475 let custom_api_url_set = api_url != LLAMA_CPP_API_URL;
1476
1477 if custom_api_url_set {
1478 h_flex()
1479 .p_1()
1480 .justify_between()
1481 .rounded_md()
1482 .border_1()
1483 .border_color(cx.theme().colors().border_variant)
1484 .bg(cx.theme().colors().background.opacity(0.5))
1485 .child(
1486 h_flex()
1487 .gap_1()
1488 .child(Icon::new(IconName::Check).color(Color::Success))
1489 .child(Label::new(api_url)),
1490 )
1491 .child(
1492 Button::new("reset-api-url", "Reset API URL")
1493 .style(ButtonStyle::Outlined)
1494 .label_size(LabelSize::Small)
1495 .start_icon(Icon::new(IconName::Undo).size(IconSize::Small))
1496 .on_click(
1497 cx.listener(|this, _, window, cx| this.reset_api_url(window, cx)),
1498 ),
1499 )
1500 } else {
1501 v_flex()
1502 .on_action(cx.listener(|this, _: &menu::Confirm, _window, cx| {
1503 this.save_api_url(cx);
1504 cx.notify();
1505 }))
1506 .gap_1p5()
1507 .child(self.api_url_editor.clone())
1508 }
1509 }
1510}
1511
1512impl Render for ConfigurationView {
1513 fn render(&mut self, _: &mut Window, cx: &mut Context<Self>) -> impl IntoElement {
1514 let is_authenticated = self.state.read(cx).is_authenticated();
1515
1516 v_flex()
1517 .gap_2()
1518 .child(Headline::new("llama.cpp").size(HeadlineSize::Small))
1519 .child(Self::render_instructions(cx))
1520 .child(self.render_api_url_editor(cx))
1521 .child(self.render_context_window_editor(cx))
1522 .child(self.render_api_key_editor(cx))
1523 .child(Divider::horizontal())
1524 .child(
1525 h_flex()
1526 .pt_2()
1527 .w_full()
1528 .justify_between()
1529 .gap_2()
1530 .child(
1531 h_flex()
1532 .w_full()
1533 .gap_2()
1534 .map(|this| {
1535 if is_authenticated {
1536 this.child(
1537 Button::new("llama-cpp-webui", "Open WebUI")
1538 .style(ButtonStyle::OutlinedGhost)
1539 .size(ButtonSize::Medium)
1540 .end_icon(
1541 Icon::new(IconName::ArrowUpRight)
1542 .size(IconSize::XSmall)
1543 .color(Color::Muted),
1544 )
1545 .on_click(move |_, _, cx| {
1546 let url =
1547 LlamaCppLanguageModelProvider::api_url(cx);
1548 cx.open_url(&url);
1549 })
1550 .into_any_element(),
1551 )
1552 .child(
1553 Button::new("llama-cpp-site", "llama.cpp")
1554 .style(ButtonStyle::OutlinedGhost)
1555 .size(ButtonSize::Medium)
1556 .end_icon(
1557 Icon::new(IconName::ArrowUpRight)
1558 .size(IconSize::XSmall)
1559 .color(Color::Muted),
1560 )
1561 .on_click(move |_, _, cx| {
1562 cx.open_url(LLAMA_CPP_DOWNLOAD_URL)
1563 })
1564 .into_any_element(),
1565 )
1566 } else {
1567 this.child(
1568 Button::new("download_llama_cpp_button", "Get llama.cpp")
1569 .style(ButtonStyle::OutlinedGhost)
1570 .size(ButtonSize::Medium)
1571 .end_icon(
1572 Icon::new(IconName::ArrowUpRight)
1573 .size(IconSize::XSmall)
1574 .color(Color::Muted),
1575 )
1576 .on_click(move |_, _, cx| {
1577 cx.open_url(LLAMA_CPP_DOWNLOAD_URL)
1578 })
1579 .into_any_element(),
1580 )
1581 }
1582 })
1583 .child(
1584 Button::new("view-models", "Browse GGUF Models")
1585 .style(ButtonStyle::OutlinedGhost)
1586 .size(ButtonSize::Medium)
1587 .end_icon(
1588 Icon::new(IconName::ArrowUpRight)
1589 .size(IconSize::XSmall)
1590 .color(Color::Muted),
1591 )
1592 .on_click(move |_, _, cx| cx.open_url(LLAMA_CPP_MODELS_URL)),
1593 ),
1594 )
1595 .map(|this| {
1596 if is_authenticated {
1597 this.child(
1598 ButtonLike::new("connected")
1599 .size(ButtonSize::Medium)
1600 .child(
1601 h_flex()
1602 .gap_1()
1603 .child(Icon::new(IconName::Check).color(Color::Success))
1604 .child(Label::new("Connected")),
1605 )
1606 .child(
1607 IconButton::new("refresh-models", IconName::RotateCcw)
1608 .icon_size(IconSize::Small)
1609 .tooltip(Tooltip::text("Refresh Models"))
1610 .on_click(cx.listener(|this, _, window, cx| {
1611 this.state.update(cx, |state, _| {
1612 state.fetched_models.clear();
1613 });
1614 this.retry_connection(window, cx);
1615 })),
1616 ),
1617 )
1618 } else {
1619 this.child(
1620 Button::new("retry_llama_cpp_models", "Connect")
1621 .style(ButtonStyle::Outlined)
1622 .size(ButtonSize::Medium)
1623 .start_icon(
1624 Icon::new(IconName::PlayOutlined).size(IconSize::XSmall),
1625 )
1626 .on_click(cx.listener(move |this, _, window, cx| {
1627 this.retry_connection(window, cx)
1628 })),
1629 )
1630 }
1631 }),
1632 )
1633 }
1634}
1635
1636#[cfg(test)]
1637mod tests {
1638 use super::*;
1639 use gpui::TestAppContext;
1640 use http_client::FakeHttpClient;
1641 use parking_lot::Mutex;
1642 use std::sync::atomic::{AtomicUsize, Ordering};
1643
1644 struct FakeCredentialsProvider {
1645 api_key: Vec<u8>,
1646 }
1647
1648 impl CredentialsProvider for FakeCredentialsProvider {
1649 fn read_credentials<'a>(
1650 &'a self,
1651 _url: &'a str,
1652 _cx: &'a AsyncApp,
1653 ) -> std::pin::Pin<
1654 Box<dyn std::future::Future<Output = Result<Option<(String, Vec<u8>)>>> + 'a>,
1655 > {
1656 let api_key = self.api_key.clone();
1657 Box::pin(async move { Ok(Some(("Bearer".to_string(), api_key))) })
1658 }
1659
1660 fn write_credentials<'a>(
1661 &'a self,
1662 _url: &'a str,
1663 _username: &'a str,
1664 _password: &'a [u8],
1665 _cx: &'a AsyncApp,
1666 ) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<()>> + 'a>> {
1667 Box::pin(async { Ok(()) })
1668 }
1669
1670 fn delete_credentials<'a>(
1671 &'a self,
1672 _url: &'a str,
1673 _cx: &'a AsyncApp,
1674 ) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<()>> + 'a>> {
1675 Box::pin(async { Ok(()) })
1676 }
1677 }
1678
1679 fn entry(id: &str, n_ctx: Option<u64>, n_ctx_train: Option<u64>) -> ModelEntry {
1680 ModelEntry {
1681 id: id.to_string(),
1682 meta: Some(llama_cpp::ModelMeta { n_ctx, n_ctx_train }),
1683 architecture: None,
1684 status: None,
1685 }
1686 }
1687
1688 #[test]
1689 fn display_name_strips_path_and_extension() {
1690 assert_eq!(
1691 display_name_for("../models/Qwen2.5-Coder-7B-Instruct-Q4_K_M.gguf"),
1692 "Qwen2.5-Coder-7B-Instruct-Q4_K_M"
1693 );
1694 assert_eq!(display_name_for("my-alias"), "my-alias");
1695 }
1696
1697 #[test]
1698 fn telemetry_id_strips_local_model_paths() {
1699 assert_eq!(
1700 telemetry_id_for("/Users/alice/models/Qwen2.5-Coder-7B-Instruct-Q4_K_M.gguf"),
1701 "llama.cpp/Qwen2.5-Coder-7B-Instruct-Q4_K_M"
1702 );
1703 assert_eq!(
1704 telemetry_id_for(r"C:\Users\alice\models\Qwen2.5-Coder-7B-Instruct-Q4_K_M.gguf"),
1705 "llama.cpp/Qwen2.5-Coder-7B-Instruct-Q4_K_M"
1706 );
1707 assert_eq!(telemetry_id_for("my-alias"), "llama.cpp/my-alias");
1708 }
1709
1710 #[test]
1711 fn model_uses_props_then_meta_for_context() {
1712 let props = Props {
1713 default_generation_settings: Some(llama_cpp::GenerationSettings { n_ctx: Some(8192) }),
1714 modalities: Some(llama_cpp::Modalities { vision: true }),
1715 chat_template_caps: Some(llama_cpp::ChatTemplateCaps {
1716 supports_tool_calls: true,
1717 supports_preserve_reasoning: true,
1718 ..Default::default()
1719 }),
1720 };
1721 // /props wins when present.
1722 let model = model_from_entry(&entry("m", Some(4096), Some(131072)), Some(&props));
1723 assert_eq!(model.max_tokens, 8192);
1724 assert!(model.supports_tools);
1725 assert!(model.supports_images);
1726 assert!(model.supports_thinking);
1727
1728 // Unprobed: falls back to the listing's runtime context, then trained
1729 // context. Tools are assumed supported until the model loads.
1730 let model = model_from_entry(&entry("m", Some(4096), Some(131072)), None);
1731 assert_eq!(model.max_tokens, 4096);
1732 assert!(model.supports_tools);
1733 assert!(!model.supports_thinking);
1734
1735 let model = model_from_entry(&entry("m", None, Some(131072)), None);
1736 assert_eq!(model.max_tokens, 131072);
1737
1738 // Nothing reported -> the optimistic unloaded-context assumption.
1739 let model = model_from_entry(&entry("m", None, None), None);
1740 assert_eq!(model.max_tokens, ASSUMED_UNLOADED_CONTEXT);
1741 assert!(model.supports_tools);
1742 }
1743
1744 #[test]
1745 fn router_entry_detects_vision_from_modalities() {
1746 let router_entry = ModelEntry {
1747 id: "vlm".to_string(),
1748 meta: None,
1749 architecture: Some(llama_cpp::Architecture {
1750 input_modalities: vec!["text".to_string(), "image".to_string()],
1751 }),
1752 status: Some(llama_cpp::ModelStatus {
1753 value: "unloaded".to_string(),
1754 }),
1755 };
1756 let model = model_from_entry(&router_entry, None);
1757 assert!(model.supports_images);
1758 // Unprobed router models optimistically advertise tools until loaded.
1759 assert!(model.supports_tools);
1760 }
1761
1762 #[test]
1763 fn settings_override_capabilities_and_context() {
1764 let mut models: HashMap<String, llama_cpp::Model> = HashMap::default();
1765 models.insert(
1766 "qwen".to_string(),
1767 llama_cpp::Model::new("qwen", Some("qwen"), Some(8192), false, false, false),
1768 );
1769
1770 let available = vec![AvailableModel {
1771 name: "qwen".to_string(),
1772 display_name: Some("Qwen Coder".to_string()),
1773 max_tokens: 16384,
1774 supports_tools: Some(true),
1775 supports_images: None,
1776 supports_thinking: Some(true),
1777 }];
1778
1779 merge_settings_into_models(&mut models, &available, None);
1780
1781 let model = models.get("qwen").unwrap();
1782 assert_eq!(model.display_name.as_deref(), Some("Qwen Coder"));
1783 assert_eq!(model.max_tokens, 16384);
1784 assert!(model.supports_tools);
1785 assert!(model.supports_thinking);
1786 // Unspecified capability keeps the discovered value.
1787 assert!(!model.supports_images);
1788 }
1789
1790 #[test]
1791 fn capability_cells_update_when_a_model_loads() {
1792 let cells: CapabilityCells = Arc::new(RwLock::new(HashMap::default()));
1793 let settings = LlamaCppSettings {
1794 auto_discover: true,
1795 ..Default::default()
1796 };
1797
1798 // Cold: the optimistic unloaded-context assumption.
1799 let cold = vec![llama_cpp::Model::new(
1800 "m",
1801 Some("m"),
1802 Some(ASSUMED_UNLOADED_CONTEXT),
1803 true,
1804 false,
1805 false,
1806 )];
1807 sync_capability_cells(&cells, &compute_effective_models(&cold, &settings));
1808 assert_eq!(
1809 cells.read().unwrap().get("m").unwrap().max_tokens,
1810 ASSUMED_UNLOADED_CONTEXT
1811 );
1812
1813 // The model loads and reports its real context. The shared map must
1814 // reflect the new value so a model reading it by name (an open
1815 // conversation) is no longer stuck on the cold-start assumption.
1816 let loaded = vec![llama_cpp::Model::new(
1817 "m",
1818 Some("m"),
1819 Some(262_144),
1820 true,
1821 false,
1822 true,
1823 )];
1824 sync_capability_cells(&cells, &compute_effective_models(&loaded, &settings));
1825 assert_eq!(cells.read().unwrap().get("m").unwrap().max_tokens, 262_144);
1826 assert!(cells.read().unwrap().get("m").unwrap().supports_thinking);
1827 }
1828
1829 #[test]
1830 fn request_preserves_assistant_thinking_when_supported() {
1831 let request = build_llama_cpp_request(
1832 "test-model",
1833 false,
1834 LiveCapabilities {
1835 max_tokens: 8192,
1836 supports_tools: false,
1837 supports_thinking: true,
1838 },
1839 LanguageModelRequest {
1840 messages: vec![language_model::LanguageModelRequestMessage {
1841 role: Role::Assistant,
1842 content: vec![
1843 MessageContent::Thinking {
1844 text: "reasoning".to_string(),
1845 signature: None,
1846 },
1847 MessageContent::Text("answer".to_string()),
1848 ],
1849 cache: false,
1850 reasoning_details: None,
1851 }],
1852 ..Default::default()
1853 },
1854 )
1855 .unwrap();
1856
1857 assert_eq!(request.messages.len(), 1);
1858 match &request.messages[0] {
1859 llama_cpp::ChatMessage::Assistant {
1860 content: Some(llama_cpp::MessageContent::Plain(content)),
1861 reasoning_content: Some(reasoning_content),
1862 tool_calls,
1863 } => {
1864 assert_eq!(content, "answer");
1865 assert_eq!(reasoning_content, "reasoning");
1866 assert!(tool_calls.is_empty());
1867 }
1868 message => panic!("unexpected message: {message:?}"),
1869 }
1870 }
1871
1872 #[test]
1873 fn request_skips_assistant_thinking_when_unsupported() {
1874 let request = build_llama_cpp_request(
1875 "test-model",
1876 false,
1877 LiveCapabilities {
1878 max_tokens: 8192,
1879 supports_tools: false,
1880 supports_thinking: false,
1881 },
1882 LanguageModelRequest {
1883 messages: vec![language_model::LanguageModelRequestMessage {
1884 role: Role::Assistant,
1885 content: vec![
1886 MessageContent::Thinking {
1887 text: "reasoning".to_string(),
1888 signature: None,
1889 },
1890 MessageContent::RedactedThinking("encrypted".to_string()),
1891 MessageContent::Text("answer".to_string()),
1892 ],
1893 cache: false,
1894 reasoning_details: None,
1895 }],
1896 ..Default::default()
1897 },
1898 )
1899 .unwrap();
1900
1901 assert_eq!(request.messages.len(), 1);
1902 match &request.messages[0] {
1903 llama_cpp::ChatMessage::Assistant {
1904 content: Some(llama_cpp::MessageContent::Plain(content)),
1905 reasoning_content,
1906 tool_calls,
1907 } => {
1908 assert_eq!(content, "answer");
1909 assert!(reasoning_content.is_none());
1910 assert!(tool_calls.is_empty());
1911 }
1912 message => panic!("unexpected message: {message:?}"),
1913 }
1914 }
1915
1916 #[test]
1917 fn request_preserves_thinking_for_assistant_tool_calls_when_supported() {
1918 let request = build_llama_cpp_request(
1919 "test-model",
1920 false,
1921 LiveCapabilities {
1922 max_tokens: 8192,
1923 supports_tools: true,
1924 supports_thinking: true,
1925 },
1926 LanguageModelRequest {
1927 messages: vec![language_model::LanguageModelRequestMessage {
1928 role: Role::Assistant,
1929 content: vec![
1930 MessageContent::Thinking {
1931 text: "reasoning".to_string(),
1932 signature: None,
1933 },
1934 MessageContent::ToolUse(LanguageModelToolUse {
1935 id: "call_1".into(),
1936 name: "weather".into(),
1937 raw_input: r#"{"city":"Oslo"}"#.to_string(),
1938 input: language_model::LanguageModelToolUseInput::Json(
1939 serde_json::json!({ "city": "Oslo" }),
1940 ),
1941 is_input_complete: true,
1942 thought_signature: None,
1943 }),
1944 ],
1945 cache: false,
1946 reasoning_details: None,
1947 }],
1948 ..Default::default()
1949 },
1950 )
1951 .unwrap();
1952
1953 assert_eq!(request.messages.len(), 1);
1954 match &request.messages[0] {
1955 llama_cpp::ChatMessage::Assistant {
1956 content: None,
1957 reasoning_content: Some(reasoning_content),
1958 tool_calls,
1959 } => {
1960 assert_eq!(reasoning_content, "reasoning");
1961 assert_eq!(tool_calls.len(), 1);
1962 }
1963 message => panic!("unexpected message: {message:?}"),
1964 }
1965 }
1966
1967 #[test]
1968 fn usage_event_precedes_stop_event() {
1969 let mut mapper = LlamaCppEventMapper::new();
1970 let events = mapper.map_event(llama_cpp::ResponseStreamEvent {
1971 model: "test-model".to_string(),
1972 object: "chat.completion.chunk".to_string(),
1973 choices: vec![llama_cpp::ChoiceDelta {
1974 index: 0,
1975 delta: llama_cpp::ResponseMessageDelta {
1976 content: None,
1977 reasoning_content: None,
1978 tool_calls: None,
1979 },
1980 finish_reason: Some("stop".to_string()),
1981 }],
1982 usage: Some(llama_cpp::Usage {
1983 prompt_tokens: 11,
1984 completion_tokens: 7,
1985 total_tokens: 18,
1986 }),
1987 });
1988
1989 assert!(matches!(
1990 events.as_slice(),
1991 [
1992 Ok(LanguageModelCompletionEvent::UsageUpdate(TokenUsage {
1993 input_tokens: 11,
1994 output_tokens: 7,
1995 cache_creation_input_tokens: 0,
1996 cache_read_input_tokens: 0,
1997 })),
1998 Ok(LanguageModelCompletionEvent::Stop(StopReason::EndTurn)),
1999 ]
2000 ));
2001 }
2002
2003 #[test]
2004 fn usage_event_precedes_tool_use_stop_event() {
2005 let mut mapper = LlamaCppEventMapper::new();
2006 let events = mapper.map_event(llama_cpp::ResponseStreamEvent {
2007 model: "test-model".to_string(),
2008 object: "chat.completion.chunk".to_string(),
2009 choices: vec![llama_cpp::ChoiceDelta {
2010 index: 0,
2011 delta: llama_cpp::ResponseMessageDelta {
2012 content: None,
2013 reasoning_content: None,
2014 tool_calls: Some(vec![llama_cpp::ToolCallChunk {
2015 index: 0,
2016 id: Some("tool-call-id".to_string()),
2017 function: Some(llama_cpp::FunctionChunk {
2018 name: Some("test_tool".to_string()),
2019 arguments: Some(r#"{"value":1}"#.to_string()),
2020 }),
2021 }]),
2022 },
2023 finish_reason: Some("tool_calls".to_string()),
2024 }],
2025 usage: Some(llama_cpp::Usage {
2026 prompt_tokens: 13,
2027 completion_tokens: 5,
2028 total_tokens: 18,
2029 }),
2030 });
2031
2032 assert!(matches!(
2033 events.as_slice(),
2034 [
2035 Ok(LanguageModelCompletionEvent::UsageUpdate(TokenUsage {
2036 input_tokens: 13,
2037 output_tokens: 5,
2038 cache_creation_input_tokens: 0,
2039 cache_read_input_tokens: 0,
2040 })),
2041 Ok(LanguageModelCompletionEvent::ToolUse(LanguageModelToolUse {
2042 id,
2043 name,
2044 ..
2045 })),
2046 Ok(LanguageModelCompletionEvent::Stop(StopReason::ToolUse)),
2047 ] if id.to_string() == "tool-call-id" && name.as_ref() == "test_tool"
2048 ));
2049 }
2050
2051 #[gpui::test]
2052 async fn authenticate_fetches_models_after_loading_api_key(cx: &mut TestAppContext) {
2053 cx.update(|cx| {
2054 let settings_store = SettingsStore::test(cx);
2055 cx.set_global(settings_store);
2056 });
2057
2058 let model_request_authorizations = Arc::new(Mutex::new(Vec::new()));
2059 let model_request_count = Arc::new(AtomicUsize::new(0));
2060 let http_client = FakeHttpClient::create({
2061 let model_request_authorizations = model_request_authorizations.clone();
2062 let model_request_count = model_request_count.clone();
2063 move |request| {
2064 let model_request_authorizations = model_request_authorizations.clone();
2065 let model_request_count = model_request_count.clone();
2066 async move {
2067 let path = request.uri().path();
2068 let authorization = request
2069 .headers()
2070 .get("Authorization")
2071 .and_then(|value| value.to_str().ok())
2072 .map(str::to_string);
2073
2074 if path == "/v1/models" {
2075 model_request_authorizations.lock().push(authorization);
2076 let request_index = model_request_count.fetch_add(1, Ordering::SeqCst);
2077 if request_index == 0 {
2078 return Ok(http_client::Response::builder()
2079 .status(503)
2080 .body(http_client::AsyncBody::from("not ready"))?);
2081 }
2082
2083 return Ok(http_client::Response::builder().status(200).body(
2084 http_client::AsyncBody::from(
2085 r#"{"data":[{"id":"test-model","meta":{"n_ctx":4096}}]}"#,
2086 ),
2087 )?);
2088 }
2089
2090 if path == "/props" {
2091 return Ok(http_client::Response::builder()
2092 .status(200)
2093 .body(http_client::AsyncBody::from("{}"))?);
2094 }
2095
2096 Ok(http_client::Response::builder()
2097 .status(404)
2098 .body(http_client::AsyncBody::default())?)
2099 }
2100 }
2101 });
2102 let credentials_provider = Arc::new(FakeCredentialsProvider {
2103 api_key: b"loaded-key".to_vec(),
2104 });
2105 let provider = cx
2106 .update(|cx| LlamaCppLanguageModelProvider::new(http_client, credentials_provider, cx));
2107
2108 cx.run_until_parked();
2109
2110 let result = cx.update(|cx| provider.authenticate(cx)).await;
2111 assert!(
2112 result.is_ok(),
2113 "authenticate should discover models after loading credentials"
2114 );
2115 assert_eq!(
2116 &*model_request_authorizations.lock(),
2117 &[None, Some("Bearer loaded-key".to_string())]
2118 );
2119 }
2120}
2121