Coverage for src/lilbee/providers/litellm_sdk.py: 100%
335 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-08-14 11:46 +0000
1"""litellm implementation of the ``LlmSdkBackend`` Protocol.
3This is the ONLY file in lilbee that imports ``litellm``. When migrating
4to a different SDK (e.g. ``liter-llm``), add a sibling module alongside
5this one and flip the single import in ``providers/factory.py``.
7All knowledge of the litellm wire format (``ollama/`` prefix, OpenAI
8content-parts schema for images) lives here. The semantic layer in
9``sdk_llm_provider`` never touches SDK-specific conventions.
10"""
12from __future__ import annotations
14import base64
15import functools
16import logging
17from collections.abc import Callable, Iterator
18from typing import Any, cast
20import httpx
22from lilbee.core.config import DEFAULT_HTTP_TIMEOUT
23from lilbee.providers.base import ProviderError, ProviderErrorKind
24from lilbee.providers.local_servers import (
25 OLLAMA,
26 detect_local_server,
27 local_server_for_key,
28 openai_models_url,
29)
30from lilbee.providers.model_ref import ProviderModelRef
31from lilbee.providers.sdk_backend import (
32 CompletionRequest,
33 CompletionResult,
34 EmbeddingRequest,
35 EmbeddingResult,
36 RerankRequest,
37 RerankResult,
38 SdkToolCall,
39 SdkToolCallDelta,
40 StreamChunk,
41 detect_backend_name,
42)
44log = logging.getLogger(__name__)
46_PROVIDER_NAME = "remote"
48# Substrings dropped from the "LiteLLM" logger before they reach the user's
49# terminal. Two classes of noise: (1) the model-cost-map fetch failure that
50# LiteLLM logs at WARNING on every offline chat call, and (2) AWS-flavored
51# advisories from sagemaker / bedrock / boto3 / botocore. lilbee's litellm
52# extra deliberately excludes boto3, so the AWS warnings aren't actionable.
53# Compared case-insensitively to catch the mixed-case variants LiteLLM emits.
54_LITELLM_SUPPRESS_SUBSTRINGS = (
55 "failed to fetch remote model cost map",
56 "boto3",
57 "botocore",
58 "sagemaker",
59 "bedrock",
60)
63class _LitellmSubstringFilter(logging.Filter):
64 """Drop ``LiteLLM`` log records whose message contains a suppressed substring."""
66 def __init__(self, needles: tuple[str, ...]) -> None:
67 super().__init__()
68 self._needles = tuple(n.lower() for n in needles)
70 def filter(self, record: logging.LogRecord) -> bool:
71 msg = record.getMessage().lower()
72 return not any(n in msg for n in self._needles)
75def install_litellm_log_filter() -> None:
76 """Attach the ``LiteLLM`` substring filter to the package logger.
78 Called automatically when this module is imported (see the module-top
79 invocation below) so the filter is in place before any litellm call
80 can emit a warning. Exposed as a function so tests can re-apply after
81 clearing the logger.
82 """
83 logging.getLogger("LiteLLM").addFilter(_LitellmSubstringFilter(_LITELLM_SUPPRESS_SUBSTRINGS))
86# Install the filter at module import. lilbee never touches litellm before
87# importing this module, so installing here always beats litellm's first
88# warning to the punch.
89install_litellm_log_filter()
92def _sdk_attr(obj: object, name: str) -> Any:
93 """Read an optional attribute off a litellm response/chunk object (absent -> None).
95 Shared helper for the view adapters' dynamic reads of the SDK's loosely-typed
96 objects, whose tool-call fields are absent (not just ``None``) across litellm
97 chunk shapes.
98 """
99 return getattr(obj, name, None)
102class _LitellmResponseView:
103 """Typed read-only view over a litellm completion-response object.
105 The litellm response shape is not in the SDK's type stubs. This
106 adapter is the one place that knows how to pull ``model``, ``choices``,
107 ``message_content`` and the streaming chunk fields out; SDK drift
108 breaks here rather than across every caller.
109 """
111 def __init__(self, response: Any) -> None:
112 self._response = response
114 @property
115 def model(self) -> str | None:
116 """The model name the SDK echoed back, if any."""
117 value = getattr(self._response, "model", None)
118 return str(value) if value is not None else None
120 def _first_choice(self) -> Any:
121 """First entry of the response's ``choices`` list, or ``None``."""
122 choices = getattr(self._response, "choices", None) or []
123 return choices[0] if choices else None
125 @property
126 def message_content(self) -> str:
127 """Content text of the first choice's message (non-stream path)."""
128 choice = self._first_choice()
129 if choice is None:
130 return ""
131 message = getattr(choice, "message", None)
132 if message is None:
133 return ""
134 return getattr(message, "content", "") or ""
136 @property
137 def delta_content(self) -> str:
138 """Content delta of the first choice (stream-path chunk)."""
139 choice = self._first_choice()
140 if choice is None:
141 return ""
142 delta = getattr(choice, "delta", None)
143 if delta is None:
144 return ""
145 return getattr(delta, "content", "") or ""
147 @property
148 def finish_reason(self) -> str | None:
149 """``finish_reason`` of the first choice, if the SDK populated it."""
150 choice = self._first_choice()
151 return getattr(choice, "finish_reason", None) if choice is not None else None
153 @property
154 def tool_calls(self) -> tuple[SdkToolCall, ...]:
155 """Tool calls from the first choice's message (non-stream path)."""
156 choice = self._first_choice()
157 if choice is None:
158 return ()
159 message = _sdk_attr(choice, "message")
160 if message is None:
161 return ()
162 raw_calls = _sdk_attr(message, "tool_calls") or []
163 return tuple(_extract_tool_call(call) for call in raw_calls)
165 @property
166 def delta_tool_calls(self) -> tuple[SdkToolCallDelta, ...]:
167 """Tool-call deltas from the first choice's streaming delta."""
168 choice = self._first_choice()
169 if choice is None:
170 return ()
171 delta = _sdk_attr(choice, "delta")
172 if delta is None:
173 return ()
174 raw_calls = _sdk_attr(delta, "tool_calls") or []
175 return tuple(
176 _extract_tool_call_delta(call, fallback_index=i) for i, call in enumerate(raw_calls)
177 )
180def _extract_tool_call(call: Any) -> SdkToolCall:
181 """Pull one ``SdkToolCall`` out of a litellm tool-call object."""
182 call_id = str(_sdk_attr(call, "id") or "")
183 function = _sdk_attr(call, "function")
184 name = str(_sdk_attr(function, "name") or "") if function is not None else ""
185 arguments = str(_sdk_attr(function, "arguments") or "") if function is not None else ""
186 return SdkToolCall(id=call_id, name=name, arguments=arguments)
189def _extract_tool_call_delta(call: Any, *, fallback_index: int) -> SdkToolCallDelta:
190 """Pull one ``SdkToolCallDelta`` out of a streaming chunk's tool-call slot.
192 Empty-string ``name`` / ``arguments`` are normalised to ``None`` so the
193 SDK stream shape matches the native worker's deltas (the dispatch's
194 ``_StreamState`` gates on ``is not None``; emitting ``""`` produces a
195 spurious empty ContentBlockDelta on every opener).
196 """
197 raw_index = _sdk_attr(call, "index")
198 index = int(raw_index) if isinstance(raw_index, int) else fallback_index
199 call_id = _sdk_attr(call, "id")
200 function = _sdk_attr(call, "function")
201 raw_name = _sdk_attr(function, "name") if function is not None else None
202 raw_args = _sdk_attr(function, "arguments") if function is not None else None
203 return SdkToolCallDelta(
204 index=index,
205 id=str(call_id) if call_id else None,
206 name=str(raw_name) if raw_name else None,
207 arguments_delta=str(raw_args) if raw_args else None,
208 )
211@functools.cache
212def litellm_available() -> bool:
213 """Return True if the ``litellm`` package is installed.
215 Uses ``importlib.util.find_spec`` rather than ``import litellm`` so the
216 check stays fast on the UI thread. Executing ``litellm`` on Windows
217 with Defender real-time scanning takes seconds (the package loads a
218 long list of provider plugins on first import); the Settings screen
219 builds synchronously and calls this in ``_FEATURE_GATED_GROUPS``, so
220 a real import here blocks the entire TUI on the first Settings open.
221 ``find_spec`` just walks ``sys.path`` to locate the package; the
222 heavy import runs later, in worker threads or remote-call paths
223 where the cost is expected.
224 """
225 import importlib.util
227 return importlib.util.find_spec("litellm") is not None
230_LITELLM_MISSING_MSG = (
231 "Remote and API models need the lilbee[litellm] extra. "
232 "Reinstall with: uv tool install --prerelease=allow 'lilbee[litellm]'"
233)
236def _require_litellm() -> Any:
237 """Import ``litellm`` or raise a user-facing ProviderError with install steps."""
238 try:
239 import litellm
240 except ImportError as exc:
241 raise ProviderError(_LITELLM_MISSING_MSG, provider=_PROVIDER_NAME) from exc
242 return litellm
245def _route_model(ref: ProviderModelRef, api_base: str | None) -> str:
246 """Format *ref* for litellm using the OpenAI ``provider/model`` convention.
248 API and local-server refs already carry their canonical prefix. A bare
249 ``local`` ref forced through the SDK (``llm_provider=remote``) gets the
250 prefix of whichever local server its ``api_base`` points at.
251 """
252 if ref.is_api or local_server_for_key(ref.provider) is not None:
253 return ref.for_openai_prefix()
254 if api_base and (spec := detect_local_server(api_base)) is not None:
255 return spec.qualify(ref.name)
256 return ref.name
259def _format_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
260 """Convert messages with inline image bytes into OpenAI content parts.
262 litellm routes to OpenAI-compatible endpoints that expect the
263 ``{"type": "image_url", "image_url": {...}}`` content-parts schema
264 for multimodal input. Messages without ``images`` pass through
265 untouched.
266 """
267 formatted: list[dict[str, Any]] = []
268 for msg in messages:
269 if "images" in msg:
270 content_parts: list[dict[str, Any]] = [{"type": "text", "text": msg.get("content", "")}]
271 for img in msg["images"]:
272 if isinstance(img, bytes):
273 b64 = base64.b64encode(img).decode()
274 content_parts.append(
275 {
276 "type": "image_url",
277 "image_url": {"url": f"data:image/png;base64,{b64}"},
278 }
279 )
280 formatted.append({"role": msg["role"], "content": content_parts})
281 else:
282 formatted.append(msg)
283 return formatted
286# User-facing message per recognised error kind. Each names the problem against
287# {model} and makes clear the cause sits with the user's provider account or
288# network, not with lilbee. UNKNOWN has no entry and falls back to the raw error.
289_KIND_MESSAGES: dict[ProviderErrorKind, str] = {
290 ProviderErrorKind.RATE_LIMIT: (
291 "{model} is rate-limited or out of quota. That's a limit on your provider "
292 "API key, not a lilbee problem. Check your plan and billing with the "
293 "provider, or pick a different model."
294 ),
295 ProviderErrorKind.AUTH: (
296 "{model} rejected your API key. Check that the key is set correctly and has "
297 "access to this model. That's between your key and the provider, not a lilbee problem."
298 ),
299 ProviderErrorKind.NOT_FOUND: (
300 "The provider doesn't offer {model} on your account. "
301 "Pick a different model or check the name."
302 ),
303 ProviderErrorKind.CONTEXT_OVERFLOW: (
304 "This conversation is too long for {model}'s context window. "
305 "Start a new chat or pick a model with a larger context."
306 ),
307 ProviderErrorKind.BAD_REQUEST: (
308 "The provider rejected the request for {model}. Check the model name and your settings."
309 ),
310 ProviderErrorKind.CONNECTION: (
311 "Couldn't reach the provider for {model}, or it timed out. Check your "
312 "connection and base URL, then try again or pick a different model."
313 ),
314 ProviderErrorKind.SERVER: (
315 "The provider for {model} is unavailable right now. That's on the provider's "
316 "side, not a lilbee problem. Try again shortly or pick a different model."
317 ),
318}
321def _embedding_index(item: Any) -> int:
322 """Return an embedding item's ``index`` across the dict and object response shapes.
324 The OpenAI embeddings response always carries ``index``; mirrors the rerank
325 path's direct read rather than a defaulted lookup.
326 """
327 idx = item["index"] if isinstance(item, dict) else item.index
328 return int(idx)
331def _embedding_vector(item: Any) -> list[float]:
332 """Return an embedding item's vector across the dict and object response shapes."""
333 vector = item["embedding"] if isinstance(item, dict) else item.embedding
334 return cast("list[float]", vector)
337def _response_model(response: Any) -> str | None:
338 """Return a litellm response's ``model`` across the dict and object shapes.
340 Optional (a proxy may omit it), so the lookup defaults to ``None``.
341 """
342 if isinstance(response, dict):
343 return response.get("model")
344 return cast("str | None", _sdk_attr(response, "model"))
347# Operation labels prefixed onto the fallback message for an unrecognised error.
348_CHAT_FAILED = "Chat failed"
349_EMBED_FAILED = "Embedding failed"
350_RERANK_FAILED = "Rerank failed"
353def _cause_chain(exc: BaseException) -> list[BaseException]:
354 """Return *exc* and its causes, root cause first.
356 litellm's mid-stream fallback keeps the real cause in ``original_exception``;
357 walking root-first stops a 503 wrapper from masking the 429 it carries.
358 """
359 chain: list[BaseException] = []
360 seen: set[int] = set()
361 cur: BaseException | None = exc
362 while cur is not None and id(cur) not in seen:
363 seen.add(id(cur))
364 chain.append(cur)
365 nxt = getattr(cur, "original_exception", None)
366 if not isinstance(nxt, BaseException):
367 nxt = cur.__cause__
368 cur = nxt if isinstance(nxt, BaseException) else None
369 chain.reverse()
370 return chain
373def _classify_litellm_error(exc: BaseException) -> ProviderErrorKind:
374 """Map a litellm exception to a ``ProviderErrorKind`` by type, never by message.
376 litellm normalises every backend's failures into one exception hierarchy, so
377 the same mapping covers all providers. The MRO walk picks the most specific
378 kind (``ContextWindowExceededError`` over its ``BadRequestError`` base).
379 """
380 try:
381 import litellm
382 except ImportError: # pragma: no cover - unreachable after a real litellm call
383 return ProviderErrorKind.UNKNOWN
384 table: dict[type, ProviderErrorKind] = {
385 litellm.AuthenticationError: ProviderErrorKind.AUTH,
386 litellm.PermissionDeniedError: ProviderErrorKind.AUTH,
387 litellm.NotFoundError: ProviderErrorKind.NOT_FOUND,
388 litellm.RateLimitError: ProviderErrorKind.RATE_LIMIT,
389 litellm.ContextWindowExceededError: ProviderErrorKind.CONTEXT_OVERFLOW,
390 litellm.BadRequestError: ProviderErrorKind.BAD_REQUEST,
391 litellm.Timeout: ProviderErrorKind.CONNECTION,
392 litellm.APIConnectionError: ProviderErrorKind.CONNECTION,
393 litellm.ServiceUnavailableError: ProviderErrorKind.SERVER,
394 litellm.InternalServerError: ProviderErrorKind.SERVER,
395 }
396 for err in _cause_chain(exc):
397 for cls in type(err).__mro__:
398 kind = table.get(cls)
399 if kind is not None:
400 return kind
401 return ProviderErrorKind.UNKNOWN
404def _provider_error(fallback: str, exc: Exception, model: str) -> ProviderError:
405 """Wrap a litellm failure as a ``ProviderError`` classified by type.
407 Recognised kinds get a blob-free, user-facing message; unrecognised ones
408 keep the raw ``{fallback}: {exc}`` shape so nothing is lost when debugging.
409 """
410 kind = _classify_litellm_error(exc)
411 template = _KIND_MESSAGES.get(kind)
412 message = template.format(model=model) if template is not None else f"{fallback}: {exc}"
413 return ProviderError(message, provider=_PROVIDER_NAME, kind=kind)
416class LitellmSdkBackend:
417 """``LlmSdkBackend`` adapter backed by the ``litellm`` SDK."""
419 @property
420 def provider_name(self) -> str:
421 """Stable identifier used when wrapping errors in ``ProviderError``."""
422 return _PROVIDER_NAME
424 def active_backend_name(self, base_url: str) -> str:
425 """Return the display name of the backend ``base_url`` points at."""
426 return detect_backend_name(base_url)
428 def available(self) -> bool:
429 """Return True if the underlying SDK is installed."""
430 return litellm_available()
432 def supports_tools(self, _model_ref: str) -> bool:
433 """Optimistic: all SDK-routed refs report tool support.
435 A model that lacks a tool template just returns an empty
436 ``tool_calls`` array, which the dispatch handles as a normal
437 end-of-turn.
438 """
439 return True
441 def configure_logging(self, *, suppress_debug: bool) -> None:
442 """Apply litellm's debug-info suppression toggle when requested."""
443 if not suppress_debug:
444 return
445 try:
446 import litellm
448 litellm.suppress_debug_info = True
449 except ImportError:
450 pass # debug-suppression is best-effort when the litellm extra is absent
452 def complete(self, request: CompletionRequest) -> CompletionResult:
453 """Run a single-shot completion through ``litellm.completion``."""
454 litellm = _require_litellm()
455 kwargs = self._completion_kwargs(request, stream=False)
456 try:
457 response = litellm.completion(**kwargs)
458 except Exception as exc:
459 raise _provider_error(_CHAT_FAILED, exc, request.ref.for_display()) from exc
460 view = _LitellmResponseView(response)
461 return CompletionResult(
462 content=view.message_content,
463 finish_reason=view.finish_reason,
464 model=view.model,
465 tool_calls=view.tool_calls,
466 )
468 def complete_stream(self, request: CompletionRequest) -> Iterator[StreamChunk]:
469 """Stream a completion through ``litellm.completion(stream=True)``."""
470 litellm = _require_litellm()
471 kwargs = self._completion_kwargs(request, stream=True)
472 model = request.ref.for_display()
473 try:
474 response = litellm.completion(**kwargs)
475 except Exception as exc:
476 raise _provider_error(_CHAT_FAILED, exc, model) from exc
477 return self._stream_chunks(response, model)
479 @staticmethod
480 def _stream_chunks(response: Any, model: str) -> Iterator[StreamChunk]:
481 """Yield ``StreamChunk`` values from a litellm streaming response.
483 Exceptions raised mid-iteration are classified into ``ProviderError``
484 so the semantic layer sees a consistent error type regardless of
485 where the SDK failed.
486 """
487 try:
488 for chunk in response:
489 view = _LitellmResponseView(chunk)
490 content = view.delta_content
491 finish_reason = view.finish_reason
492 tool_call_deltas = view.delta_tool_calls
493 if content or finish_reason or tool_call_deltas:
494 yield StreamChunk(
495 content=content,
496 finish_reason=finish_reason,
497 tool_call_deltas=tool_call_deltas,
498 )
499 except ProviderError:
500 raise
501 except Exception as exc:
502 raise _provider_error(_CHAT_FAILED, exc, model) from exc
504 @staticmethod
505 def _completion_kwargs(request: CompletionRequest, *, stream: bool) -> dict[str, Any]:
506 """Translate a ``CompletionRequest`` into litellm kwargs."""
507 kwargs: dict[str, Any] = {
508 "model": _route_model(request.ref, request.api_base),
509 "messages": _format_messages(request.messages),
510 "stream": stream,
511 }
512 if request.api_base:
513 kwargs["api_base"] = request.api_base
514 if request.api_key:
515 kwargs["api_key"] = request.api_key
516 if request.options:
517 kwargs.update(request.options)
518 if "response_format" in kwargs:
519 # Best-effort: a provider without structured-output support should
520 # drop the field and answer normally, not refuse the call. Callers
521 # that send it parse the reply defensively either way.
522 kwargs["drop_params"] = True
523 return kwargs
525 def embed(self, request: EmbeddingRequest) -> EmbeddingResult:
526 """Embed inputs through ``litellm.embedding``."""
527 litellm = _require_litellm()
528 kwargs: dict[str, Any] = {
529 "model": _route_model(request.ref, request.api_base),
530 "input": request.inputs,
531 }
532 if request.api_base:
533 kwargs["api_base"] = request.api_base
534 if request.api_key:
535 kwargs["api_key"] = request.api_key
536 try:
537 response = litellm.embedding(**kwargs)
538 except Exception as exc:
539 raise _provider_error(_EMBED_FAILED, exc, request.ref.for_display()) from exc
540 data = response["data"] if isinstance(response, dict) else response.data
541 # Order by the response's ``index`` rather than arrival order: a proxy or
542 # gateway may return the batch out of order, and the consumer zips vectors
543 # to inputs positionally, so a reorder would silently mis-pair every chunk
544 # with the wrong vector. ``index`` is required (always present in a
545 # spec-conforming response), mirroring the rerank path's direct read.
546 ordered = sorted(data, key=_embedding_index)
547 # Reordering is not the only way the positional zip breaks. A gateway that
548 # drops an item, or repeats an index, yields a batch that still sorts
549 # cleanly but no longer corresponds one-to-one with the inputs, and the
550 # consumer would pair every later chunk with the wrong vector and store it.
551 # A spec-conforming response carries exactly one item per input, indexed
552 # 0..n-1, so anything else is refused rather than silently mis-paired.
553 expected = len(request.inputs)
554 if [_embedding_index(item) for item in ordered] != list(range(expected)):
555 raise ProviderError(
556 f"Embedding response does not match the request: expected {expected} "
557 f"vectors indexed 0-{expected - 1}, got {len(ordered)}. The endpoint "
558 "returned an incomplete or misindexed batch.",
559 provider=_PROVIDER_NAME,
560 kind=ProviderErrorKind.SERVER,
561 )
562 vectors = [_embedding_vector(item) for item in ordered]
563 return EmbeddingResult(vectors=vectors, model=_response_model(response))
565 def rerank(self, request: RerankRequest) -> RerankResult:
566 """Rerank documents via ``litellm.rerank`` (Cohere, Voyage, Jina, Together, HF TEI).
568 The SDK returns results sorted by relevance; we restore input
569 order via each result's ``index`` so scores line up with the
570 caller's ``candidates`` list.
571 """
572 if not request.candidates:
573 return RerankResult(scores=[])
574 litellm = _require_litellm()
575 kwargs: dict[str, Any] = {
576 "model": _route_model(request.ref, request.api_base),
577 "query": request.query,
578 "documents": request.candidates,
579 }
580 if request.api_base:
581 kwargs["api_base"] = request.api_base
582 if request.api_key:
583 kwargs["api_key"] = request.api_key
584 try:
585 response = litellm.rerank(**kwargs)
586 except Exception as exc:
587 raise _provider_error(_RERANK_FAILED, exc, request.ref.for_display()) from exc
588 results = response["results"] if isinstance(response, dict) else response.results
589 scores = [0.0] * len(request.candidates)
590 for item in results:
591 idx = item["index"] if isinstance(item, dict) else item.index
592 score = item["relevance_score"] if isinstance(item, dict) else item.relevance_score
593 scores[idx] = float(score)
594 return RerankResult(scores=scores, model=_response_model(response))
596 def list_models(self, *, base_url: str, api_key: str) -> list[str]:
597 """List models from Ollama (``/api/tags``) or an OpenAI-compatible ``/v1/models``."""
598 clean_base = base_url.rstrip("/")
599 spec = detect_local_server(clean_base)
600 if spec is OLLAMA:
601 return self._list_ollama_models(clean_base)
602 return self._list_openai_models(clean_base, api_key)
604 def list_chat_models(self, provider: str) -> list[str]:
605 """Return chat-mode model ids from litellm's static catalog.
607 Returns whatever litellm exposes for *provider*, alphabetically.
608 Empty list when litellm is not installed or the provider has no
609 chat-mode entries.
610 """
611 try:
612 import litellm
613 except ImportError:
614 return []
615 return self._all_chat_models_for(provider, litellm)
617 @staticmethod
618 def _all_chat_models_for(provider: str, litellm: Any) -> list[str]:
619 """Filter litellm's catalog down to chat-mode entries for ``provider``.
621 litellm's catalog stores some providers' models bare (``gpt-4o``)
622 and others prefixed (``mistral/codestral-latest``,
623 ``openrouter/anthropic/claude-3.5-sonnet``). Strip any leading
624 ``{provider}/`` so callers see uniformly bare names; the canonical
625 ``provider/name`` form is reapplied at the routing layer via
626 :meth:`ProviderModelRef.for_openai_prefix`.
627 """
628 models = litellm.models_by_provider.get(provider, set())
629 prefix = f"{provider}/"
630 bare: set[str] = set()
631 for model_name in models:
632 info = litellm.model_cost.get(model_name, {})
633 if info.get("mode") != "chat":
634 continue
635 bare.add(model_name.removeprefix(prefix))
636 return sorted(bare)
638 @staticmethod
639 def _list_ollama_models(base_url: str) -> list[str]:
640 """List models via the Ollama ``/api/tags`` endpoint."""
641 try:
642 resp = httpx.get(f"{base_url}/api/tags", timeout=DEFAULT_HTTP_TIMEOUT)
643 resp.raise_for_status()
644 data = resp.json()
645 return [m["name"] for m in data.get("models", [])]
646 except httpx.HTTPError as exc:
647 raise ProviderError(f"Cannot list models: {exc}", provider=_PROVIDER_NAME) from exc
649 @staticmethod
650 def _list_openai_models(base_url: str, api_key: str) -> list[str]:
651 """List models via an OpenAI-compatible ``/v1/models`` endpoint."""
652 headers: dict[str, str] = {}
653 if api_key:
654 headers["Authorization"] = f"Bearer {api_key}"
655 try:
656 resp = httpx.get(
657 openai_models_url(base_url), headers=headers, timeout=DEFAULT_HTTP_TIMEOUT
658 )
659 resp.raise_for_status()
660 data = resp.json()
661 return [m["id"] for m in data.get("data", [])]
662 except httpx.HTTPError:
663 log.debug("Failed to list models via /v1/models", exc_info=True)
664 return []
666 def pull_model(
667 self,
668 model: str,
669 *,
670 base_url: str,
671 on_progress: Callable[..., Any] | None = None,
672 ) -> None:
673 """Refuse to pull: local servers (Ollama, LM Studio) are read-only.
675 Their models are managed in their own app and surface here once
676 present, so lilbee never downloads them over the network.
677 """
678 spec = detect_local_server(base_url.rstrip("/"))
679 server = spec.display_name if spec is not None else "This server"
680 raise ProviderError(
681 f"{server} doesn't download models over the network. "
682 f"Add the model in its own app, then pick it here.",
683 provider=_PROVIDER_NAME,
684 )
686 def show_model(self, model: str, *, base_url: str) -> dict[str, Any] | None:
687 """Get model info via the Ollama ``/api/show`` endpoint.
689 Returns the raw ``parameters`` text and the ``capabilities`` list
690 (newer Ollama versions) so callers can check for vision support.
691 Returns ``None`` for servers without a metadata endpoint (LM Studio).
692 """
693 clean_base = base_url.rstrip("/")
694 spec = detect_local_server(clean_base)
695 if spec is None or not spec.supports_show:
696 return None
697 # Ollama's API uses bare model names; the routing-layer prefix has
698 # to come off before the request goes out.
699 ollama_name = model.removeprefix(OLLAMA.wire_prefix)
700 try:
701 resp = httpx.post(
702 f"{clean_base}/api/show",
703 json={"name": ollama_name},
704 timeout=DEFAULT_HTTP_TIMEOUT,
705 )
706 resp.raise_for_status()
707 data = resp.json()
708 except httpx.HTTPError:
709 return None
711 result: dict[str, Any] = {}
713 params = data.get("parameters", "")
714 if isinstance(params, str) and params:
715 result["parameters"] = params
716 elif params:
717 result["parameters"] = str(params)
719 capabilities = data.get("capabilities")
720 if isinstance(capabilities, list):
721 result["capabilities"] = capabilities
723 return result or None