Coverage for src/lilbee/providers/litellm_sdk.py: 100%
350 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-09-28 17:20 +0000
1"""litellm implementation of the ``LlmSdkBackend`` Protocol.
3This is the ONLY file in lilbee that imports ``litellm``. When migrating
4to a different SDK (e.g. ``liter-llm``), add a sibling module alongside
5this one and flip the single import in ``providers/factory.py``.
7All knowledge of the litellm wire format (``ollama/`` prefix, OpenAI
8content-parts schema for images) lives here. The semantic layer in
9``sdk_llm_provider`` never touches SDK-specific conventions.
10"""
12from __future__ import annotations
14import base64
15import functools
16import logging
17from collections.abc import Callable, Iterator
18from importlib.metadata import PackageNotFoundError
19from importlib.metadata import version as _dist_version
20from typing import Any, cast
22import httpx
24from lilbee.core.config import DEFAULT_HTTP_TIMEOUT
25from lilbee.providers.base import ProviderError, ProviderErrorKind
26from lilbee.providers.local_servers import (
27 OLLAMA,
28 detect_local_server,
29 local_server_for_key,
30 openai_models_url,
31)
32from lilbee.providers.model_ref import ProviderModelRef
33from lilbee.providers.sdk_backend import (
34 CompletionRequest,
35 CompletionResult,
36 EmbeddingRequest,
37 EmbeddingResult,
38 RerankRequest,
39 RerankResult,
40 SdkToolCall,
41 SdkToolCallDelta,
42 StreamChunk,
43 detect_backend_name,
44)
46log = logging.getLogger(__name__)
48_PROVIDER_NAME = "remote"
50# Substrings dropped from the "LiteLLM" logger before they reach the user's
51# terminal. Two classes of noise: (1) the model-cost-map fetch failure that
52# LiteLLM logs at WARNING on every offline chat call, and (2) AWS-flavored
53# advisories from sagemaker / bedrock / boto3 / botocore. lilbee's litellm
54# extra deliberately excludes boto3, so the AWS warnings aren't actionable.
55# Compared case-insensitively to catch the mixed-case variants LiteLLM emits.
56_LITELLM_SUPPRESS_SUBSTRINGS = (
57 "failed to fetch remote model cost map",
58 "boto3",
59 "botocore",
60 "sagemaker",
61 "bedrock",
62)
65class _LitellmSubstringFilter(logging.Filter):
66 """Drop ``LiteLLM`` log records whose message contains a suppressed substring."""
68 def __init__(self, needles: tuple[str, ...]) -> None:
69 super().__init__()
70 self._needles = tuple(n.lower() for n in needles)
72 def filter(self, record: logging.LogRecord) -> bool:
73 msg = record.getMessage().lower()
74 return not any(n in msg for n in self._needles)
77def install_litellm_log_filter() -> None:
78 """Attach the ``LiteLLM`` substring filter to the package logger.
80 Called automatically when this module is imported (see the module-top
81 invocation below) so the filter is in place before any litellm call
82 can emit a warning. Exposed as a function so tests can re-apply after
83 clearing the logger.
84 """
85 logging.getLogger("LiteLLM").addFilter(_LitellmSubstringFilter(_LITELLM_SUPPRESS_SUBSTRINGS))
88# Install the filter at module import. lilbee never touches litellm before
89# importing this module, so installing here always beats litellm's first
90# warning to the punch.
91install_litellm_log_filter()
94def _sdk_attr(obj: object, name: str) -> Any:
95 """Read an optional attribute off a litellm response/chunk object (absent -> None).
97 Shared helper for the view adapters' dynamic reads of the SDK's loosely-typed
98 objects, whose tool-call fields are absent (not just ``None``) across litellm
99 chunk shapes.
100 """
101 return getattr(obj, name, None)
104class _LitellmResponseView:
105 """Typed read-only view over a litellm completion-response object.
107 The litellm response shape is not in the SDK's type stubs. This
108 adapter is the one place that knows how to pull ``model``, ``choices``,
109 ``message_content`` and the streaming chunk fields out; SDK drift
110 breaks here rather than across every caller.
111 """
113 def __init__(self, response: Any) -> None:
114 self._response = response
116 @property
117 def model(self) -> str | None:
118 """The model name the SDK echoed back, if any."""
119 value = getattr(self._response, "model", None)
120 return str(value) if value is not None else None
122 def _first_choice(self) -> Any:
123 """First entry of the response's ``choices`` list, or ``None``."""
124 choices = getattr(self._response, "choices", None) or []
125 return choices[0] if choices else None
127 @property
128 def message_content(self) -> str:
129 """Content text of the first choice's message (non-stream path)."""
130 choice = self._first_choice()
131 if choice is None:
132 return ""
133 message = getattr(choice, "message", None)
134 if message is None:
135 return ""
136 return getattr(message, "content", "") or ""
138 @property
139 def delta_content(self) -> str:
140 """Content delta of the first choice (stream-path chunk)."""
141 choice = self._first_choice()
142 if choice is None:
143 return ""
144 delta = getattr(choice, "delta", None)
145 if delta is None:
146 return ""
147 return getattr(delta, "content", "") or ""
149 @property
150 def finish_reason(self) -> str | None:
151 """``finish_reason`` of the first choice, if the SDK populated it."""
152 choice = self._first_choice()
153 return getattr(choice, "finish_reason", None) if choice is not None else None
155 @property
156 def tool_calls(self) -> tuple[SdkToolCall, ...]:
157 """Tool calls from the first choice's message (non-stream path)."""
158 choice = self._first_choice()
159 if choice is None:
160 return ()
161 message = _sdk_attr(choice, "message")
162 if message is None:
163 return ()
164 raw_calls = _sdk_attr(message, "tool_calls") or []
165 return tuple(_extract_tool_call(call) for call in raw_calls)
167 @property
168 def delta_tool_calls(self) -> tuple[SdkToolCallDelta, ...]:
169 """Tool-call deltas from the first choice's streaming delta."""
170 choice = self._first_choice()
171 if choice is None:
172 return ()
173 delta = _sdk_attr(choice, "delta")
174 if delta is None:
175 return ()
176 raw_calls = _sdk_attr(delta, "tool_calls") or []
177 return tuple(
178 _extract_tool_call_delta(call, fallback_index=i) for i, call in enumerate(raw_calls)
179 )
182def _extract_tool_call(call: Any) -> SdkToolCall:
183 """Pull one ``SdkToolCall`` out of a litellm tool-call object."""
184 call_id = str(_sdk_attr(call, "id") or "")
185 function = _sdk_attr(call, "function")
186 name = str(_sdk_attr(function, "name") or "") if function is not None else ""
187 arguments = str(_sdk_attr(function, "arguments") or "") if function is not None else ""
188 return SdkToolCall(id=call_id, name=name, arguments=arguments)
191def _extract_tool_call_delta(call: Any, *, fallback_index: int) -> SdkToolCallDelta:
192 """Pull one ``SdkToolCallDelta`` out of a streaming chunk's tool-call slot.
194 Empty-string ``name`` / ``arguments`` are normalised to ``None`` so the
195 SDK stream shape matches the native worker's deltas (the dispatch's
196 ``_StreamState`` gates on ``is not None``; emitting ``""`` produces a
197 spurious empty ContentBlockDelta on every opener).
198 """
199 raw_index = _sdk_attr(call, "index")
200 index = int(raw_index) if isinstance(raw_index, int) else fallback_index
201 call_id = _sdk_attr(call, "id")
202 function = _sdk_attr(call, "function")
203 raw_name = _sdk_attr(function, "name") if function is not None else None
204 raw_args = _sdk_attr(function, "arguments") if function is not None else None
205 return SdkToolCallDelta(
206 index=index,
207 id=str(call_id) if call_id else None,
208 name=str(raw_name) if raw_name else None,
209 arguments_delta=str(raw_args) if raw_args else None,
210 )
213_UV_EXCLUDES_URL = "https://lilbee.sh/uv-excludes.txt"
215LITELLM_MISSING_MSG = (
216 "Remote and API models need the lilbee[litellm] extra. If you installed lilbee with "
217 "uv tool install, run your install command again with litellm added to the extras, "
218 f"and keep --excludes {_UV_EXCLUDES_URL} if you use the crawler. If you installed "
219 "lilbee with pip, run: pip install 'lilbee[litellm]'. If the crawler extra is also "
220 "installed, lilbee then asks you to remove the unclecode-litellm fork and shows how."
221)
223_LITELLM_DIST = "litellm"
224# crawl4ai requires unclecode-litellm, a fork that installs into litellm's package
225# directory, so the litellm package alone does not tell which one is installed.
226_LITELLM_FORK_DIST = "unclecode-litellm"
227_LITELLM_FORK_MSG = (
228 "Both litellm and unclecode-litellm are installed. unclecode-litellm is a fork of "
229 "litellm that crawl4ai requires, and it writes into the same litellm package, so "
230 "remote and API models cannot load reliably. If you installed lilbee with "
231 "uv tool install, run your install command again with --reinstall --excludes "
232 f"{_UV_EXCLUDES_URL} added. If you installed lilbee with pip, run: "
233 "pip uninstall -y unclecode-litellm && "
234 "pip install --force-reinstall --no-deps litellm=={version}"
235)
238def _installed_version(dist: str) -> str | None:
239 """Return the installed version of distribution *dist*, or None when it is absent."""
240 try:
241 return _dist_version(dist)
242 except PackageNotFoundError:
243 return None
246@functools.cache
247def litellm_available() -> bool:
248 """Return True if the ``litellm`` distribution is installed, not only crawl4ai's fork.
250 Reads package metadata instead of importing ``litellm``, whose first import
251 takes seconds on Windows and would block the Settings screen's compose.
252 """
253 return _installed_version(_LITELLM_DIST) is not None
256def _require_litellm() -> Any:
257 """Import ``litellm`` or raise a user-facing ProviderError with install steps."""
258 litellm_version = _installed_version(_LITELLM_DIST)
259 if litellm_version is None:
260 raise ProviderError(LITELLM_MISSING_MSG, provider=_PROVIDER_NAME)
261 # Temporary: pip cannot exclude the fork. Remove once crawl4ai ships
262 # unclecode/crawl4ai#2107.
263 if _installed_version(_LITELLM_FORK_DIST) is not None:
264 raise ProviderError(
265 _LITELLM_FORK_MSG.format(version=litellm_version), provider=_PROVIDER_NAME
266 )
267 try:
268 import litellm
269 except ImportError as exc:
270 raise ProviderError(LITELLM_MISSING_MSG, provider=_PROVIDER_NAME) from exc
271 return litellm
274def _route_model(ref: ProviderModelRef, api_base: str | None) -> str:
275 """Format *ref* for litellm using the OpenAI ``provider/model`` convention.
277 API and local-server refs already carry their canonical prefix. A bare
278 ``local`` ref forced through the SDK (``llm_provider=remote``) gets the
279 prefix of whichever local server its ``api_base`` points at.
280 """
281 if ref.is_api or local_server_for_key(ref.provider) is not None:
282 return ref.for_openai_prefix()
283 if api_base and (spec := detect_local_server(api_base)) is not None:
284 return spec.qualify(ref.name)
285 return ref.name
288def _format_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
289 """Convert messages with inline image bytes into OpenAI content parts.
291 litellm routes to OpenAI-compatible endpoints that expect the
292 ``{"type": "image_url", "image_url": {...}}`` content-parts schema
293 for multimodal input. Messages without ``images`` pass through
294 untouched.
295 """
296 formatted: list[dict[str, Any]] = []
297 for msg in messages:
298 if "images" in msg:
299 content_parts: list[dict[str, Any]] = [{"type": "text", "text": msg.get("content", "")}]
300 for img in msg["images"]:
301 if isinstance(img, bytes):
302 b64 = base64.b64encode(img).decode()
303 content_parts.append(
304 {
305 "type": "image_url",
306 "image_url": {"url": f"data:image/png;base64,{b64}"},
307 }
308 )
309 formatted.append({"role": msg["role"], "content": content_parts})
310 else:
311 formatted.append(msg)
312 return formatted
315# User-facing message per recognised error kind. Each names the problem against
316# {model} and makes clear the cause sits with the user's provider account or
317# network, not with lilbee. UNKNOWN has no entry and falls back to the raw error.
318_KIND_MESSAGES: dict[ProviderErrorKind, str] = {
319 ProviderErrorKind.RATE_LIMIT: (
320 "{model} is rate-limited or out of quota. That's a limit on your provider "
321 "API key, not a lilbee problem. Check your plan and billing with the "
322 "provider, or pick a different model."
323 ),
324 ProviderErrorKind.AUTH: (
325 "{model} rejected your API key. Check that the key is set correctly and has "
326 "access to this model. That's between your key and the provider, not a lilbee problem."
327 ),
328 ProviderErrorKind.NOT_FOUND: (
329 "The provider doesn't offer {model} on your account. "
330 "Pick a different model or check the name."
331 ),
332 ProviderErrorKind.CONTEXT_OVERFLOW: (
333 "This conversation is too long for {model}'s context window. "
334 "Start a new chat or pick a model with a larger context."
335 ),
336 ProviderErrorKind.BAD_REQUEST: (
337 "The provider rejected the request for {model}. Check the model name and your settings."
338 ),
339 ProviderErrorKind.CONNECTION: (
340 "Couldn't reach the provider for {model}, or it timed out. Check your "
341 "connection and base URL, then try again or pick a different model."
342 ),
343 ProviderErrorKind.SERVER: (
344 "The provider for {model} is unavailable right now. That's on the provider's "
345 "side, not a lilbee problem. Try again shortly or pick a different model."
346 ),
347}
350def _embedding_index(item: Any) -> int:
351 """Return an embedding item's ``index`` across the dict and object response shapes.
353 The OpenAI embeddings response always carries ``index``; mirrors the rerank
354 path's direct read rather than a defaulted lookup.
355 """
356 idx = item["index"] if isinstance(item, dict) else item.index
357 return int(idx)
360def _embedding_vector(item: Any) -> list[float]:
361 """Return an embedding item's vector across the dict and object response shapes."""
362 vector = item["embedding"] if isinstance(item, dict) else item.embedding
363 return cast("list[float]", vector)
366def _response_model(response: Any) -> str | None:
367 """Return a litellm response's ``model`` across the dict and object shapes.
369 Optional (a proxy may omit it), so the lookup defaults to ``None``.
370 """
371 if isinstance(response, dict):
372 return response.get("model")
373 return cast("str | None", _sdk_attr(response, "model"))
376# Operation labels prefixed onto the fallback message for an unrecognised error.
377_CHAT_FAILED = "Chat failed"
378_EMBED_FAILED = "Embedding failed"
379_RERANK_FAILED = "Rerank failed"
382def _cause_chain(exc: BaseException) -> list[BaseException]:
383 """Return *exc* and its causes, root cause first.
385 litellm's mid-stream fallback keeps the real cause in ``original_exception``;
386 walking root-first stops a 503 wrapper from masking the 429 it carries.
387 """
388 chain: list[BaseException] = []
389 seen: set[int] = set()
390 cur: BaseException | None = exc
391 while cur is not None and id(cur) not in seen:
392 seen.add(id(cur))
393 chain.append(cur)
394 nxt = getattr(cur, "original_exception", None)
395 if not isinstance(nxt, BaseException):
396 nxt = cur.__cause__
397 cur = nxt if isinstance(nxt, BaseException) else None
398 chain.reverse()
399 return chain
402def _classify_litellm_error(exc: BaseException) -> ProviderErrorKind:
403 """Map a litellm exception to a ``ProviderErrorKind`` by type, never by message.
405 litellm normalises every backend's failures into one exception hierarchy, so
406 the same mapping covers all providers. The MRO walk picks the most specific
407 kind (``ContextWindowExceededError`` over its ``BadRequestError`` base).
408 """
409 try:
410 import litellm
411 except ImportError: # pragma: no cover - unreachable after a real litellm call
412 return ProviderErrorKind.UNKNOWN
413 table: dict[type, ProviderErrorKind] = {
414 litellm.AuthenticationError: ProviderErrorKind.AUTH,
415 litellm.PermissionDeniedError: ProviderErrorKind.AUTH,
416 litellm.NotFoundError: ProviderErrorKind.NOT_FOUND,
417 litellm.RateLimitError: ProviderErrorKind.RATE_LIMIT,
418 litellm.ContextWindowExceededError: ProviderErrorKind.CONTEXT_OVERFLOW,
419 litellm.BadRequestError: ProviderErrorKind.BAD_REQUEST,
420 litellm.Timeout: ProviderErrorKind.CONNECTION,
421 litellm.APIConnectionError: ProviderErrorKind.CONNECTION,
422 litellm.ServiceUnavailableError: ProviderErrorKind.SERVER,
423 litellm.InternalServerError: ProviderErrorKind.SERVER,
424 }
425 for err in _cause_chain(exc):
426 for cls in type(err).__mro__:
427 kind = table.get(cls)
428 if kind is not None:
429 return kind
430 return ProviderErrorKind.UNKNOWN
433def _provider_error(fallback: str, exc: Exception, model: str) -> ProviderError:
434 """Wrap a litellm failure as a ``ProviderError`` classified by type.
436 Recognised kinds get a blob-free, user-facing message; unrecognised ones
437 keep the raw ``{fallback}: {exc}`` shape so nothing is lost when debugging.
438 """
439 kind = _classify_litellm_error(exc)
440 template = _KIND_MESSAGES.get(kind)
441 message = template.format(model=model) if template is not None else f"{fallback}: {exc}"
442 return ProviderError(message, provider=_PROVIDER_NAME, kind=kind)
445class LitellmSdkBackend:
446 """``LlmSdkBackend`` adapter backed by the ``litellm`` SDK."""
448 @property
449 def provider_name(self) -> str:
450 """Stable identifier used when wrapping errors in ``ProviderError``."""
451 return _PROVIDER_NAME
453 def active_backend_name(self, base_url: str) -> str:
454 """Return the display name of the backend ``base_url`` points at."""
455 return detect_backend_name(base_url)
457 def available(self) -> bool:
458 """Return True if the underlying SDK is installed."""
459 return litellm_available()
461 def supports_tools(self, _model_ref: str) -> bool:
462 """Optimistic: all SDK-routed refs report tool support.
464 A model that lacks a tool template just returns an empty
465 ``tool_calls`` array, which the dispatch handles as a normal
466 end-of-turn.
467 """
468 return True
470 def configure_logging(self, *, suppress_debug: bool) -> None:
471 """Apply litellm's debug-info suppression toggle when requested."""
472 if not suppress_debug:
473 return
474 try:
475 import litellm
477 litellm.suppress_debug_info = True
478 except ImportError:
479 pass # debug-suppression is best-effort when the litellm extra is absent
481 def complete(self, request: CompletionRequest) -> CompletionResult:
482 """Run a single-shot completion through ``litellm.completion``."""
483 litellm = _require_litellm()
484 kwargs = self._completion_kwargs(request, stream=False)
485 try:
486 response = litellm.completion(**kwargs)
487 except Exception as exc:
488 raise _provider_error(_CHAT_FAILED, exc, request.ref.for_display()) from exc
489 view = _LitellmResponseView(response)
490 return CompletionResult(
491 content=view.message_content,
492 finish_reason=view.finish_reason,
493 model=view.model,
494 tool_calls=view.tool_calls,
495 )
497 def complete_stream(self, request: CompletionRequest) -> Iterator[StreamChunk]:
498 """Stream a completion through ``litellm.completion(stream=True)``."""
499 litellm = _require_litellm()
500 kwargs = self._completion_kwargs(request, stream=True)
501 model = request.ref.for_display()
502 try:
503 response = litellm.completion(**kwargs)
504 except Exception as exc:
505 raise _provider_error(_CHAT_FAILED, exc, model) from exc
506 return self._stream_chunks(response, model)
508 @staticmethod
509 def _stream_chunks(response: Any, model: str) -> Iterator[StreamChunk]:
510 """Yield ``StreamChunk`` values from a litellm streaming response.
512 Exceptions raised mid-iteration are classified into ``ProviderError``
513 so the semantic layer sees a consistent error type regardless of
514 where the SDK failed.
515 """
516 try:
517 for chunk in response:
518 view = _LitellmResponseView(chunk)
519 content = view.delta_content
520 finish_reason = view.finish_reason
521 tool_call_deltas = view.delta_tool_calls
522 if content or finish_reason or tool_call_deltas:
523 yield StreamChunk(
524 content=content,
525 finish_reason=finish_reason,
526 tool_call_deltas=tool_call_deltas,
527 )
528 except ProviderError:
529 raise
530 except Exception as exc:
531 raise _provider_error(_CHAT_FAILED, exc, model) from exc
533 @staticmethod
534 def _completion_kwargs(request: CompletionRequest, *, stream: bool) -> dict[str, Any]:
535 """Translate a ``CompletionRequest`` into litellm kwargs."""
536 kwargs: dict[str, Any] = {
537 "model": _route_model(request.ref, request.api_base),
538 "messages": _format_messages(request.messages),
539 "stream": stream,
540 }
541 if request.api_base:
542 kwargs["api_base"] = request.api_base
543 if request.api_key:
544 kwargs["api_key"] = request.api_key
545 if request.options:
546 kwargs.update(request.options)
547 if "response_format" in kwargs:
548 # Best-effort: a provider without structured-output support should
549 # drop the field and answer normally, not refuse the call. Callers
550 # that send it parse the reply defensively either way.
551 kwargs["drop_params"] = True
552 return kwargs
554 def embed(self, request: EmbeddingRequest) -> EmbeddingResult:
555 """Embed inputs through ``litellm.embedding``."""
556 litellm = _require_litellm()
557 kwargs: dict[str, Any] = {
558 "model": _route_model(request.ref, request.api_base),
559 "input": request.inputs,
560 }
561 if request.api_base:
562 kwargs["api_base"] = request.api_base
563 if request.api_key:
564 kwargs["api_key"] = request.api_key
565 try:
566 response = litellm.embedding(**kwargs)
567 except Exception as exc:
568 raise _provider_error(_EMBED_FAILED, exc, request.ref.for_display()) from exc
569 data = response["data"] if isinstance(response, dict) else response.data
570 # Order by the response's ``index`` rather than arrival order: a proxy or
571 # gateway may return the batch out of order, and the consumer zips vectors
572 # to inputs positionally, so a reorder would silently mis-pair every chunk
573 # with the wrong vector. ``index`` is required (always present in a
574 # spec-conforming response), mirroring the rerank path's direct read.
575 ordered = sorted(data, key=_embedding_index)
576 # Reordering is not the only way the positional zip breaks. A gateway that
577 # drops an item, or repeats an index, yields a batch that still sorts
578 # cleanly but no longer corresponds one-to-one with the inputs, and the
579 # consumer would pair every later chunk with the wrong vector and store it.
580 # A spec-conforming response carries exactly one item per input, indexed
581 # 0..n-1, so anything else is refused rather than silently mis-paired.
582 expected = len(request.inputs)
583 if [_embedding_index(item) for item in ordered] != list(range(expected)):
584 raise ProviderError(
585 f"Embedding response does not match the request: expected {expected} "
586 f"vectors indexed 0-{expected - 1}, got {len(ordered)}. The endpoint "
587 "returned an incomplete or misindexed batch.",
588 provider=_PROVIDER_NAME,
589 kind=ProviderErrorKind.SERVER,
590 )
591 vectors = [_embedding_vector(item) for item in ordered]
592 return EmbeddingResult(vectors=vectors, model=_response_model(response))
594 def rerank(self, request: RerankRequest) -> RerankResult:
595 """Rerank documents via ``litellm.rerank`` (Cohere, Voyage, Jina, Together, HF TEI).
597 The SDK returns results sorted by relevance; we restore input
598 order via each result's ``index`` so scores line up with the
599 caller's ``candidates`` list.
600 """
601 if not request.candidates:
602 return RerankResult(scores=[])
603 litellm = _require_litellm()
604 kwargs: dict[str, Any] = {
605 "model": _route_model(request.ref, request.api_base),
606 "query": request.query,
607 "documents": request.candidates,
608 }
609 if request.api_base:
610 kwargs["api_base"] = request.api_base
611 if request.api_key:
612 kwargs["api_key"] = request.api_key
613 try:
614 response = litellm.rerank(**kwargs)
615 except Exception as exc:
616 raise _provider_error(_RERANK_FAILED, exc, request.ref.for_display()) from exc
617 results = response["results"] if isinstance(response, dict) else response.results
618 scores = [0.0] * len(request.candidates)
619 for item in results:
620 idx = item["index"] if isinstance(item, dict) else item.index
621 score = item["relevance_score"] if isinstance(item, dict) else item.relevance_score
622 scores[idx] = float(score)
623 return RerankResult(scores=scores, model=_response_model(response))
625 def list_models(self, *, base_url: str, api_key: str) -> list[str]:
626 """List models from Ollama (``/api/tags``) or an OpenAI-compatible ``/v1/models``."""
627 clean_base = base_url.rstrip("/")
628 spec = detect_local_server(clean_base)
629 if spec is OLLAMA:
630 return self._list_ollama_models(clean_base)
631 return self._list_openai_models(clean_base, api_key)
633 def list_chat_models(self, provider: str) -> list[str]:
634 """Return chat-mode model ids from litellm's static catalog.
636 Returns whatever litellm exposes for *provider*, alphabetically.
637 Empty list when litellm is not installed or the provider has no
638 chat-mode entries.
639 """
640 try:
641 import litellm
642 except ImportError:
643 return []
644 return self._all_chat_models_for(provider, litellm)
646 @staticmethod
647 def _all_chat_models_for(provider: str, litellm: Any) -> list[str]:
648 """Filter litellm's catalog down to chat-mode entries for ``provider``.
650 litellm's catalog stores some providers' models bare (``gpt-4o``)
651 and others prefixed (``mistral/codestral-latest``,
652 ``openrouter/anthropic/claude-3.5-sonnet``). Strip any leading
653 ``{provider}/`` so callers see uniformly bare names; the canonical
654 ``provider/name`` form is reapplied at the routing layer via
655 :meth:`ProviderModelRef.for_openai_prefix`.
656 """
657 models = litellm.models_by_provider.get(provider, set())
658 prefix = f"{provider}/"
659 bare: set[str] = set()
660 for model_name in models:
661 info = litellm.model_cost.get(model_name, {})
662 if info.get("mode") != "chat":
663 continue
664 bare.add(model_name.removeprefix(prefix))
665 return sorted(bare)
667 @staticmethod
668 def _list_ollama_models(base_url: str) -> list[str]:
669 """List models via the Ollama ``/api/tags`` endpoint."""
670 try:
671 resp = httpx.get(f"{base_url}/api/tags", timeout=DEFAULT_HTTP_TIMEOUT)
672 resp.raise_for_status()
673 data = resp.json()
674 return [m["name"] for m in data.get("models", [])]
675 except httpx.HTTPError as exc:
676 raise ProviderError(f"Cannot list models: {exc}", provider=_PROVIDER_NAME) from exc
678 @staticmethod
679 def _list_openai_models(base_url: str, api_key: str) -> list[str]:
680 """List models via an OpenAI-compatible ``/v1/models`` endpoint."""
681 headers: dict[str, str] = {}
682 if api_key:
683 headers["Authorization"] = f"Bearer {api_key}"
684 try:
685 resp = httpx.get(
686 openai_models_url(base_url), headers=headers, timeout=DEFAULT_HTTP_TIMEOUT
687 )
688 resp.raise_for_status()
689 data = resp.json()
690 return [m["id"] for m in data.get("data", [])]
691 except httpx.HTTPError:
692 log.debug("Failed to list models via /v1/models", exc_info=True)
693 return []
695 def pull_model(
696 self,
697 model: str,
698 *,
699 base_url: str,
700 on_progress: Callable[..., Any] | None = None,
701 ) -> None:
702 """Refuse to pull: local servers (Ollama, LM Studio) are read-only.
704 Their models are managed in their own app and surface here once
705 present, so lilbee never downloads them over the network.
706 """
707 spec = detect_local_server(base_url.rstrip("/"))
708 server = spec.display_name if spec is not None else "This server"
709 raise ProviderError(
710 f"{server} doesn't download models over the network. "
711 f"Add the model in its own app, then pick it here.",
712 provider=_PROVIDER_NAME,
713 )
715 def show_model(self, model: str, *, base_url: str) -> dict[str, Any] | None:
716 """Get model info via the Ollama ``/api/show`` endpoint.
718 Returns the raw ``parameters`` text and the ``capabilities`` list
719 (newer Ollama versions) so callers can check for vision support.
720 Returns ``None`` for servers without a metadata endpoint (LM Studio).
721 """
722 clean_base = base_url.rstrip("/")
723 spec = detect_local_server(clean_base)
724 if spec is None or not spec.supports_show:
725 return None
726 # Ollama's API uses bare model names; the routing-layer prefix has
727 # to come off before the request goes out.
728 ollama_name = model.removeprefix(OLLAMA.wire_prefix)
729 try:
730 resp = httpx.post(
731 f"{clean_base}/api/show",
732 json={"name": ollama_name},
733 timeout=DEFAULT_HTTP_TIMEOUT,
734 )
735 resp.raise_for_status()
736 data = resp.json()
737 except httpx.HTTPError:
738 return None
740 result: dict[str, Any] = {}
742 params = data.get("parameters", "")
743 if isinstance(params, str) and params:
744 result["parameters"] = params
745 elif params:
746 result["parameters"] = str(params)
748 capabilities = data.get("capabilities")
749 if isinstance(capabilities, list):
750 result["capabilities"] = capabilities
752 return result or None