Coverage for src/lilbee/providers/fleet/binary.py: 100%

1"""Resolve the bundled engine binaries (llama-server, llama-swap, gguf-parser).""" 

2 

3from __future__ import annotations 

4 

5import shutil 

6from enum import StrEnum 

7from importlib.metadata import version as _pkg_version 

8from pathlib import Path 

9 

10from lilbee.providers.base import ProviderError, ProviderErrorKind 

11 

12# Every index is spelled out rather than pointing at the README. This is read at 

13# the moment the engine is needed, often on a remote box. One index per hardware 

14# because the builds are not interchangeable: cpu is built with every GPU backend 

15# off, metal only ships a macOS arm64 wheel, and vulkan only Linux/Windows ones, 

16# so naming a single "default" hands somebody an engine that ignores their GPU. 

17_INSTALL_HINT = ( 

18 "The engine is lilbee's 'engine' extra, published on lilbee.sh rather than " 

19 "PyPI, so the index is part of the command:\n" 

20 " NVIDIA (CUDA): pip install --pre 'lilbee[engine]' --extra-index-url https://lilbee.sh/cu125/\n" 

21 " AMD (ROCm): pip install --pre 'lilbee[engine]' --extra-index-url https://lilbee.sh/rocm/\n" 

22 " Apple silicon: pip install --pre 'lilbee[engine]' --extra-index-url https://lilbee.sh/metal/\n" 

23 " Other GPUs: pip install --pre 'lilbee[engine]' --extra-index-url https://lilbee.sh/vulkan/\n" 

24 " No GPU: pip install --pre 'lilbee[engine]' --extra-index-url https://lilbee.sh/cpu/\n" 

25 "A standalone binary from https://github.com/tobocop2/lilbee/releases/latest " 

26 "bundles the engine instead. To bring your own, set LILBEE_LLAMA_SERVER_PATH to " 

27 "a llama-server binary and put llama-swap / gguf-parser on PATH." 

28) 

29 

30 

31class EngineTool(StrEnum): 

32 """A bundled engine executable resolved from the ``lilbee-engine`` wheel.""" 

33 

34 LLAMA_SERVER = "llama-server" 

35 LLAMA_SWAP = "llama-swap" 

36 GGUF_PARSER = "gguf-parser" 

37 

38 

39_BUNDLED_ACCESSORS = { 

40 EngineTool.LLAMA_SERVER: "get_llama_server_path", 

41 EngineTool.LLAMA_SWAP: "get_llama_swap_path", 

42 EngineTool.GGUF_PARSER: "get_gguf_parser_path", 

43} 

44 

45 

46def _bundled_tool(tool: EngineTool) -> Path | None: 

47 """Path to *tool* from the ``lilbee-engine`` wheel, or ``None`` if absent.""" 

48 try: 

49 import lilbee_engine 

50 except ImportError: 

51 return None 

52 path = Path(getattr(lilbee_engine, _BUNDLED_ACCESSORS[tool])()) 

53 return path if path.is_file() else None 

54 

55 

56def engine_pin() -> str: 

57 """Identity of the engine this lilbee would spawn; sharing keys on it. 

58 

59 Two dimensions must match for two processes to share one engine: the engine 

60 BUILD (a configured ``LILBEE_LLAMA_SERVER_PATH`` is its own identity so a 

61 bring-your-own engine never silently shares with a bundled one) and the 

62 load-affecting CONFIG baked into the launch argv (kv-cache type, expert 

63 offload, n-gpu-layers, ctx target, ...). A process whose load config differs 

64 computes a different pin, so ``contract_matches`` refuses the bind and it 

65 overflows to its own engine rather than silently running on the incumbent's 

66 flags. Total: never raises, because it runs on every state write. 

67 """ 

68 return f"{_engine_build_id()}|{_load_config_signature()}" 

69 

70 

71def _engine_build_id() -> str: 

72 """The engine build's identity: configured path, wheel pin, PATH, or unpinned. 

73 

74 A BYO (``custom:``) or PATH-resolved (``path:``) binary is identified by its 

75 location AND a cheap build fingerprint (size + mtime), so replacing the binary 

76 in place (a brew upgrade, a re-download) changes the pin and never binds a new 

77 process to an engine spawned from the old build. The bundled wheel needs no 

78 fingerprint: its pin already encodes the build. 

79 """ 

80 from lilbee.core.config import cfg 

81 

82 if cfg.llama_server_path: 

83 return f"custom:{cfg.llama_server_path}@{_binary_signature(Path(cfg.llama_server_path))}" 

84 try: 

85 import lilbee_engine 

86 except ImportError: 

87 lilbee_engine = None 

88 if lilbee_engine is not None: 

89 try: 

90 return str(lilbee_engine.get_engine_pin()) 

91 except AttributeError: # pre-pin wheels lack the accessor 

92 return f"wheel:{_engine_wheel_version()}" 

93 found = shutil.which(EngineTool.LLAMA_SERVER.value) 

94 if found is not None: 

95 return f"path:{found}@{_binary_signature(Path(found))}" 

96 return "unpinned" 

97 

98 

99def _engine_wheel_version() -> str: 

100 """The engine wheel's version, or a marker when it has no distribution metadata. 

101 

102 ``lilbee_engine`` can be importable with nothing to look up: an extracted 

103 wheel on sys.path, a vendored copy, or a distribution registered under a name 

104 that does not normalize to ``lilbee-engine``. Since this feeds the pin, and 

105 the pin is computed on every state write, a missing version degrades to a 

106 marker rather than raising out of ``engine_pin``. 

107 """ 

108 from importlib.metadata import PackageNotFoundError 

109 

110 try: 

111 return _pkg_version("lilbee-engine") 

112 except PackageNotFoundError: 

113 return "unknown" 

114 

115 

116def _binary_signature(path: Path) -> str: 

117 """A cheap build fingerprint of the binary at *path*: size and mtime. 

118 

119 An in-place replacement changes both, so the pin stops matching the old build. 

120 Best-effort and total (engine_pin runs on every state write): an unstatable 

121 path degrades to a fixed marker rather than raising. 

122 """ 

123 try: 

124 st = path.stat() 

125 except OSError: 

126 return "unstatable" 

127 return f"{st.st_size}-{st.st_mtime_ns}" 

128 

129 

130# Ctx sizing keys share by window coverage (contract.chat_ctx_covers), not 

131# value equality: a running window that covers the demand serves both peers. 

132# chat_n_ctx_target in particular defaults per process from its cgroup-capped 

133# RAM, so exact equality here restarted a warm engine per co-tenant. 

134_CTX_SIZING_KEYS = frozenset({"num_ctx", "num_ctx_max", "chat_n_ctx_target"}) 

135 

136 

137def _load_config_signature() -> str: 

138 """A deterministic digest of the settings an engine bakes in at launch. 

139 

140 These decide cross-process sharing, since an engine launched with one set 

141 cannot serve a peer that configured another: the ``LOAD_AFFECTING_KEYS`` a 

142 single process reloads on (minus the ctx sizing keys, matched by coverage 

143 instead), plus the placement keys that fix which devices a launch uses, so 

144 a peer with different placement binds its own engine. 

145 """ 

146 from lilbee.core.config import cfg 

147 from lilbee.core.config.keys import LOAD_AFFECTING_KEYS, PLACEMENT_PIN_KEYS 

148 

149 keys = (LOAD_AFFECTING_KEYS - _CTX_SIZING_KEYS) | PLACEMENT_PIN_KEYS 

150 return ";".join(f"{key}={getattr(cfg, key, None)}" for key in sorted(keys)) 

151 

152 

153def resolve_engine_tool(tool: EngineTool) -> Path: 

154 """Resolve *tool*: configured llama-server path, then bundled wheel, then PATH. 

155 

156 Never downloads anything; the binaries arrive via the bundled ``lilbee-engine`` 

157 wheel or bring-your-own. Only llama-server honors ``LILBEE_LLAMA_SERVER_PATH`` 

158 (an explicit setting beats the bundled wheel); the other tools resolve from the 

159 wheel, then ``PATH``. 

160 """ 

161 if tool is EngineTool.LLAMA_SERVER: 

162 from lilbee.core.config import cfg 

163 

164 if cfg.llama_server_path: 

165 configured = Path(cfg.llama_server_path) 

166 if not configured.is_file(): 

167 raise ProviderError(f"LILBEE_LLAMA_SERVER_PATH is not a file: {configured}") 

168 return configured 

169 

170 bundled = _bundled_tool(tool) 

171 if bundled is not None: 

172 return bundled 

173 

174 found = shutil.which(tool.value) 

175 if found is not None: 

176 return Path(found) 

177 

178 # Only llama-server carries NOT_FOUND: it marks the engine-less host that 

179 # legitimately serves nothing. A missing sibling tool (gguf-parser) must not 

180 # take that kind, or the sizing fallback would misreport it as a model that 

181 # isn't installed. 

182 kind = ( 

183 ProviderErrorKind.NOT_FOUND 

184 if tool is EngineTool.LLAMA_SERVER 

185 else ProviderErrorKind.UNKNOWN 

186 ) 

187 raise ProviderError(f"{tool.value} binary not found. {_INSTALL_HINT}", kind=kind) 

188 

189 

190def resolve_llama_server() -> Path: 

191 """Resolve the ``llama-server`` executable.""" 

192 return resolve_engine_tool(EngineTool.LLAMA_SERVER) 

193 

194 

195def resolve_llama_swap() -> Path: 

196 """Resolve the ``llama-swap`` executable.""" 

197 return resolve_engine_tool(EngineTool.LLAMA_SWAP) 

198 

199 

200def resolve_gguf_parser() -> Path: 

201 """Resolve the ``gguf-parser`` executable.""" 

202 return resolve_engine_tool(EngineTool.GGUF_PARSER) 

203 

204 

205def llama_server_runtime_env() -> dict[str, str]: 

206 """Extra environment for a spawned ``llama-server``. 

207 

208 The bundled wheel ships its own ggml/llama/mtmd next to the binary with a baked 

209 rpath (``@loader_path`` on macOS, ``$ORIGIN`` on Linux), but a CUDA build also 

210 links the CUDA 12 runtime, which driver-only GPU images omit. On Linux this 

211 adds any installed CUDA-runtime wheel libs to ``LD_LIBRARY_PATH``; elsewhere it 

212 is empty. 

213 """ 

214 from lilbee.providers.fleet.cuda_runtime import cuda_runtime_env 

215 

216 return cuda_runtime_env()