Coverage for src/lilbee/providers/fleet/gpu_select.py: 100%

1"""Ask the host's Vulkan loader what ggml is going to see. 

2 

3Probes the loader via ``ctypes`` for the facts the engine's ``--list-devices`` 

4text does not carry: each adapter's device type, its ``deviceUUID``, whether it 

5supports the one feature ggml requires of it, and how much of its memory is 

6actually free. Placement uses these to agree with the engine about which devices 

7exist and how big they are; where the two disagree, a fleet gets sized against 

8hardware llama-server never uses. 

9 

10It deliberately does not choose a device. Selection belongs to ggml, which 

11applies its own type filter, support check and same-UUID dedup at launch; 

12pinning through ``GGML_VK_VISIBLE_DEVICES`` would switch all three off, so 

13Vulkan devices are pinned by the name the engine printed or not at all. 

14""" 

15 

16from __future__ import annotations 

17 

18import ctypes 

19import ctypes.util 

20import fnmatch 

21import json 

22import logging 

23import ntpath 

24import os 

25import subprocess 

26import sys 

27from collections import Counter 

28from ctypes import POINTER, byref, c_char, c_char_p, c_uint8, c_uint32, c_uint64, c_void_p 

29from dataclasses import dataclass 

30from enum import IntEnum, StrEnum 

31from functools import lru_cache 

32 

33from lilbee.providers.fleet.gpu_hardware import installed_gpu_vendor_ids 

34from lilbee.providers.fleet.vulkan_icd_discovery import ( 

35 iter_vulkan_manifest_paths, 

36) 

37 

38log = logging.getLogger(__name__) 

39 

40# The child that runs the loader, and how long it may take. The bound matters: 

41# a wedged ICD can hang inside vkCreateInstance rather than fault, and the 

42# placement read that asked must not hang with it. 

43_PROBE_MODULE = "lilbee.providers.fleet.vulkan_probe" 

44_PROBE_TIMEOUT_S = 10.0 

45_PROBE_KILL_WAIT_S = 5.0 

46 

47# vk.h constants. Mirrored here so we don't drag a vulkan-headers 

48# dependency in for four magic numbers. See the upstream definitions in 

49# https://github.com/KhronosGroup/Vulkan-Headers/blob/main/include/vulkan/vulkan_core.h 

50# (VkStructureType enum and the VK_API_VERSION_1_0 / VK_SUCCESS macros). 

51_VK_STRUCTURE_TYPE_APPLICATION_INFO = 0 

52_VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO = 1 

53_VK_SUCCESS = 0 

54_VK_API_VERSION_1_0 = (1 << 22) | (0 << 12) | 0 

55# 1.1 is asked for first, purely to make vkGetPhysicalDeviceProperties2 (and the 

56# device UUID it carries) core rather than an extension; a loader that refuses 

57# it gets the 1.0 request back and the probe simply has no UUIDs to dedup by. 

58_VK_API_VERSION_1_1 = (1 << 22) | (1 << 12) | 0 

59_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2 = 1000059000 

60_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2 = 1000059001 

61_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ID_PROPERTIES = 1000071004 

62_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_16BIT_STORAGE_FEATURES = 1000083000 

63_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PROPERTIES_2 = 1000059006 

64_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT = 1000237000 

65# The device extension that turns heap sizes into a live budget. Without it the 

66# only figure available is the heap's capacity, which never moves. 

67_VK_EXT_MEMORY_BUDGET = b"VK_EXT_memory_budget" 

68_VK_MAX_EXTENSION_NAME_SIZE = 256 

69 

70 

71class VkDeviceType(IntEnum): 

72 """``VkPhysicalDeviceType`` enum from vulkan_core.h. 

73 

74 Values match the C ABI verbatim; the loader writes one of these 

75 into the ``deviceType`` field of ``VkPhysicalDeviceProperties``. 

76 """ 

77 

78 OTHER = 0 

79 INTEGRATED_GPU = 1 

80 DISCRETE_GPU = 2 

81 VIRTUAL_GPU = 3 

82 CPU = 4 

83 

84 

85# The device types ggml's Vulkan backend will actually run on. Anything else -- 

86# a software rasterizer, a paravirtual adapter, an unknown type -- is not a 

87# device the engine would choose, so planning against one guarantees a mismatch. 

88USABLE_VULKAN_TYPES = frozenset({VkDeviceType.DISCRETE_GPU, VkDeviceType.INTEGRATED_GPU}) 

89 

90 

91# vk.h sizes for the inline char arrays inside VkPhysicalDeviceProperties. 

92# Both constants are part of the Vulkan 1.0 ABI and frozen forever; see 

93# VK_MAX_PHYSICAL_DEVICE_NAME_SIZE and VK_UUID_SIZE in 

94# https://github.com/KhronosGroup/Vulkan-Headers/blob/main/include/vulkan/vulkan_core.h 

95_VK_MAX_PHYSICAL_DEVICE_NAME_SIZE = 256 

96_VK_UUID_SIZE = 16 

97 

98 

99@dataclass(frozen=True) 

100class VulkanDevice: 

101 """One Vulkan adapter as reported by the loader.""" 

102 

103 index: int 

104 device_type: int 

105 device_name: str 

106 vendor_id: int 

107 vram_bytes: int = 0 

108 # VkPhysicalDeviceIDProperties::deviceUUID, empty when the loader could not 

109 # be asked for it. The spec requires it to be immutable for a given device 

110 # across instances, processes, driver APIs, driver versions and reboots, so 

111 # two entries sharing one is one piece of silicon behind two drivers. 

112 device_uuid: bytes = b"" 

113 # storageBuffer16BitAccess, the single feature ggml's Vulkan backend requires 

114 # of a device before it will use it. Read from 

115 # VkPhysicalDevice16BitStorageFeatures rather than the 

116 # VkPhysicalDeviceVulkan11Features ggml itself uses: same bit, but the latter 

117 # arrived in Vulkan 1.2 and this probe asks for a 1.1 instance. 

118 # ``None`` when the loader could not be asked, which is not a refusal. 

119 storage_buffer_16bit: bool | None = None 

120 # Device-local memory not already committed, from VK_EXT_memory_budget. 

121 # ``None`` when the device does not expose that extension, which is the 

122 # difference between "nothing else is using this card" and "cannot tell". 

123 free_bytes: int | None = None 

124 

125 

126class PCIVendorID(IntEnum): 

127 """PCI-SIG vendor IDs for the GPU vendors that ship Vulkan ICDs. 

128 

129 Values are the canonical PCI vendor IDs that 

130 ``VkPhysicalDeviceProperties.vendorID`` surfaces. They are issued by 

131 PCI-SIG and frozen per company; see the public PCI vendor-ID 

132 registry at https://pcisig.com/membership/member-companies (also 

133 mirrored at https://devicehunt.com/all-pci-vendors). Only the 

134 vendors we have explicit ICD-disable globs for are enumerated; 

135 unknown vendors fall through the dispatch as no-op. 

136 """ 

137 

138 NVIDIA = 0x10DE # NVIDIA Corporation 

139 AMD = 0x1002 # Advanced Micro Devices, Inc. [AMD/ATI] 

140 INTEL = 0x8086 # Intel Corporation 

141 

142 

143# Vulkan loader manifest filename globs, per vendor. The loader matches these 

144# against the JSON manifest filename in its known-drivers list (see 

145# https://github.com/KhronosGroup/Vulkan-Loader/blob/main/docs/LoaderInterfaceArchitecture.md). 

146# Each vendor ships under multiple names across drivers/OSes; list every form 

147# we may encounter so disabling one vendor's drivers doesn't half-disable them. 

148_VENDOR_ICD_GLOBS: dict[PCIVendorID, tuple[str, ...]] = { 

149 # nv-vk*.json (Windows), nvidia_*.json (Linux). Both match nv*. 

150 PCIVendorID.NVIDIA: ("nv*",), 

151 # amdvlk64.json (Windows AMDVLK), amd_icd*.json (Linux AMDVLK), 

152 # amd-vulkan*.json (legacy AMDVLK builds), radeon_icd.*.json 

153 # (Mesa RADV on Linux). Adding amd_icd* explicitly because no 

154 # other glob covers the Linux AMDVLK manifest. 

155 PCIVendorID.AMD: ("amdvlk*", "amd_icd*", "amd-vulkan*", "radeon*"), 

156 # intel_icd.*.json (Mesa Intel ANV on Linux), igvk*.json (Windows). 

157 PCIVendorID.INTEL: ("intel*", "igvk*"), 

158} 

159 

160 

161class VulkanIcdEnvVar(StrEnum): 

162 """Every documented Vulkan loader env var that influences ICD selection. 

163 

164 Names are the verbatim loader env vars from the Khronos 

165 LoaderInterfaceArchitecture spec; the StrEnum lets each member be 

166 used directly as a ``str`` argument to ``os.environ.get`` / 

167 ``os.environ.setdefault`` without ``.value`` plumbing. Any value 

168 being non-empty in the environment is treated as a user override 

169 and suppresses the dual-vendor auto-pin. 

170 """ 

171 

172 DRIVER_FILES = "VK_DRIVER_FILES" 

173 ICD_FILENAMES = "VK_ICD_FILENAMES" 

174 ADD_DRIVER_FILES = "VK_ADD_DRIVER_FILES" 

175 LOADER_DRIVERS_DISABLE = "VK_LOADER_DRIVERS_DISABLE" 

176 LOADER_DRIVERS_SELECT = "VK_LOADER_DRIVERS_SELECT" 

177 

178 

179# Field layouts from the Vulkan 1.0 spec. ctypes maps the C structs 

180# verbatim so the loader populates them directly; only the prefix 

181# fields we read are commented (the trailing fields are kept for ABI 

182# alignment, not consumed). 

183 

184 

185class _VkApplicationInfo(ctypes.Structure): 

186 _fields_ = [ 

187 ("sType", c_uint32), 

188 ("pNext", c_void_p), 

189 ("pApplicationName", c_char_p), 

190 ("applicationVersion", c_uint32), 

191 ("pEngineName", c_char_p), 

192 ("engineVersion", c_uint32), 

193 ("apiVersion", c_uint32), 

194 ] 

195 

196 

197class _VkInstanceCreateInfo(ctypes.Structure): 

198 _fields_ = [ 

199 ("sType", c_uint32), 

200 ("pNext", c_void_p), 

201 ("flags", c_uint32), 

202 ("pApplicationInfo", POINTER(_VkApplicationInfo)), 

203 ("enabledLayerCount", c_uint32), 

204 ("ppEnabledLayerNames", POINTER(c_char_p)), 

205 ("enabledExtensionCount", c_uint32), 

206 ("ppEnabledExtensionNames", POINTER(c_char_p)), 

207 ] 

208 

209 

210class _VkPhysicalDeviceLimits(ctypes.Structure): 

211 # Opaque to us; we only need the parent struct's *layout* to match 

212 # the driver-populated bytes so the loader can write a vendorID and 

213 # deviceType into the prefix fields we actually read. 

214 # 

215 # 504 bytes, per VkPhysicalDeviceLimits in 

216 # https://github.com/KhronosGroup/Vulkan-Headers/blob/main/include/vulkan/vulkan_core.h 

217 # The size is part of the frozen Vulkan 1.0 layout, so it does not drift 

218 # across driver versions. 

219 # 

220 # Declared as uint64 rather than bytes for its ALIGNMENT, not its size. The 

221 # real struct mixes uint32, uint64 (VkDeviceSize), size_t and float, so its C 

222 # alignment is 8. A c_uint8 array aligns to 1, which let ctypes seat this 

223 # field at offset 292 in the parent instead of the 296 the ABI pads it to, 

224 # making the mirror 816 bytes against the driver's 824. The driver fills the 

225 # caller's buffer using its own layout, so every probe wrote sparseProperties 

226 # four bytes past the end of a Python-heap allocation -- absorbed by allocator 

227 # slack, which is what kept it silent. 

228 _fields_ = [("_opaque", c_uint64 * 63)] 

229 

230 

231class _VkPhysicalDeviceSparseProperties(ctypes.Structure): 

232 # 5 ULONG32 booleans, also part of the Vulkan 1.0 ABI; see same header. 

233 _fields_ = [("_opaque", c_uint32 * 5)] 

234 

235 

236class _VkPhysicalDeviceProperties(ctypes.Structure): 

237 _fields_ = [ 

238 ("apiVersion", c_uint32), 

239 ("driverVersion", c_uint32), 

240 ("vendorID", c_uint32), 

241 ("deviceID", c_uint32), 

242 ("deviceType", c_uint32), 

243 ("deviceName", c_char * _VK_MAX_PHYSICAL_DEVICE_NAME_SIZE), 

244 ("pipelineCacheUUID", c_uint8 * _VK_UUID_SIZE), 

245 ("limits", _VkPhysicalDeviceLimits), 

246 ("sparseProperties", _VkPhysicalDeviceSparseProperties), 

247 ] 

248 

249 

250# VkPhysicalDeviceMemoryProperties layout (Vulkan 1.0 ABI, frozen). Array 

251# bounds and the device-local heap flag are from vulkan_core.h. The 

252# device-local heap size is the cross-vendor VRAM signal (the same heap 

253# nvidia-smi/rocm-smi report) used for placement bin-packing. 

254_VK_MAX_MEMORY_TYPES = 32 

255_VK_MAX_MEMORY_HEAPS = 16 

256_VK_MEMORY_HEAP_DEVICE_LOCAL_BIT = 0x00000001 

257 

258 

259class _VkMemoryType(ctypes.Structure): 

260 _fields_ = [("propertyFlags", c_uint32), ("heapIndex", c_uint32)] 

261 

262 

263class _VkMemoryHeap(ctypes.Structure): 

264 _fields_ = [("size", c_uint64), ("flags", c_uint32)] 

265 

266 

267class _VkPhysicalDevice16BitStorageFeatures(ctypes.Structure): 

268 # Promoted to core in Vulkan 1.1 from VK_KHR_16bit_storage. Chained onto 

269 # VkPhysicalDeviceFeatures2; only the first flag is read. 

270 _fields_ = [ 

271 ("sType", c_uint32), 

272 ("pNext", c_void_p), 

273 ("storageBuffer16BitAccess", c_uint32), 

274 ("uniformAndStorageBuffer16BitAccess", c_uint32), 

275 ("storagePushConstant16", c_uint32), 

276 ("storageInputOutput16", c_uint32), 

277 ] 

278 

279 

280class _VkPhysicalDeviceFeatures2(ctypes.Structure): 

281 # VkPhysicalDeviceFeatures is a flat run of VkBool32s whose count grows with 

282 # no version of the spec but is easy to miscount, and the driver writes the 

283 # whole thing into this buffer. Declared larger than the real struct so a 

284 # miscount cannot become a heap overrun the way the limits mirror once did; 

285 # the field sits last, so the extra words shift nothing the driver reads. 

286 _fields_ = [ 

287 ("sType", c_uint32), 

288 ("pNext", c_void_p), 

289 ("features", c_uint32 * 128), 

290 ] 

291 

292 

293class _VkExtensionProperties(ctypes.Structure): 

294 _fields_ = [ 

295 ("extensionName", c_char * _VK_MAX_EXTENSION_NAME_SIZE), 

296 ("specVersion", c_uint32), 

297 ] 

298 

299 

300class _VkPhysicalDeviceIDProperties(ctypes.Structure): 

301 # VkPhysicalDeviceIDProperties, promoted to core in Vulkan 1.1. Chained onto 

302 # VkPhysicalDeviceProperties2 via pNext; the driver fills every field, so the 

303 # trailing ones are declared for layout even though only deviceUUID is read. 

304 _fields_ = [ 

305 ("sType", c_uint32), 

306 ("pNext", c_void_p), 

307 ("deviceUUID", c_uint8 * _VK_UUID_SIZE), 

308 ("driverUUID", c_uint8 * _VK_UUID_SIZE), 

309 ("deviceLUID", c_uint8 * 8), 

310 ("deviceNodeMask", c_uint32), 

311 ("deviceLUIDValid", c_uint32), 

312 ] 

313 

314 

315class _VkPhysicalDeviceProperties2(ctypes.Structure): 

316 _fields_ = [ 

317 ("sType", c_uint32), 

318 ("pNext", c_void_p), 

319 ("properties", _VkPhysicalDeviceProperties), 

320 ] 

321 

322 

323class _VkPhysicalDeviceMemoryProperties(ctypes.Structure): 

324 _fields_ = [ 

325 ("memoryTypeCount", c_uint32), 

326 ("memoryTypes", _VkMemoryType * _VK_MAX_MEMORY_TYPES), 

327 ("memoryHeapCount", c_uint32), 

328 ("memoryHeaps", _VkMemoryHeap * _VK_MAX_MEMORY_HEAPS), 

329 ] 

330 

331 

332class _VkPhysicalDeviceMemoryProperties2(ctypes.Structure): 

333 _fields_ = [ 

334 ("sType", c_uint32), 

335 ("pNext", c_void_p), 

336 ("memoryProperties", _VkPhysicalDeviceMemoryProperties), 

337 ] 

338 

339 

340class _VkPhysicalDeviceMemoryBudgetPropertiesEXT(ctypes.Structure): 

341 # heapBudget is what this process may still allocate from each heap and 

342 # heapUsage what it already has; the difference across the device-local heaps 

343 # is the only cross-vendor figure that moves when another process takes VRAM. 

344 _fields_ = [ 

345 ("sType", c_uint32), 

346 ("pNext", c_void_p), 

347 ("heapBudget", c_uint64 * _VK_MAX_MEMORY_HEAPS), 

348 ("heapUsage", c_uint64 * _VK_MAX_MEMORY_HEAPS), 

349 ] 

350 

351 

352def enumerate_gpu_vram() -> list[tuple[int, int, int]] | None: 

353 """Return ``[(device_index, device_local_vram_bytes, free_bytes), ...]`` or ``None``. 

354 

355 Cross-vendor via the Vulkan probe (NVIDIA/AMD/Intel). ``None`` when the 

356 loader/probe is unavailable (macOS Metal, no Vulkan driver), so the 

357 placement planner can degrade to count-only or in-process. 

358 

359 Only discrete and integrated adapters are returned, the same rule ggml's 

360 Vulkan backend applies when it picks a device, so this cannot offer 

361 placement something the engine would refuse to run on. Matching that rule 

362 is the point: where the two disagree about which devices exist, placement 

363 sizes against a device llama-server never uses. 

364 

365 Two kinds are excluded. Mesa's llvmpipe is a software rasterizer that 

366 advertises itself through Vulkan and reports system RAM as its device 

367 memory, so beside integrated graphics it appears at an identical size and 

368 is indistinguishable by VRAM alone; planning against it splits the model 

369 across a real GPU and a CPU renderer. Paravirtual adapters (virgl, VMware, 

370 VirtIO-GPU) report as ``VIRTUAL_GPU`` and are typically compute-incapable 

371 or proxies that fail on allocation. 

372 

373 The device type is the only signal separating any of these, and the caller 

374 has no access to it: this returns sizes, and the ``--list-devices`` text it 

375 feeds carries no names on the fallback path. 

376 """ 

377 devices = _enumerate_vulkan_devices() 

378 if devices is None: 

379 return None 

380 return [ 

381 (d.index, d.vram_bytes, d.free_bytes if d.free_bytes is not None else d.vram_bytes) 

382 for d in devices 

383 if d.device_type in USABLE_VULKAN_TYPES 

384 ] 

385 

386 

387@lru_cache(maxsize=1) 

388def integrated_vulkan_indices() -> frozenset[int]: 

389 """Loader indices of adapters whose memory is the host's. 

390 

391 Empty when the loader is unavailable or the probe fails, which reads as 

392 "assume dedicated" and preserves the behaviour discrete hosts already have. 

393 

394 Cached because the device parser asks per device line: without it an 

395 N-device host paid N loader loads and N instance creations to answer the 

396 same question. Which adapters are integrated is a property of the machine, 

397 so one answer per process is right. 

398 """ 

399 devices = _enumerate_vulkan_devices() 

400 if not devices: 

401 return frozenset() 

402 return frozenset(d.index for d in devices if d.device_type == VkDeviceType.INTEGRATED_GPU) 

403 

404 

405def vulkan_free_bytes_by_name() -> dict[str, int]: 

406 """Device-local memory still free, keyed by the name the loader reports. 

407 

408 Deliberately not cached: free memory is a live number, and freezing it for 

409 the process lifetime would hand every later probe the first reading taken. 

410 Callers sample it once per parse rather than per device line. 

411 

412 Only devices whose driver exposes ``VK_EXT_memory_budget`` appear; the rest 

413 have no live figure to offer and are absent rather than guessed at. A name 

414 two adapters share is also absent: two identical cards have their own free 

415 figures, and nothing in the engine's text says which line is which. Guessing 

416 would report one card's headroom for the other. 

417 """ 

418 devices = _enumerate_vulkan_devices() 

419 if not devices: 

420 return {} 

421 seen = Counter(d.device_name for d in devices) 

422 return { 

423 d.device_name: d.free_bytes 

424 for d in devices 

425 if d.free_bytes is not None and seen[d.device_name] == 1 

426 } 

427 

428 

429@lru_cache(maxsize=1) 

430def vulkan_device_types_by_name() -> dict[str, VkDeviceType]: 

431 """Adapter type keyed by the name the loader reports, empty when unavailable. 

432 

433 Keyed by name rather than index because the engine's ``--list-devices`` 

434 ordinals are assigned after ggml has filtered and deduplicated the loader's 

435 list, so ``Vulkan0`` is only the loader's device 0 when nothing ahead of it 

436 was dropped. The name is the one field both views print verbatim from 

437 ``VkPhysicalDeviceProperties``, so it correlates the two without either 

438 side having to replicate the other's filtering. 

439 

440 Two adapters of the same model share a name, which is harmless: they share 

441 a type too, and the type is all this answers. 

442 """ 

443 devices = _enumerate_vulkan_devices() 

444 if not devices: 

445 return {} 

446 return { 

447 d.device_name: device_type 

448 for d in devices 

449 if (device_type := _known_device_type(d.device_type)) is not None 

450 } 

451 

452 

453def discrete_gpu_from_vendor(vendor_id: int) -> bool | None: 

454 """Whether the loader reports a discrete adapter from *vendor_id*. 

455 

456 ``None`` when the loader cannot be reached, which is a different answer from 

457 "no": a caller deciding whether to fail loud must not read silence as proof 

458 that a card is absent, nor as proof that one is present. 

459 """ 

460 devices = _enumerate_vulkan_devices() 

461 if not devices: 

462 return None 

463 return any( 

464 d.vendor_id == vendor_id and d.device_type == VkDeviceType.DISCRETE_GPU for d in devices 

465 ) 

466 

467 

468# Vendors whose PCI display controllers are always dedicated cards. AMD and 

469# Intel both ship integrated parts under their own IDs, so their presence says 

470# nothing about whether a discrete card exists; NVIDIA's desktop and laptop 

471# parts are discrete without exception here. 

472_DISCRETE_ONLY_VENDORS: frozenset[int] = frozenset({PCIVendorID.NVIDIA}) 

473 

474 

475def host_has_no_discrete_gpu() -> bool: 

476 """Whether the Vulkan loader can see adapters and none of them is discrete. 

477 

478 The vendor-neutral answer to a question CUDA and ROCm cannot be asked 

479 through text: their ``--list-devices`` lines carry no device type, so an AMD 

480 APU and a Jetson enumerate exactly like a discrete card while reporting 

481 system RAM as their memory. Every such part also ships a Vulkan driver, and 

482 a machine whose loader reports adapters but no discrete one has no discrete 

483 GPU for CUDA or ROCm to be enumerating. 

484 

485 The verdict rests on an integrated adapter actually being there. Software 

486 rasterizers report through the loader on any host with mesa installed, even 

487 with no vendor ICD present at all, which is ordinary on headless CUDA boxes 

488 and in containers; concluding from a list that holds only those would mark a 

489 real discrete card as sharing the host's memory and shrink its budget. 

490 

491 False when the loader is unreachable, when any discrete adapter exists, or 

492 when nothing but rasterizers answered, so a host with a real card is never 

493 talked into the shared-memory budget. A host holding both a discrete card 

494 and an APU also answers False, which leaves the APU sized as dedicated; 

495 correlating individual devices across two backends' naming needs more than 

496 the type. 

497 

498 The loader is not the only witness, and on a hybrid laptop it is the wrong 

499 one. Optimus and its equivalents leave the discrete card powered down until 

500 something asks for it through prime-run, so the loader enumerates the 

501 integrated adapter alone while a dedicated card sits on the PCI bus. Taking 

502 that list at its word marked a real 4 GB card as sharing system memory, 

503 which is the exact outcome the paragraph above promises never to reach. PCI 

504 settles it: a vendor that only ever ships discrete parts, present in the 

505 device tree but absent from the loader's list, means the loader is telling 

506 an incomplete story rather than a complete one. 

507 """ 

508 types = set(vulkan_device_types_by_name().values()) 

509 if VkDeviceType.DISCRETE_GPU in types: 

510 return False 

511 if _DISCRETE_ONLY_VENDORS & installed_gpu_vendor_ids(): 

512 return False 

513 return VkDeviceType.INTEGRATED_GPU in types 

514 

515 

516def _known_device_type(value: int) -> VkDeviceType | None: 

517 """The enum member for a raw ``deviceType``, ``None`` for a value vk.h doesn't define.""" 

518 try: 

519 return VkDeviceType(value) 

520 except ValueError: 

521 return None 

522 

523 

524def enumerate_in_process() -> list[VulkanDevice] | None: 

525 """Open libvulkan, create a throwaway instance, enumerate adapters. 

526 

527 Returns ``None`` if the loader can't be found or any Vulkan call 

528 fails; empty list ("loader present, no adapters") is a distinct 

529 outcome and propagates back. 

530 

531 Runs the loader in whatever process calls it, which is why 

532 :func:`_enumerate_vulkan_devices` calls it in a child rather than directly: 

533 ``vkCreateInstance`` loads every vendor ICD on the host, and a faulting one 

534 raises no exception, it raises a signal. 

535 """ 

536 lib = _load_vulkan_loader() 

537 if lib is None: 

538 return None 

539 try: 

540 devices = _list_devices_with_instance(lib) 

541 return _deduplicate_by_uuid(_drop_devices_the_engine_refuses(devices)) 

542 except OSError: 

543 # ctypes argument / call-site errors land here; treat as 

544 # "probe failed" rather than crashing the host process. 

545 return None 

546 

547 

548def _run_probe_child() -> tuple[str, int, str]: 

549 """Run the enumeration in a child; returns ``(stdout, returncode, stderr)``.""" 

550 from lilbee.providers.fleet.gpu_env import engine_environ 

551 from lilbee.providers.fleet.proc import run_bounded 

552 

553 argv = [sys.executable, "-m", _PROBE_MODULE] 

554 stdout, returncode = run_bounded( 

555 argv, 

556 env=engine_environ(), 

557 timeout_s=_PROBE_TIMEOUT_S, 

558 kill_wait_s=_PROBE_KILL_WAIT_S, 

559 label="vulkan-probe", 

560 ) 

561 return stdout, returncode, "" 

562 

563 

564def _enumerate_vulkan_devices() -> list[VulkanDevice] | None: 

565 """The adapters the Vulkan loader reports, or ``None`` for no opinion. 

566 

567 Asked of a short-lived child. ``vkCreateInstance`` pre-loads every vendor ICD 

568 on the host, and a broken or conflicting one faults inside the loader; a 

569 fault is a signal, so no ``except`` here could keep the daemon alive. In a 

570 child, dying is simply an answer. Every failure reads the same as an 

571 unreachable loader, which is the state the callers were written for. 

572 

573 An empty list still means "loader present, no adapters", which is a 

574 different fact and propagates back intact. 

575 

576 Uncached, and each caller decides for itself whether to hold the answer: the 

577 device types are a property of the machine and are cached, while free memory 

578 is a live number that is read fresh every time it is asked for. 

579 """ 

580 from lilbee.providers.base import ProviderError 

581 from lilbee.providers.fleet.vulkan_probe import from_json 

582 

583 try: 

584 stdout, returncode, stderr = _run_probe_child() 

585 except (ProviderError, OSError, subprocess.SubprocessError) as exc: 

586 log.debug("Vulkan probe child could not be run: %s", exc) 

587 return None 

588 if returncode != 0: 

589 log.debug("Vulkan probe child exited %s: %s", returncode, stderr.strip() or stdout.strip()) 

590 return None 

591 try: 

592 return from_json(json.loads(stdout)) 

593 except (ValueError, KeyError, TypeError) as exc: 

594 log.debug("Vulkan probe child printed no usable device list: %s", exc) 

595 return None 

596 

597 

598def _drop_devices_the_engine_refuses(devices: list[VulkanDevice]) -> list[VulkanDevice]: 

599 """Drop adapters ggml's Vulkan backend would exclude from its device pool. 

600 

601 ``ggml_vk_device_is_supported`` gates on exactly one feature, 

602 ``storageBuffer16BitAccess``, and excludes devices without it silently, with 

603 no error anywhere. Some Adreno parts are the documented case. Keeping such a 

604 device means placement sizes a fleet against VRAM the engine will never 

605 touch, and the engine quietly runs on the CPU or another adapter instead. 

606 

607 Only a definite ``False`` drops a device: a loader too old to be asked 

608 reports ``None``, and that is not a refusal. 

609 """ 

610 return [d for d in devices if d.storage_buffer_16bit is not False] 

611 

612 

613def _deduplicate_by_uuid(devices: list[VulkanDevice]) -> list[VulkanDevice]: 

614 """Collapse adapters that share a ``deviceUUID`` into one, keeping the first. 

615 

616 Two ICDs able to drive the same card (RADV beside AMDVLK is the case ggml's 

617 own dedup names) enumerate it twice. ggml counts it once, so without this 

618 lilbee plans a two-GPU fleet on one piece of silicon and tensor-splits a 

619 model across a card and itself. 

620 

621 ggml breaks the same tie with a driver-priority table, picking which 

622 driver's entry survives. Lowest index is enough here because nothing lilbee 

623 reads off a device tells the two entries apart: the type, the name and the 

624 device-local heap size describe the silicon, not the driver, and no caller 

625 pins by the raw enumeration index any more. 

626 

627 Devices with no UUID are all kept, since "the loader would not say" is not 

628 evidence that two adapters are one. 

629 """ 

630 seen: set[bytes] = set() 

631 unique: list[VulkanDevice] = [] 

632 for device in devices: 

633 if device.device_uuid and device.device_uuid in seen: 

634 log.debug( 

635 "Vulkan device %d (%s) is device %s under a second driver; ignoring the duplicate", 

636 device.index, 

637 device.device_name, 

638 next(d.index for d in unique if d.device_uuid == device.device_uuid), 

639 ) 

640 continue 

641 if device.device_uuid: 

642 seen.add(device.device_uuid) 

643 unique.append(device) 

644 return unique 

645 

646 

647def _load_vulkan_loader() -> ctypes.CDLL | None: 

648 """Locate and load the Vulkan loader for the current platform. 

649 

650 Returns ``None`` when the loader isn't installed, which is the 

651 expected outcome on stock macOS (we ship a Metal wheel there) and 

652 on hosts without a Vulkan-capable driver. 

653 """ 

654 candidates: tuple[str, ...] 

655 if sys.platform == "win32": 

656 candidates = ("vulkan-1.dll",) 

657 elif sys.platform == "darwin": 

658 # MoltenVK exposes a different ABI than libvulkan; lilbee's 

659 # macOS wheel uses Metal directly, so skipping the probe on 

660 # Darwin is correct. 

661 return None 

662 else: 

663 candidates = ("libvulkan.so.1", "libvulkan.so") 

664 

665 for name in candidates: 

666 try: 

667 return ctypes.CDLL(name) 

668 except OSError: 

669 continue 

670 # ctypes.util.find_library is a last-resort fallback for distros 

671 # where the soname isn't directly loadable. 

672 resolved = ctypes.util.find_library("vulkan") 

673 if resolved is not None: 

674 try: 

675 return ctypes.CDLL(resolved) 

676 except OSError: 

677 return None 

678 return None 

679 

680 

681def _create_probe_instance(create_instance: ctypes._FuncPointer) -> tuple[c_void_p | None, int]: 

682 """Create the throwaway instance, asking for 1.1 and settling for 1.0. 

683 

684 Returns the instance and the API version it was created with. 1.1 makes 

685 ``vkGetPhysicalDeviceProperties2`` core, which is where the device UUID 

686 lives; a 1.0-only loader rejects the request outright, so the 1.0 retry is 

687 what keeps the probe working there at all rather than silently reporting no 

688 adapters. 

689 """ 

690 for api_version in (_VK_API_VERSION_1_1, _VK_API_VERSION_1_0): 

691 app_info = _VkApplicationInfo( 

692 sType=_VK_STRUCTURE_TYPE_APPLICATION_INFO, 

693 pNext=None, 

694 pApplicationName=b"lilbee-gpu-probe", 

695 applicationVersion=0, 

696 pEngineName=b"lilbee", 

697 engineVersion=0, 

698 apiVersion=api_version, 

699 ) 

700 create_info = _VkInstanceCreateInfo( 

701 sType=_VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO, 

702 pNext=None, 

703 flags=0, 

704 pApplicationInfo=ctypes.pointer(app_info), 

705 enabledLayerCount=0, 

706 ppEnabledLayerNames=None, 

707 enabledExtensionCount=0, 

708 ppEnabledExtensionNames=None, 

709 ) 

710 instance = c_void_p() 

711 result = create_instance(byref(create_info), None, byref(instance)) 

712 if result == _VK_SUCCESS and instance.value: 

713 return instance, api_version 

714 return None, 0 

715 

716 

717def _resolve_properties2(lib: ctypes.CDLL) -> ctypes._FuncPointer | None: 

718 """``vkGetPhysicalDeviceProperties2`` with argtypes stamped, ``None`` if absent.""" 

719 try: 

720 get_properties2 = lib.vkGetPhysicalDeviceProperties2 

721 except AttributeError: 

722 return None 

723 get_properties2.argtypes = [c_void_p, POINTER(_VkPhysicalDeviceProperties2)] 

724 get_properties2.restype = None 

725 return get_properties2 

726 

727 

728def _resolve_memory_budget( 

729 lib: ctypes.CDLL, 

730) -> tuple[ctypes._FuncPointer, ctypes._FuncPointer] | None: 

731 """``(vkGetPhysicalDeviceMemoryProperties2, vkEnumerateDeviceExtensionProperties)``.""" 

732 try: 

733 get_memory2 = lib.vkGetPhysicalDeviceMemoryProperties2 

734 enum_extensions = lib.vkEnumerateDeviceExtensionProperties 

735 except AttributeError: 

736 return None 

737 get_memory2.argtypes = [c_void_p, POINTER(_VkPhysicalDeviceMemoryProperties2)] 

738 get_memory2.restype = None 

739 enum_extensions.argtypes = [ 

740 c_void_p, 

741 c_char_p, 

742 POINTER(c_uint32), 

743 POINTER(_VkExtensionProperties), 

744 ] 

745 enum_extensions.restype = c_uint32 

746 return get_memory2, enum_extensions 

747 

748 

749def _supports_memory_budget(handle: c_void_p, enum_extensions: ctypes._FuncPointer) -> bool: 

750 """Whether the device advertises ``VK_EXT_memory_budget``. 

751 

752 Asked rather than assumed: chaining the budget struct onto a device that 

753 does not support it leaves it zeroed, and zero budget is indistinguishable 

754 from a full card. 

755 """ 

756 count = c_uint32(0) 

757 if enum_extensions(handle, None, byref(count), None) != _VK_SUCCESS or count.value == 0: 

758 return False 

759 props = (_VkExtensionProperties * count.value)() 

760 if enum_extensions(handle, None, byref(count), props) != _VK_SUCCESS: 

761 return False 

762 return any(props[i].extensionName == _VK_EXT_MEMORY_BUDGET for i in range(count.value)) 

763 

764 

765def _free_device_local_bytes( 

766 handle: c_void_p, memory_budget: tuple[ctypes._FuncPointer, ctypes._FuncPointer] | None 

767) -> int | None: 

768 """Device-local memory still available, or ``None`` when it cannot be asked. 

769 

770 ``None`` rather than the heap size on purpose. Reporting capacity as free is 

771 how a desktop holding gigabytes of compositor and browser VRAM was planned 

772 as an empty card. 

773 """ 

774 if memory_budget is None: 

775 return None 

776 get_memory2, enum_extensions = memory_budget 

777 if not _supports_memory_budget(handle, enum_extensions): 

778 return None 

779 budget = _VkPhysicalDeviceMemoryBudgetPropertiesEXT( 

780 sType=_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT, pNext=None 

781 ) 

782 props2 = _VkPhysicalDeviceMemoryProperties2( 

783 sType=_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PROPERTIES_2, 

784 pNext=ctypes.cast(ctypes.pointer(budget), c_void_p), 

785 ) 

786 get_memory2(handle, byref(props2)) 

787 mem = props2.memoryProperties 

788 free = 0 

789 for i in range(mem.memoryHeapCount): 

790 if mem.memoryHeaps[i].flags & _VK_MEMORY_HEAP_DEVICE_LOCAL_BIT: 

791 free += max(0, int(budget.heapBudget[i]) - int(budget.heapUsage[i])) 

792 return free 

793 

794 

795def _resolve_features2(lib: ctypes.CDLL) -> ctypes._FuncPointer | None: 

796 """``vkGetPhysicalDeviceFeatures2`` with argtypes stamped, ``None`` if absent.""" 

797 try: 

798 get_features2 = lib.vkGetPhysicalDeviceFeatures2 

799 except AttributeError: 

800 return None 

801 get_features2.argtypes = [c_void_p, POINTER(_VkPhysicalDeviceFeatures2)] 

802 get_features2.restype = None 

803 return get_features2 

804 

805 

806def _storage_buffer_16bit( 

807 handle: c_void_p, get_features2: ctypes._FuncPointer | None 

808) -> bool | None: 

809 """Whether the adapter supports ``storageBuffer16BitAccess``, ``None`` if unasked. 

810 

811 The one feature ggml's Vulkan backend requires before it will put a device 

812 in its pool, and it drops devices that lack it silently. Some Adreno parts 

813 expose ``uniformAndStorageBuffer16BitAccess`` without it, so the two are not 

814 interchangeable and only the first flag answers the question. 

815 """ 

816 if get_features2 is None: 

817 return None 

818 storage = _VkPhysicalDevice16BitStorageFeatures( 

819 sType=_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_16BIT_STORAGE_FEATURES, pNext=None 

820 ) 

821 features2 = _VkPhysicalDeviceFeatures2( 

822 sType=_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, 

823 pNext=ctypes.cast(ctypes.pointer(storage), c_void_p), 

824 ) 

825 get_features2(handle, byref(features2)) 

826 return bool(storage.storageBuffer16BitAccess) 

827 

828 

829def _device_uuid(handle: c_void_p, get_properties2: ctypes._FuncPointer | None) -> bytes: 

830 """The adapter's ``deviceUUID``, empty when it cannot be asked for. 

831 

832 An all-zero UUID is returned as empty too: it is what a driver leaves behind 

833 when it ignores the chained struct, and treating it as a real identity would 

834 collapse every such adapter into one. 

835 """ 

836 if get_properties2 is None: 

837 return b"" 

838 id_props = _VkPhysicalDeviceIDProperties( 

839 sType=_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ID_PROPERTIES, pNext=None 

840 ) 

841 props2 = _VkPhysicalDeviceProperties2( 

842 sType=_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, 

843 pNext=ctypes.cast(ctypes.pointer(id_props), c_void_p), 

844 ) 

845 get_properties2(handle, byref(props2)) 

846 uuid = bytes(id_props.deviceUUID) 

847 return b"" if not any(uuid) else uuid 

848 

849 

850def _list_devices_with_instance(lib: ctypes.CDLL) -> list[VulkanDevice]: 

851 """Create a temporary VkInstance, enumerate physical devices, destroy. 

852 

853 Mirrors what ``vulkaninfo --summary`` does internally. The 

854 instance is short-lived (created and destroyed in the same call) 

855 so the probe leaves no driver state behind. 

856 """ 

857 ( 

858 create_instance, 

859 destroy_instance, 

860 enum_physical, 

861 get_properties, 

862 get_memory, 

863 ) = _resolve_vk_symbols(lib) 

864 

865 instance, api_version = _create_probe_instance(create_instance) 

866 if instance is None: 

867 return [] 

868 core_1_1 = api_version >= _VK_API_VERSION_1_1 

869 get_properties2 = _resolve_properties2(lib) if core_1_1 else None 

870 get_features2 = _resolve_features2(lib) if core_1_1 else None 

871 memory_budget = _resolve_memory_budget(lib) if core_1_1 else None 

872 

873 try: 

874 count = c_uint32(0) 

875 result = enum_physical(instance, byref(count), None) 

876 if result != _VK_SUCCESS or count.value == 0: 

877 return [] 

878 handles = (c_void_p * count.value)() 

879 result = enum_physical(instance, byref(count), handles) 

880 if result != _VK_SUCCESS: 

881 return [] 

882 devices: list[VulkanDevice] = [] 

883 for i in range(count.value): 

884 # Indexing the array yields a bare address; the queries below take a 

885 # handle, so make it one rather than widen every signature to int. 

886 handle = c_void_p(handles[i]) 

887 props = _VkPhysicalDeviceProperties() 

888 get_properties(handle, byref(props)) 

889 mem = _VkPhysicalDeviceMemoryProperties() 

890 get_memory(handle, byref(mem)) 

891 devices.append( 

892 VulkanDevice( 

893 index=i, 

894 device_type=int(props.deviceType), 

895 device_name=props.deviceName.decode("utf-8", errors="replace"), 

896 vendor_id=int(props.vendorID), 

897 vram_bytes=_device_local_vram(mem), 

898 device_uuid=_device_uuid(handle, get_properties2), 

899 storage_buffer_16bit=_storage_buffer_16bit(handle, get_features2), 

900 free_bytes=_free_device_local_bytes(handle, memory_budget), 

901 ) 

902 ) 

903 return devices 

904 finally: 

905 destroy_instance(instance, None) 

906 

907 

908def _resolve_vk_symbols( 

909 lib: ctypes.CDLL, 

910) -> tuple[ 

911 ctypes._FuncPointer, 

912 ctypes._FuncPointer, 

913 ctypes._FuncPointer, 

914 ctypes._FuncPointer, 

915 ctypes._FuncPointer, 

916]: 

917 """Look up the five Vulkan symbols this probe needs and stamp argtypes. 

918 

919 All argtypes / restypes are set here so ctypes uses the same 

920 calling convention as the C ABI; missing this on Windows produces 

921 silent stack corruption. 

922 """ 

923 create_instance = lib.vkCreateInstance 

924 create_instance.argtypes = [ 

925 POINTER(_VkInstanceCreateInfo), 

926 c_void_p, 

927 POINTER(c_void_p), 

928 ] 

929 create_instance.restype = c_uint32 

930 

931 destroy_instance = lib.vkDestroyInstance 

932 destroy_instance.argtypes = [c_void_p, c_void_p] 

933 destroy_instance.restype = None 

934 

935 enum_physical = lib.vkEnumeratePhysicalDevices 

936 enum_physical.argtypes = [c_void_p, POINTER(c_uint32), POINTER(c_void_p)] 

937 enum_physical.restype = c_uint32 

938 

939 get_properties = lib.vkGetPhysicalDeviceProperties 

940 get_properties.argtypes = [c_void_p, POINTER(_VkPhysicalDeviceProperties)] 

941 get_properties.restype = None 

942 

943 get_memory = lib.vkGetPhysicalDeviceMemoryProperties 

944 get_memory.argtypes = [c_void_p, POINTER(_VkPhysicalDeviceMemoryProperties)] 

945 get_memory.restype = None 

946