Coverage for src/lilbee/providers/fleet/gpu_select.py: 100%

1"""Ask the host's Vulkan loader what ggml is going to see. 

2 

3Probes the loader via ``ctypes`` for the facts the engine's ``--list-devices`` 

4text does not carry: each adapter's device type, its ``deviceUUID``, whether it 

5supports the one feature ggml requires of it, and how much of its memory is 

6actually free. Placement uses these to agree with the engine about which devices 

7exist and how big they are; where the two disagree, a fleet gets sized against 

8hardware llama-server never uses. 

9 

10It deliberately does not choose a device. Selection belongs to ggml, which 

11applies its own type filter, support check and same-UUID dedup at launch; 

12pinning through ``GGML_VK_VISIBLE_DEVICES`` would switch all three off, so 

13Vulkan devices are pinned by the name the engine printed or not at all. 

14""" 

15 

16from __future__ import annotations 

17 

18import ctypes 

19import ctypes.util 

20import fnmatch 

21import json 

22import logging 

23import ntpath 

24import os 

25import subprocess 

26import sys 

27from collections import Counter 

28from ctypes import POINTER, byref, c_char, c_char_p, c_uint8, c_uint32, c_uint64, c_void_p 

29from dataclasses import dataclass 

30from enum import IntEnum, StrEnum 

31from functools import lru_cache 

32 

33from lilbee.providers.fleet.gpu_hardware import installed_gpu_vendor_ids 

34from lilbee.providers.fleet.vulkan_icd_discovery import ( 

35 iter_vulkan_manifest_paths, 

36) 

37 

38log = logging.getLogger(__name__) 

39 

40# The child that runs the loader, and how long it may take. The bound matters: 

41# a wedged ICD can hang inside vkCreateInstance rather than fault, and the 

42# placement read that asked must not hang with it. 

43_PROBE_MODULE = "lilbee.providers.fleet.vulkan_probe" 

44_PROBE_TIMEOUT_S = 10.0 

45_PROBE_KILL_WAIT_S = 5.0 

46 

47# vk.h constants. Mirrored here so we don't drag a vulkan-headers 

48# dependency in for four magic numbers. See the upstream definitions in 

49# https://github.com/KhronosGroup/Vulkan-Headers/blob/main/include/vulkan/vulkan_core.h 

50# (VkStructureType enum and the VK_API_VERSION_1_0 / VK_SUCCESS macros). 

51_VK_STRUCTURE_TYPE_APPLICATION_INFO = 0 

52_VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO = 1 

53_VK_SUCCESS = 0 

54_VK_API_VERSION_1_0 = (1 << 22) | (0 << 12) | 0 

55# 1.1 is asked for first, purely to make vkGetPhysicalDeviceProperties2 (and the 

56# device UUID it carries) core rather than an extension; a loader that refuses 

57# it gets the 1.0 request back and the probe simply has no UUIDs to dedup by. 

58_VK_API_VERSION_1_1 = (1 << 22) | (1 << 12) | 0 

59_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2 = 1000059000 

60_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2 = 1000059001 

61_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ID_PROPERTIES = 1000071004 

62_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_16BIT_STORAGE_FEATURES = 1000083000 

63_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PROPERTIES_2 = 1000059006 

64_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT = 1000237000 

65# The device extension that turns heap sizes into a live budget. Without it the 

66# only figure available is the heap's capacity, which never moves. 

67_VK_EXT_MEMORY_BUDGET = b"VK_EXT_memory_budget" 

68_VK_MAX_EXTENSION_NAME_SIZE = 256 

69 

70 

71class VkDeviceType(IntEnum): 

72 """``VkPhysicalDeviceType`` enum from vulkan_core.h. 

73 

74 Values match the C ABI verbatim; the loader writes one of these 

75 into the ``deviceType`` field of ``VkPhysicalDeviceProperties``. 

76 """ 

77 

78 OTHER = 0 

79 INTEGRATED_GPU = 1 

80 DISCRETE_GPU = 2 

81 VIRTUAL_GPU = 3 

82 CPU = 4 

83 

84 

85# The device types ggml's Vulkan backend will actually run on. Anything else -- 

86# a software rasterizer, a paravirtual adapter, an unknown type -- is not a 

87# device the engine would choose, so planning against one guarantees a mismatch. 

88USABLE_VULKAN_TYPES = frozenset({VkDeviceType.DISCRETE_GPU, VkDeviceType.INTEGRATED_GPU}) 

89 

90 

91# vk.h sizes for the inline char arrays inside VkPhysicalDeviceProperties. 

92# Both constants are part of the Vulkan 1.0 ABI and frozen forever; see 

93# VK_MAX_PHYSICAL_DEVICE_NAME_SIZE and VK_UUID_SIZE in 

94# https://github.com/KhronosGroup/Vulkan-Headers/blob/main/include/vulkan/vulkan_core.h 

95_VK_MAX_PHYSICAL_DEVICE_NAME_SIZE = 256 

96_VK_UUID_SIZE = 16 

97 

98 

99@dataclass(frozen=True) 

100class VulkanDevice: 

101 """One Vulkan adapter as reported by the loader.""" 

102 

103 index: int 

104 device_type: int 

105 device_name: str 

106 vendor_id: int 

107 vram_bytes: int = 0 

108 # VkPhysicalDeviceIDProperties::deviceUUID, empty when the loader could not 

109 # be asked for it. The spec requires it to be immutable for a given device 

110 # across instances, processes, driver APIs, driver versions and reboots, so 

111 # two entries sharing one is one piece of silicon behind two drivers. 

112 device_uuid: bytes = b"" 

113 # storageBuffer16BitAccess, the single feature ggml's Vulkan backend requires 

114 # of a device before it will use it. Read from 

115 # VkPhysicalDevice16BitStorageFeatures rather than the 

116 # VkPhysicalDeviceVulkan11Features ggml itself uses: same bit, but the latter 

117 # arrived in Vulkan 1.2 and this probe asks for a 1.1 instance. 

118 # ``None`` when the loader could not be asked, which is not a refusal. 

119 storage_buffer_16bit: bool | None = None 

120 # Device-local memory not already committed, from VK_EXT_memory_budget. 

121 # ``None`` when the device does not expose that extension, which is the 

122 # difference between "nothing else is using this card" and "cannot tell". 

123 free_bytes: int | None = None 

124 

125 

126class PCIVendorID(IntEnum): 

127 """PCI-SIG vendor IDs for the GPU vendors that ship Vulkan ICDs. 

128 

129 Values are the canonical PCI vendor IDs that 

130 ``VkPhysicalDeviceProperties.vendorID`` surfaces. They are issued by 

131 PCI-SIG and frozen per company; see the public PCI vendor-ID 

132 registry at https://pcisig.com/membership/member-companies (also 

133 mirrored at https://devicehunt.com/all-pci-vendors). Only the 

134 vendors we have explicit ICD-disable globs for are enumerated; 

135 unknown vendors fall through the dispatch as no-op. 

136 """ 

137 

138 NVIDIA = 0x10DE # NVIDIA Corporation 

139 AMD = 0x1002 # Advanced Micro Devices, Inc. [AMD/ATI] 

140 INTEL = 0x8086 # Intel Corporation 

141 

142 

143# Vulkan loader manifest filename globs, per vendor. The loader matches these 

144# against the JSON manifest filename in its known-drivers list (see 

145# https://github.com/KhronosGroup/Vulkan-Loader/blob/main/docs/LoaderInterfaceArchitecture.md). 

146# Each vendor ships under multiple names across drivers/OSes; list every form 

147# we may encounter so disabling one vendor's drivers doesn't half-disable them. 

148_VENDOR_ICD_GLOBS: dict[PCIVendorID, tuple[str, ...]] = { 

149 # nv-vk*.json (Windows), nvidia_*.json (Linux). Both match nv*. 

150 PCIVendorID.NVIDIA: ("nv*",), 

151 # amdvlk64.json (Windows AMDVLK), amd_icd*.json (Linux AMDVLK), 

152 # amd-vulkan*.json (legacy AMDVLK builds), radeon_icd.*.json 

153 # (Mesa RADV on Linux). Adding amd_icd* explicitly because no 

154 # other glob covers the Linux AMDVLK manifest. 

155 PCIVendorID.AMD: ("amdvlk*", "amd_icd*", "amd-vulkan*", "radeon*"), 

156 # intel_icd.*.json (Mesa Intel ANV on Linux), igvk*.json (Windows). 

157 PCIVendorID.INTEL: ("intel*", "igvk*"), 

158} 

159 

160 

161class VulkanIcdEnvVar(StrEnum): 

162 """Every documented Vulkan loader env var that influences ICD selection. 

163 

164 Names are the verbatim loader env vars from the Khronos 

165 LoaderInterfaceArchitecture spec; the StrEnum lets each member be 

166 used directly as a ``str`` argument to ``os.environ.get`` / 

167 ``os.environ.setdefault`` without ``.value`` plumbing. Any value 

168 being non-empty in the environment is treated as a user override 

169 and suppresses the dual-vendor auto-pin. 

170 """ 

171 

172 DRIVER_FILES = "VK_DRIVER_FILES" 

173 ICD_FILENAMES = "VK_ICD_FILENAMES" 

174 ADD_DRIVER_FILES = "VK_ADD_DRIVER_FILES" 

175 LOADER_DRIVERS_DISABLE = "VK_LOADER_DRIVERS_DISABLE" 

176 LOADER_DRIVERS_SELECT = "VK_LOADER_DRIVERS_SELECT" 

177 

178 

179# Field layouts from the Vulkan 1.0 spec. ctypes maps the C structs 

180# verbatim so the loader populates them directly; only the prefix 

181# fields we read are commented (the trailing fields are kept for ABI 

182# alignment, not consumed). 

183 

184 

185class _VkApplicationInfo(ctypes.Structure): 

186 _fields_ = [ 

187 ("sType", c_uint32), 

188 ("pNext", c_void_p), 

189 ("pApplicationName", c_char_p), 

190 ("applicationVersion", c_uint32), 

191 ("pEngineName", c_char_p), 

192 ("engineVersion", c_uint32), 

193 ("apiVersion", c_uint32), 

194 ] 

195 

196 

197class _VkInstanceCreateInfo(ctypes.Structure): 

198 _fields_ = [ 

199 ("sType", c_uint32), 

200 ("pNext", c_void_p), 

201 ("flags", c_uint32), 

202 ("pApplicationInfo", POINTER(_VkApplicationInfo)), 

203 ("enabledLayerCount", c_uint32), 

204 ("ppEnabledLayerNames", POINTER(c_char_p)), 

205 ("enabledExtensionCount", c_uint32), 

206 ("ppEnabledExtensionNames", POINTER(c_char_p)), 

207 ] 

208 

209 

210class _VkPhysicalDeviceLimits(ctypes.Structure): 

211 # Opaque to us; we only need the parent struct's *layout* to match 

212 # the driver-populated bytes so the loader can write a vendorID and 

213 # deviceType into the prefix fields we actually read. 

214 # 

215 # 504 bytes, per VkPhysicalDeviceLimits in 

216 # https://github.com/KhronosGroup/Vulkan-Headers/blob/main/include/vulkan/vulkan_core.h 

217 # The size is part of the frozen Vulkan 1.0 layout, so it does not drift 

218 # across driver versions. 

219 # 

220 # Declared as uint64 rather than bytes for its ALIGNMENT, not its size. The 

221 # real struct mixes uint32, uint64 (VkDeviceSize), size_t and float, so its C 

222 # alignment is 8. A c_uint8 array aligns to 1, which let ctypes seat this 

223 # field at offset 292 in the parent instead of the 296 the ABI pads it to, 

224 # making the mirror 816 bytes against the driver's 824. The driver fills the 

225 # caller's buffer using its own layout, so every probe wrote sparseProperties 

226 # four bytes past the end of a Python-heap allocation -- absorbed by allocator 

227 # slack, which is what kept it silent. 

228 _fields_ = [("_opaque", c_uint64 * 63)] 

229 

230 

231class _VkPhysicalDeviceSparseProperties(ctypes.Structure): 

232 # 5 ULONG32 booleans, also part of the Vulkan 1.0 ABI; see same header. 

233 _fields_ = [("_opaque", c_uint32 * 5)] 

234 

235 

236class _VkPhysicalDeviceProperties(ctypes.Structure): 

237 _fields_ = [ 

238 ("apiVersion", c_uint32), 

239 ("driverVersion", c_uint32), 

240 ("vendorID", c_uint32), 

241 ("deviceID", c_uint32), 

242 ("deviceType", c_uint32), 

243 ("deviceName", c_char * _VK_MAX_PHYSICAL_DEVICE_NAME_SIZE), 

244 ("pipelineCacheUUID", c_uint8 * _VK_UUID_SIZE), 

245 ("limits", _VkPhysicalDeviceLimits), 

246 ("sparseProperties", _VkPhysicalDeviceSparseProperties), 

247 ] 

248 

249 

250# VkPhysicalDeviceMemoryProperties layout (Vulkan 1.0 ABI, frozen). Array 

251# bounds and the device-local heap flag are from vulkan_core.h. The 

252# device-local heap size is the cross-vendor VRAM signal (the same heap 

253# nvidia-smi/rocm-smi report) used for placement bin-packing. 

254_VK_MAX_MEMORY_TYPES = 32 

255_VK_MAX_MEMORY_HEAPS = 16 

256_VK_MEMORY_HEAP_DEVICE_LOCAL_BIT = 0x00000001 

257 

258 

259class _VkMemoryType(ctypes.Structure): 

260 _fields_ = [("propertyFlags", c_uint32), ("heapIndex", c_uint32)] 

261 

262 

263class _VkMemoryHeap(ctypes.Structure): 

264 _fields_ = [("size", c_uint64), ("flags", c_uint32)] 

265 

266 

267class _VkPhysicalDevice16BitStorageFeatures(ctypes.Structure): 

268 # Promoted to core in Vulkan 1.1 from VK_KHR_16bit_storage. Chained onto 

269 # VkPhysicalDeviceFeatures2; only the first flag is read. 

270 _fields_ = [ 

271 ("sType", c_uint32), 

272 ("pNext", c_void_p), 

273 ("storageBuffer16BitAccess", c_uint32), 

274 ("uniformAndStorageBuffer16BitAccess", c_uint32), 

275 ("storagePushConstant16", c_uint32), 

276 ("storageInputOutput16", c_uint32), 

277 ] 

278 

279 

280class _VkPhysicalDeviceFeatures2(ctypes.Structure): 

281 # VkPhysicalDeviceFeatures is a flat run of VkBool32s whose count grows with 

282 # no version of the spec but is easy to miscount, and the driver writes the 

283 # whole thing into this buffer. Declared larger than the real struct so a 

284 # miscount cannot become a heap overrun the way the limits mirror once did; 

285 # the field sits last, so the extra words shift nothing the driver reads. 

286 _fields_ = [ 

287 ("sType", c_uint32), 

288 ("pNext", c_void_p), 

289 ("features", c_uint32 * 128), 

290 ] 

291 

292 

293class _VkExtensionProperties(ctypes.Structure): 

294 _fields_ = [ 

295 ("extensionName", c_char * _VK_MAX_EXTENSION_NAME_SIZE), 

296 ("specVersion", c_uint32), 

297 ] 

298 

299 

300class _VkPhysicalDeviceIDProperties(ctypes.Structure): 

301 # VkPhysicalDeviceIDProperties, promoted to core in Vulkan 1.1. Chained onto 

302 # VkPhysicalDeviceProperties2 via pNext; the driver fills every field, so the 

303 # trailing ones are declared for layout even though only deviceUUID is read. 

304 _fields_ = [ 

305 ("sType", c_uint32), 

306 ("pNext", c_void_p), 

307 ("deviceUUID", c_uint8 * _VK_UUID_SIZE), 

308 ("driverUUID", c_uint8 * _VK_UUID_SIZE), 

309 ("deviceLUID", c_uint8 * 8), 

310 ("deviceNodeMask", c_uint32), 

311 ("deviceLUIDValid", c_uint32), 

312 ] 

313 

314 

315class _VkPhysicalDeviceProperties2(ctypes.Structure): 

316 _fields_ = [ 

317 ("sType", c_uint32), 

318 ("pNext", c_void_p), 

319 ("properties", _VkPhysicalDeviceProperties), 

320 ] 

321 

322 

323class _VkPhysicalDeviceMemoryProperties(ctypes.Structure): 

324 _fields_ = [ 

325 ("memoryTypeCount", c_uint32), 

326 ("memoryTypes", _VkMemoryType * _VK_MAX_MEMORY_TYPES), 

327 ("memoryHeapCount", c_uint32), 

328 ("memoryHeaps", _VkMemoryHeap * _VK_MAX_MEMORY_HEAPS), 

329 ] 

330 

331 

332class _VkPhysicalDeviceMemoryProperties2(ctypes.Structure): 

333 _fields_ = [ 

334 ("sType", c_uint32), 

335 ("pNext", c_void_p), 

336 ("memoryProperties", _VkPhysicalDeviceMemoryProperties), 

337 ] 

338 

339 

340class _VkPhysicalDeviceMemoryBudgetPropertiesEXT(ctypes.Structure): 

341 # heapBudget is what this process may still allocate from each heap and 

342 # heapUsage what it already has; the difference across the device-local heaps 

343 # is the only cross-vendor figure that moves when another process takes VRAM. 

344 _fields_ = [ 

345 ("sType", c_uint32), 

346 ("pNext", c_void_p), 

347 ("heapBudget", c_uint64 * _VK_MAX_MEMORY_HEAPS), 

348 ("heapUsage", c_uint64 * _VK_MAX_MEMORY_HEAPS), 

349 ] 

350 

351 

352def enumerate_gpu_vram() -> list[tuple[int, int, int]] | None: 

353 """Return ``[(device_index, device_local_vram_bytes, free_bytes), ...]`` or ``None``. 

354 

355 Cross-vendor via the Vulkan probe (NVIDIA/AMD/Intel). ``None`` when the 

356 loader/probe is unavailable (macOS Metal, no Vulkan driver), so the 

357 placement planner can degrade to count-only or in-process. 

358 

359 Only discrete and integrated adapters are returned, the same rule ggml's 

360 Vulkan backend applies when it picks a device, so this cannot offer 

361 placement something the engine would refuse to run on. Matching that rule 

362 is the point: where the two disagree about which devices exist, placement 

363 sizes against a device llama-server never uses. 

364 

365 Two kinds are excluded. Mesa's llvmpipe is a software rasterizer that 

366 advertises itself through Vulkan and reports system RAM as its device 

367 memory, so beside integrated graphics it appears at an identical size and 

368 is indistinguishable by VRAM alone; planning against it splits the model 

369 across a real GPU and a CPU renderer. Paravirtual adapters (virgl, VMware, 

370 VirtIO-GPU) report as ``VIRTUAL_GPU`` and are typically compute-incapable 

371 or proxies that fail on allocation. 

372 

373 The device type is the only signal separating any of these, and the caller 

374 has no access to it: this returns sizes, and the ``--list-devices`` text it 

375 feeds carries no names on the fallback path. 

376 """ 

377 devices = _enumerate_vulkan_devices() 

378 if devices is None: 

379 return None 

380 return [ 

381 (d.index, d.vram_bytes, d.free_bytes if d.free_bytes is not None else d.vram_bytes) 

382 for d in devices 

383 if d.device_type in USABLE_VULKAN_TYPES 

384 ] 

385 

386 

387@lru_cache(maxsize=1) 

388def integrated_vulkan_indices() -> frozenset[int]: 

389 """Loader indices of adapters whose memory is the host's. 

390 

391 Empty when the loader is unavailable or the probe fails, which reads as 

392 "assume dedicated" and preserves the behaviour discrete hosts already have. 

393 

394 Cached because the device parser asks per device line: without it an 

395 N-device host paid N loader loads and N instance creations to answer the 

396 same question. Which adapters are integrated is a property of the machine, 

397 so one answer per process is right. 

398 """ 

399 devices = _enumerate_vulkan_devices() 

400 if not devices: 

401 return frozenset() 

402 return frozenset(d.index for d in devices if d.device_type == VkDeviceType.INTEGRATED_GPU) 

403 

404 

405def vulkan_free_bytes_by_name() -> dict[str, int]: 

406 """Device-local memory still free, keyed by the name the loader reports. 

407 

408 Deliberately not cached: free memory is a live number, and freezing it for 

409 the process lifetime would hand every later probe the first reading taken. 

410 Callers sample it once per parse rather than per device line. 

411 

412 Only devices whose driver exposes ``VK_EXT_memory_budget`` appear; the rest 

413 have no live figure to offer and are absent rather than guessed at. A name 

414 two adapters share is also absent: two identical cards have their own free 

415 figures, and nothing in the engine's text says which line is which. Guessing 

416 would report one card's headroom for the other. 

417 """ 

418 devices = _enumerate_vulkan_devices() 

419 if not devices: 

420 return {} 

421 seen = Counter(d.device_name for d in devices) 

422 return { 

423 d.device_name: d.free_bytes 

424 for d in devices 

425 if d.free_bytes is not None and seen[d.device_name] == 1 

426 } 

427 

428 

429@lru_cache(maxsize=1) 

430def vulkan_device_types_by_name() -> dict[str, VkDeviceType]: 

431 """Adapter type keyed by the name the loader reports, empty when unavailable. 

432 

433 Keyed by name rather than index because the engine's ``--list-devices`` 

434 ordinals are assigned after ggml has filtered and deduplicated the loader's 

435 list, so ``Vulkan0`` is only the loader's device 0 when nothing ahead of it 

436 was dropped. The name is the one field both views print verbatim from 

437 ``VkPhysicalDeviceProperties``, so it correlates the two without either 

438 side having to replicate the other's filtering. 

439 

440 Two adapters of the same model share a name, which is harmless: they share 

441 a type too, and the type is all this answers. 

442 """ 

443 devices = _enumerate_vulkan_devices() 

444 if not devices: 

445 return {} 

446 return { 

447 d.device_name: device_type 

448 for d in devices 

449 if (device_type := _known_device_type(d.device_type)) is not None 

450 } 

451 

452 

453def discrete_gpu_from_vendor(vendor_id: int) -> bool | None: 

454 """Whether the loader reports a discrete adapter from *vendor_id*. 

455 

456 ``None`` when the loader cannot be reached, which is a different answer from 

457 "no": a caller deciding whether to fail loud must not read silence as proof 

458 that a card is absent, nor as proof that one is present. 

459 """ 

460 devices = _enumerate_vulkan_devices() 

461 if not devices: 

462 return None 

463 return any( 

464 d.vendor_id == vendor_id and d.device_type == VkDeviceType.DISCRETE_GPU for d in devices 

465 ) 

466 

467 

468# Vendors whose PCI display controllers are always dedicated cards. AMD and 

469# Intel both ship integrated parts under their own IDs, so their presence says 

470# nothing about whether a discrete card exists; NVIDIA's desktop and laptop 

471# parts are discrete without exception here. 

472_DISCRETE_ONLY_VENDORS: frozenset[int] = frozenset({PCIVendorID.NVIDIA}) 

473 

474 

475def host_has_no_discrete_gpu() -> bool: 

476 """Whether the Vulkan loader can see adapters and none of them is discrete. 

477 

478 The vendor-neutral answer to a question CUDA and ROCm cannot be asked 

479 through text: their ``--list-devices`` lines carry no device type, so an AMD 

480 APU and a Jetson enumerate exactly like a discrete card while reporting 

481 system RAM as their memory. Every such part also ships a Vulkan driver, and 

482 a machine whose loader reports adapters but no discrete one has no discrete 

483 GPU for CUDA or ROCm to be enumerating. 

484 

485 The verdict rests on an integrated adapter actually being there. Software 

486 rasterizers report through the loader on any host with mesa installed, even 

487 with no vendor ICD present at all, which is ordinary on headless CUDA boxes 

488 and in containers; concluding from a list that holds only those would mark a 

489 real discrete card as sharing the host's memory and shrink its budget. 

490 

491 False when the loader is unreachable, when any discrete adapter exists, or 

492 when nothing but rasterizers answered, so a host with a real card is never 

493 talked into the shared-memory budget. A host holding both a discrete card 

494 and an APU also answers False, which leaves the APU sized as dedicated; 

495 correlating individual devices across two backends' naming needs more than 

496 the type. 

497 

498 The loader is not the only witness, and on a hybrid laptop it is the wrong 

499 one. Optimus and its equivalents leave the discrete card powered down until 

500 something asks for it through prime-run, so the loader enumerates the 

501 integrated adapter alone while a dedicated card sits on the PCI bus. Taking 

502 that list at its word marked a real 4 GB card as sharing system memory, 

503 which is the exact outcome the paragraph above promises never to reach. PCI 

504 settles it: a vendor that only ever ships discrete parts, present in the 

505 device tree but absent from the loader's list, means the loader is telling 

506 an incomplete story rather than a complete one. 

507 """ 

508 types = set(vulkan_device_types_by_name().values()) 

509 if VkDeviceType.DISCRETE_GPU in types: 

510 return False 

511 if _DISCRETE_ONLY_VENDORS & installed_gpu_vendor_ids(): 

512 return False 

513 return VkDeviceType.INTEGRATED_GPU in types 

514 

515 

516def _known_device_type(value: int) -> VkDeviceType | None: 

517 """The enum member for a raw ``deviceType``, ``None`` for a value vk.h doesn't define.""" 

518 try: 

519 return VkDeviceType(value) 

520 except ValueError: 

521 return None 

522 

523 

524def enumerate_in_process() -> list[VulkanDevice] | None: 

525 """Open libvulkan, create a throwaway instance, enumerate adapters. 

526 

527 Returns ``None`` if the loader can't be found or any Vulkan call 

528 fails; empty list ("loader present, no adapters") is a distinct 

529 outcome and propagates back. 

530 

531 Runs the loader in whatever process calls it, which is why 

532 :func:`_enumerate_vulkan_devices` calls it in a child rather than directly: 

533 ``vkCreateInstance`` loads every vendor ICD on the host, and a faulting one 

534 raises no exception, it raises a signal. 

535 """ 

536 lib = _load_vulkan_loader() 

537 if lib is None: 

538 return None 

539 try: 

540 devices = _list_devices_with_instance(lib) 

541 return _deduplicate_by_uuid(_drop_devices_the_engine_refuses(devices)) 

542 except OSError: 

543 # ctypes argument / call-site errors land here; treat as 

544 # "probe failed" rather than crashing the host process. 

545 return None 

546 

547 

548def _run_probe_child() -> tuple[str, int, str]: 

549 """Run the enumeration in a child; returns ``(stdout, returncode, stderr)``.""" 

550 from lilbee.providers.fleet.proc import run_bounded 

551 

552 argv = [sys.executable, "-m", _PROBE_MODULE] 

553 stdout, returncode = run_bounded( 

554 argv, 

555 timeout_s=_PROBE_TIMEOUT_S, 

556 kill_wait_s=_PROBE_KILL_WAIT_S, 

557 label="vulkan-probe", 

558 ) 

559 return stdout, returncode, "" 

560 

561 

562def _enumerate_vulkan_devices() -> list[VulkanDevice] | None: 

563 """The adapters the Vulkan loader reports, or ``None`` for no opinion. 

564 

565 Asked of a short-lived child. ``vkCreateInstance`` pre-loads every vendor ICD 

566 on the host, and a broken or conflicting one faults inside the loader; a 

567 fault is a signal, so no ``except`` here could keep the daemon alive. In a 

568 child, dying is simply an answer. Every failure reads the same as an 

569 unreachable loader, which is the state the callers were written for. 

570 

571 An empty list still means "loader present, no adapters", which is a 

572 different fact and propagates back intact. 

573 

574 Uncached, and each caller decides for itself whether to hold the answer: the 

575 device types are a property of the machine and are cached, while free memory 

576 is a live number that is read fresh every time it is asked for. 

577 """ 

578 from lilbee.providers.base import ProviderError 

579 from lilbee.providers.fleet.vulkan_probe import from_json 

580 

581 try: 

582 stdout, returncode, stderr = _run_probe_child() 

583 except (ProviderError, OSError, subprocess.SubprocessError) as exc: 

584 log.debug("Vulkan probe child could not be run: %s", exc) 

585 return None 

586 if returncode != 0: 

587 log.debug("Vulkan probe child exited %s: %s", returncode, stderr.strip() or stdout.strip()) 

588 return None 

589 try: 

590 return from_json(json.loads(stdout)) 

591 except (ValueError, KeyError, TypeError) as exc: 

592 log.debug("Vulkan probe child printed no usable device list: %s", exc) 

593 return None 

594 

595 

596def _drop_devices_the_engine_refuses(devices: list[VulkanDevice]) -> list[VulkanDevice]: 

597 """Drop adapters ggml's Vulkan backend would exclude from its device pool. 

598 

599 ``ggml_vk_device_is_supported`` gates on exactly one feature, 

600 ``storageBuffer16BitAccess``, and excludes devices without it silently, with 

601 no error anywhere. Some Adreno parts are the documented case. Keeping such a 

602 device means placement sizes a fleet against VRAM the engine will never 

603 touch, and the engine quietly runs on the CPU or another adapter instead. 

604 

605 Only a definite ``False`` drops a device: a loader too old to be asked 

606 reports ``None``, and that is not a refusal. 

607 """ 

608 return [d for d in devices if d.storage_buffer_16bit is not False] 

609 

610 

611def _deduplicate_by_uuid(devices: list[VulkanDevice]) -> list[VulkanDevice]: 

612 """Collapse adapters that share a ``deviceUUID`` into one, keeping the first. 

613 

614 Two ICDs able to drive the same card (RADV beside AMDVLK is the case ggml's 

615 own dedup names) enumerate it twice. ggml counts it once, so without this 

616 lilbee plans a two-GPU fleet on one piece of silicon and tensor-splits a 

617 model across a card and itself. 

618 

619 ggml breaks the same tie with a driver-priority table, picking which 

620 driver's entry survives. Lowest index is enough here because nothing lilbee 

621 reads off a device tells the two entries apart: the type, the name and the 

622 device-local heap size describe the silicon, not the driver, and no caller 

623 pins by the raw enumeration index any more. 

624 

625 Devices with no UUID are all kept, since "the loader would not say" is not 

626 evidence that two adapters are one. 

627 """ 

628 seen: set[bytes] = set() 

629 unique: list[VulkanDevice] = [] 

630 for device in devices: 

631 if device.device_uuid and device.device_uuid in seen: 

632 log.debug( 

633 "Vulkan device %d (%s) is device %s under a second driver; ignoring the duplicate", 

634 device.index, 

635 device.device_name, 

636 next(d.index for d in unique if d.device_uuid == device.device_uuid), 

637 ) 

638 continue 

639 if device.device_uuid: 

640 seen.add(device.device_uuid) 

641 unique.append(device) 

642 return unique 

643 

644 

645def _load_vulkan_loader() -> ctypes.CDLL | None: 

646 """Locate and load the Vulkan loader for the current platform. 

647 

648 Returns ``None`` when the loader isn't installed, which is the 

649 expected outcome on stock macOS (we ship a Metal wheel there) and 

650 on hosts without a Vulkan-capable driver. 

651 """ 

652 candidates: tuple[str, ...] 

653 if sys.platform == "win32": 

654 candidates = ("vulkan-1.dll",) 

655 elif sys.platform == "darwin": 

656 # MoltenVK exposes a different ABI than libvulkan; lilbee's 

657 # macOS wheel uses Metal directly, so skipping the probe on 

658 # Darwin is correct. 

659 return None 

660 else: 

661 candidates = ("libvulkan.so.1", "libvulkan.so") 

662 

663 for name in candidates: 

664 try: 

665 return ctypes.CDLL(name) 

666 except OSError: 

667 continue 

668 # ctypes.util.find_library is a last-resort fallback for distros 

669 # where the soname isn't directly loadable. 

670 resolved = ctypes.util.find_library("vulkan") 

671 if resolved is not None: 

672 try: 

673 return ctypes.CDLL(resolved) 

674 except OSError: 

675 return None 

676 return None 

677 

678 

679def _create_probe_instance(create_instance: ctypes._FuncPointer) -> tuple[c_void_p | None, int]: 

680 """Create the throwaway instance, asking for 1.1 and settling for 1.0. 

681 

682 Returns the instance and the API version it was created with. 1.1 makes 

683 ``vkGetPhysicalDeviceProperties2`` core, which is where the device UUID 

684 lives; a 1.0-only loader rejects the request outright, so the 1.0 retry is 

685 what keeps the probe working there at all rather than silently reporting no 

686 adapters. 

687 """ 

688 for api_version in (_VK_API_VERSION_1_1, _VK_API_VERSION_1_0): 

689 app_info = _VkApplicationInfo( 

690 sType=_VK_STRUCTURE_TYPE_APPLICATION_INFO, 

691 pNext=None, 

692 pApplicationName=b"lilbee-gpu-probe", 

693 applicationVersion=0, 

694 pEngineName=b"lilbee", 

695 engineVersion=0, 

696 apiVersion=api_version, 

697 ) 

698 create_info = _VkInstanceCreateInfo( 

699 sType=_VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO, 

700 pNext=None, 

701 flags=0, 

702 pApplicationInfo=ctypes.pointer(app_info), 

703 enabledLayerCount=0, 

704 ppEnabledLayerNames=None, 

705 enabledExtensionCount=0, 

706 ppEnabledExtensionNames=None, 

707 ) 

708 instance = c_void_p() 

709 result = create_instance(byref(create_info), None, byref(instance)) 

710 if result == _VK_SUCCESS and instance.value: 

711 return instance, api_version 

712 return None, 0 

713 

714 

715def _resolve_properties2(lib: ctypes.CDLL) -> ctypes._FuncPointer | None: 

716 """``vkGetPhysicalDeviceProperties2`` with argtypes stamped, ``None`` if absent.""" 

717 try: 

718 get_properties2 = lib.vkGetPhysicalDeviceProperties2 

719 except AttributeError: 

720 return None 

721 get_properties2.argtypes = [c_void_p, POINTER(_VkPhysicalDeviceProperties2)] 

722 get_properties2.restype = None 

723 return get_properties2 

724 

725 

726def _resolve_memory_budget( 

727 lib: ctypes.CDLL, 

728) -> tuple[ctypes._FuncPointer, ctypes._FuncPointer] | None: 

729 """``(vkGetPhysicalDeviceMemoryProperties2, vkEnumerateDeviceExtensionProperties)``.""" 

730 try: 

731 get_memory2 = lib.vkGetPhysicalDeviceMemoryProperties2 

732 enum_extensions = lib.vkEnumerateDeviceExtensionProperties 

733 except AttributeError: 

734 return None 

735 get_memory2.argtypes = [c_void_p, POINTER(_VkPhysicalDeviceMemoryProperties2)] 

736 get_memory2.restype = None 

737 enum_extensions.argtypes = [ 

738 c_void_p, 

739 c_char_p, 

740 POINTER(c_uint32), 

741 POINTER(_VkExtensionProperties), 

742 ] 

743 enum_extensions.restype = c_uint32 

744 return get_memory2, enum_extensions 

745 

746 

747def _supports_memory_budget(handle: c_void_p, enum_extensions: ctypes._FuncPointer) -> bool: 

748 """Whether the device advertises ``VK_EXT_memory_budget``. 

749 

750 Asked rather than assumed: chaining the budget struct onto a device that 

751 does not support it leaves it zeroed, and zero budget is indistinguishable 

752 from a full card. 

753 """ 

754 count = c_uint32(0) 

755 if enum_extensions(handle, None, byref(count), None) != _VK_SUCCESS or count.value == 0: 

756 return False 

757 props = (_VkExtensionProperties * count.value)() 

758 if enum_extensions(handle, None, byref(count), props) != _VK_SUCCESS: 

759 return False 

760 return any(props[i].extensionName == _VK_EXT_MEMORY_BUDGET for i in range(count.value)) 

761 

762 

763def _free_device_local_bytes( 

764 handle: c_void_p, memory_budget: tuple[ctypes._FuncPointer, ctypes._FuncPointer] | None 

765) -> int | None: 

766 """Device-local memory still available, or ``None`` when it cannot be asked. 

767 

768 ``None`` rather than the heap size on purpose. Reporting capacity as free is 

769 how a desktop holding gigabytes of compositor and browser VRAM was planned 

770 as an empty card. 

771 """ 

772 if memory_budget is None: 

773 return None 

774 get_memory2, enum_extensions = memory_budget 

775 if not _supports_memory_budget(handle, enum_extensions): 

776 return None 

777 budget = _VkPhysicalDeviceMemoryBudgetPropertiesEXT( 

778 sType=_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT, pNext=None 

779 ) 

780 props2 = _VkPhysicalDeviceMemoryProperties2( 

781 sType=_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PROPERTIES_2, 

782 pNext=ctypes.cast(ctypes.pointer(budget), c_void_p), 

783 ) 

784 get_memory2(handle, byref(props2)) 

785 mem = props2.memoryProperties 

786 free = 0 

787 for i in range(mem.memoryHeapCount): 

788 if mem.memoryHeaps[i].flags & _VK_MEMORY_HEAP_DEVICE_LOCAL_BIT: 

789 free += max(0, int(budget.heapBudget[i]) - int(budget.heapUsage[i])) 

790 return free 

791 

792 

793def _resolve_features2(lib: ctypes.CDLL) -> ctypes._FuncPointer | None: 

794 """``vkGetPhysicalDeviceFeatures2`` with argtypes stamped, ``None`` if absent.""" 

795 try: 

796 get_features2 = lib.vkGetPhysicalDeviceFeatures2 

797 except AttributeError: 

798 return None 

799 get_features2.argtypes = [c_void_p, POINTER(_VkPhysicalDeviceFeatures2)] 

800 get_features2.restype = None 

801 return get_features2 

802 

803 

804def _storage_buffer_16bit( 

805 handle: c_void_p, get_features2: ctypes._FuncPointer | None 

806) -> bool | None: 

807 """Whether the adapter supports ``storageBuffer16BitAccess``, ``None`` if unasked. 

808 

809 The one feature ggml's Vulkan backend requires before it will put a device 

810 in its pool, and it drops devices that lack it silently. Some Adreno parts 

811 expose ``uniformAndStorageBuffer16BitAccess`` without it, so the two are not 

812 interchangeable and only the first flag answers the question. 

813 """ 

814 if get_features2 is None: 

815 return None 

816 storage = _VkPhysicalDevice16BitStorageFeatures( 

817 sType=_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_16BIT_STORAGE_FEATURES, pNext=None 

818 ) 

819 features2 = _VkPhysicalDeviceFeatures2( 

820 sType=_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, 

821 pNext=ctypes.cast(ctypes.pointer(storage), c_void_p), 

822 ) 

823 get_features2(handle, byref(features2)) 

824 return bool(storage.storageBuffer16BitAccess) 

825 

826 

827def _device_uuid(handle: c_void_p, get_properties2: ctypes._FuncPointer | None) -> bytes: 

828 """The adapter's ``deviceUUID``, empty when it cannot be asked for. 

829 

830 An all-zero UUID is returned as empty too: it is what a driver leaves behind 

831 when it ignores the chained struct, and treating it as a real identity would 

832 collapse every such adapter into one. 

833 """ 

834 if get_properties2 is None: 

835 return b"" 

836 id_props = _VkPhysicalDeviceIDProperties( 

837 sType=_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ID_PROPERTIES, pNext=None 

838 ) 

839 props2 = _VkPhysicalDeviceProperties2( 

840 sType=_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, 

841 pNext=ctypes.cast(ctypes.pointer(id_props), c_void_p), 

842 ) 

843 get_properties2(handle, byref(props2)) 

844 uuid = bytes(id_props.deviceUUID) 

845 return b"" if not any(uuid) else uuid 

846 

847 

848def _list_devices_with_instance(lib: ctypes.CDLL) -> list[VulkanDevice]: 

849 """Create a temporary VkInstance, enumerate physical devices, destroy. 

850 

851 Mirrors what ``vulkaninfo --summary`` does internally. The 

852 instance is short-lived (created and destroyed in the same call) 

853 so the probe leaves no driver state behind. 

854 """ 

855 ( 

856 create_instance, 

857 destroy_instance, 

858 enum_physical, 

859 get_properties, 

860 get_memory, 

861 ) = _resolve_vk_symbols(lib) 

862 

863 instance, api_version = _create_probe_instance(create_instance) 

864 if instance is None: 

865 return [] 

866 core_1_1 = api_version >= _VK_API_VERSION_1_1 

867 get_properties2 = _resolve_properties2(lib) if core_1_1 else None 

868 get_features2 = _resolve_features2(lib) if core_1_1 else None 

869 memory_budget = _resolve_memory_budget(lib) if core_1_1 else None 

870 

871 try: 

872 count = c_uint32(0) 

873 result = enum_physical(instance, byref(count), None) 

874 if result != _VK_SUCCESS or count.value == 0: 

875 return [] 

876 handles = (c_void_p * count.value)() 

877 result = enum_physical(instance, byref(count), handles) 

878 if result != _VK_SUCCESS: 

879 return [] 

880 devices: list[VulkanDevice] = [] 

881 for i in range(count.value): 

882 # Indexing the array yields a bare address; the queries below take a 

883 # handle, so make it one rather than widen every signature to int. 

884 handle = c_void_p(handles[i]) 

885 props = _VkPhysicalDeviceProperties() 

886 get_properties(handle, byref(props)) 

887 mem = _VkPhysicalDeviceMemoryProperties() 

888 get_memory(handle, byref(mem)) 

889 devices.append( 

890 VulkanDevice( 

891 index=i, 

892 device_type=int(props.deviceType), 

893 device_name=props.deviceName.decode("utf-8", errors="replace"), 

894 vendor_id=int(props.vendorID), 

895 vram_bytes=_device_local_vram(mem), 

896 device_uuid=_device_uuid(handle, get_properties2), 

897 storage_buffer_16bit=_storage_buffer_16bit(handle, get_features2), 

898 free_bytes=_free_device_local_bytes(handle, memory_budget), 

899 ) 

900 ) 

901 return devices 

902 finally: 

903 destroy_instance(instance, None) 

904 

905 

906def _resolve_vk_symbols( 

907 lib: ctypes.CDLL, 

908) -> tuple[ 

909 ctypes._FuncPointer, 

910 ctypes._FuncPointer, 

911 ctypes._FuncPointer, 

912 ctypes._FuncPointer, 

913 ctypes._FuncPointer, 

914]: 

915 """Look up the five Vulkan symbols this probe needs and stamp argtypes. 

916 

917 All argtypes / restypes are set here so ctypes uses the same 

918 calling convention as the C ABI; missing this on Windows produces 

919 silent stack corruption. 

920 """ 

921 create_instance = lib.vkCreateInstance 

922 create_instance.argtypes = [ 

923 POINTER(_VkInstanceCreateInfo), 

924 c_void_p, 

925 POINTER(c_void_p), 

926 ] 

927 create_instance.restype = c_uint32 

928 

929 destroy_instance = lib.vkDestroyInstance 

930 destroy_instance.argtypes = [c_void_p, c_void_p] 

931 destroy_instance.restype = None 

932 

933 enum_physical = lib.vkEnumeratePhysicalDevices 

934 enum_physical.argtypes = [c_void_p, POINTER(c_uint32), POINTER(c_void_p)] 

935 enum_physical.restype = c_uint32 

936 

937 get_properties = lib.vkGetPhysicalDeviceProperties 

938 get_properties.argtypes = [c_void_p, POINTER(_VkPhysicalDeviceProperties)] 

939 get_properties.restype = None 

940 

941 get_memory = lib.vkGetPhysicalDeviceMemoryProperties 

942 get_memory.argtypes = [c_void_p, POINTER(_VkPhysicalDeviceMemoryProperties)] 

943 get_memory.restype = None 

944 

945 return create_instance, destroy_instance, enum_physical, get_properties, get_memory 

946 

947