Coverage for src/lilbee/providers/fleet/gpu_select.py: 100%

1"""Ask the host's Vulkan loader what ggml is going to see. 

2 

3Probes the loader via ``ctypes`` for the facts the engine's ``--list-devices`` 

4text does not carry: each adapter's device type, its ``deviceUUID``, whether it 

5supports the one feature ggml requires of it, and how much of its memory is 

6actually free. Placement uses these to agree with the engine about which devices 

7exist and how big they are; where the two disagree, a fleet gets sized against 

8hardware llama-server never uses. 

9 

10It deliberately does not choose a device. Selection belongs to ggml, which 

11applies its own type filter, support check and same-UUID dedup at launch; 

12pinning through ``GGML_VK_VISIBLE_DEVICES`` would switch all three off, so 

13Vulkan devices are pinned by the name the engine printed or not at all. 

14""" 

15 

16from __future__ import annotations 

17 

18import ctypes 

19import ctypes.util 

20import fnmatch 

21import json 

22import logging 

23import ntpath 

24import os 

25import sys 

26from collections import Counter 

27from ctypes import POINTER, byref, c_char, c_char_p, c_uint8, c_uint32, c_uint64, c_void_p 

28from dataclasses import dataclass 

29from enum import IntEnum, StrEnum 

30from functools import lru_cache 

31 

32from lilbee.providers.fleet.gpu_hardware import installed_gpu_vendor_ids 

33from lilbee.providers.fleet.vulkan_icd_discovery import ( 

34 iter_vulkan_manifest_paths, 

35) 

36 

37log = logging.getLogger(__name__) 

38 

39# The child that runs the loader, and how long it may take. The bound matters: 

40# a wedged ICD can hang inside vkCreateInstance rather than fault, and the 

41# placement read that asked must not hang with it. 

42_PROBE_MODULE = "lilbee.providers.fleet.vulkan_probe" 

43_PROBE_TIMEOUT_S = 10.0 

44_PROBE_KILL_WAIT_S = 5.0 

45 

46# vk.h constants. Mirrored here so we don't drag a vulkan-headers 

47# dependency in for four magic numbers. See the upstream definitions in 

48# https://github.com/KhronosGroup/Vulkan-Headers/blob/main/include/vulkan/vulkan_core.h 

49# (VkStructureType enum and the VK_API_VERSION_1_0 / VK_SUCCESS macros). 

50_VK_STRUCTURE_TYPE_APPLICATION_INFO = 0 

51_VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO = 1 

52_VK_SUCCESS = 0 

53_VK_API_VERSION_1_0 = (1 << 22) | (0 << 12) | 0 

54# 1.1 is asked for first, purely to make vkGetPhysicalDeviceProperties2 (and the 

55# device UUID it carries) core rather than an extension; a loader that refuses 

56# it gets the 1.0 request back and the probe simply has no UUIDs to dedup by. 

57_VK_API_VERSION_1_1 = (1 << 22) | (1 << 12) | 0 

58_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2 = 1000059000 

59_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2 = 1000059001 

60_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ID_PROPERTIES = 1000071004 

61_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_16BIT_STORAGE_FEATURES = 1000083000 

62_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PROPERTIES_2 = 1000059006 

63_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT = 1000237000 

64# The device extension that turns heap sizes into a live budget. Without it the 

65# only figure available is the heap's capacity, which never moves. 

66_VK_EXT_MEMORY_BUDGET = b"VK_EXT_memory_budget" 

67_VK_MAX_EXTENSION_NAME_SIZE = 256 

68 

69 

70class VkDeviceType(IntEnum): 

71 """``VkPhysicalDeviceType`` enum from vulkan_core.h. 

72 

73 Values match the C ABI verbatim; the loader writes one of these 

74 into the ``deviceType`` field of ``VkPhysicalDeviceProperties``. 

75 """ 

76 

77 OTHER = 0 

78 INTEGRATED_GPU = 1 

79 DISCRETE_GPU = 2 

80 VIRTUAL_GPU = 3 

81 CPU = 4 

82 

83 

84# The device types ggml's Vulkan backend will actually run on. Anything else -- 

85# a software rasterizer, a paravirtual adapter, an unknown type -- is not a 

86# device the engine would choose, so planning against one guarantees a mismatch. 

87USABLE_VULKAN_TYPES = frozenset({VkDeviceType.DISCRETE_GPU, VkDeviceType.INTEGRATED_GPU}) 

88 

89 

90# vk.h sizes for the inline char arrays inside VkPhysicalDeviceProperties. 

91# Both constants are part of the Vulkan 1.0 ABI and frozen forever; see 

92# VK_MAX_PHYSICAL_DEVICE_NAME_SIZE and VK_UUID_SIZE in 

93# https://github.com/KhronosGroup/Vulkan-Headers/blob/main/include/vulkan/vulkan_core.h 

94_VK_MAX_PHYSICAL_DEVICE_NAME_SIZE = 256 

95_VK_UUID_SIZE = 16 

96 

97 

98@dataclass(frozen=True) 

99class VulkanDevice: 

100 """One Vulkan adapter as reported by the loader.""" 

101 

102 index: int 

103 device_type: int 

104 device_name: str 

105 vendor_id: int 

106 vram_bytes: int = 0 

107 # VkPhysicalDeviceIDProperties::deviceUUID, empty when the loader could not 

108 # be asked for it. The spec requires it to be immutable for a given device 

109 # across instances, processes, driver APIs, driver versions and reboots, so 

110 # two entries sharing one is one piece of silicon behind two drivers. 

111 device_uuid: bytes = b"" 

112 # storageBuffer16BitAccess, the single feature ggml's Vulkan backend requires 

113 # of a device before it will use it. Read from 

114 # VkPhysicalDevice16BitStorageFeatures rather than the 

115 # VkPhysicalDeviceVulkan11Features ggml itself uses: same bit, but the latter 

116 # arrived in Vulkan 1.2 and this probe asks for a 1.1 instance. 

117 # ``None`` when the loader could not be asked, which is not a refusal. 

118 storage_buffer_16bit: bool | None = None 

119 # Device-local memory not already committed, from VK_EXT_memory_budget. 

120 # ``None`` when the device does not expose that extension, which is the 

121 # difference between "nothing else is using this card" and "cannot tell". 

122 free_bytes: int | None = None 

123 

124 

125class PCIVendorID(IntEnum): 

126 """PCI-SIG vendor IDs for the GPU vendors that ship Vulkan ICDs. 

127 

128 Values are the canonical PCI vendor IDs that 

129 ``VkPhysicalDeviceProperties.vendorID`` surfaces. They are issued by 

130 PCI-SIG and frozen per company; see the public PCI vendor-ID 

131 registry at https://pcisig.com/membership/member-companies (also 

132 mirrored at https://devicehunt.com/all-pci-vendors). Only the 

133 vendors we have explicit ICD-disable globs for are enumerated; 

134 unknown vendors fall through the dispatch as no-op. 

135 """ 

136 

137 NVIDIA = 0x10DE # NVIDIA Corporation 

138 AMD = 0x1002 # Advanced Micro Devices, Inc. [AMD/ATI] 

139 INTEL = 0x8086 # Intel Corporation 

140 

141 

142# Vulkan loader manifest filename globs, per vendor. The loader matches these 

143# against the JSON manifest filename in its known-drivers list (see 

144# https://github.com/KhronosGroup/Vulkan-Loader/blob/main/docs/LoaderInterfaceArchitecture.md). 

145# Each vendor ships under multiple names across drivers/OSes; list every form 

146# we may encounter so disabling one vendor's drivers doesn't half-disable them. 

147_VENDOR_ICD_GLOBS: dict[PCIVendorID, tuple[str, ...]] = { 

148 # nv-vk*.json (Windows), nvidia_*.json (Linux). Both match nv*. 

149 PCIVendorID.NVIDIA: ("nv*",), 

150 # amdvlk64.json (Windows AMDVLK), amd_icd*.json (Linux AMDVLK), 

151 # amd-vulkan*.json (legacy AMDVLK builds), radeon_icd.*.json 

152 # (Mesa RADV on Linux). Adding amd_icd* explicitly because no 

153 # other glob covers the Linux AMDVLK manifest. 

154 PCIVendorID.AMD: ("amdvlk*", "amd_icd*", "amd-vulkan*", "radeon*"), 

155 # intel_icd.*.json (Mesa Intel ANV on Linux), igvk*.json (Windows). 

156 PCIVendorID.INTEL: ("intel*", "igvk*"), 

157} 

158 

159 

160class VulkanIcdEnvVar(StrEnum): 

161 """Every documented Vulkan loader env var that influences ICD selection. 

162 

163 Names are the verbatim loader env vars from the Khronos 

164 LoaderInterfaceArchitecture spec; the StrEnum lets each member be 

165 used directly as a ``str`` argument to ``os.environ.get`` / 

166 ``os.environ.setdefault`` without ``.value`` plumbing. Any value 

167 being non-empty in the environment is treated as a user override 

168 and suppresses the dual-vendor auto-pin. 

169 """ 

170 

171 DRIVER_FILES = "VK_DRIVER_FILES" 

172 ICD_FILENAMES = "VK_ICD_FILENAMES" 

173 ADD_DRIVER_FILES = "VK_ADD_DRIVER_FILES" 

174 LOADER_DRIVERS_DISABLE = "VK_LOADER_DRIVERS_DISABLE" 

175 LOADER_DRIVERS_SELECT = "VK_LOADER_DRIVERS_SELECT" 

176 

177 

178# Field layouts from the Vulkan 1.0 spec. ctypes maps the C structs 

179# verbatim so the loader populates them directly; only the prefix 

180# fields we read are commented (the trailing fields are kept for ABI 

181# alignment, not consumed). 

182 

183 

184class _VkApplicationInfo(ctypes.Structure): 

185 _fields_ = [ 

186 ("sType", c_uint32), 

187 ("pNext", c_void_p), 

188 ("pApplicationName", c_char_p), 

189 ("applicationVersion", c_uint32), 

190 ("pEngineName", c_char_p), 

191 ("engineVersion", c_uint32), 

192 ("apiVersion", c_uint32), 

193 ] 

194 

195 

196class _VkInstanceCreateInfo(ctypes.Structure): 

197 _fields_ = [ 

198 ("sType", c_uint32), 

199 ("pNext", c_void_p), 

200 ("flags", c_uint32), 

201 ("pApplicationInfo", POINTER(_VkApplicationInfo)), 

202 ("enabledLayerCount", c_uint32), 

203 ("ppEnabledLayerNames", POINTER(c_char_p)), 

204 ("enabledExtensionCount", c_uint32), 

205 ("ppEnabledExtensionNames", POINTER(c_char_p)), 

206 ] 

207 

208 

209class _VkPhysicalDeviceLimits(ctypes.Structure): 

210 # Opaque to us; we only need the parent struct's *layout* to match 

211 # the driver-populated bytes so the loader can write a vendorID and 

212 # deviceType into the prefix fields we actually read. 

213 # 

214 # 504 bytes, per VkPhysicalDeviceLimits in 

215 # https://github.com/KhronosGroup/Vulkan-Headers/blob/main/include/vulkan/vulkan_core.h 

216 # The size is part of the frozen Vulkan 1.0 layout, so it does not drift 

217 # across driver versions. 

218 # 

219 # Declared as uint64 rather than bytes for its ALIGNMENT, not its size. The 

220 # real struct mixes uint32, uint64 (VkDeviceSize), size_t and float, so its C 

221 # alignment is 8. A c_uint8 array aligns to 1, which let ctypes seat this 

222 # field at offset 292 in the parent instead of the 296 the ABI pads it to, 

223 # making the mirror 816 bytes against the driver's 824. The driver fills the 

224 # caller's buffer using its own layout, so every probe wrote sparseProperties 

225 # four bytes past the end of a Python-heap allocation -- absorbed by allocator 

226 # slack, which is what kept it silent. 

227 _fields_ = [("_opaque", c_uint64 * 63)] 

228 

229 

230class _VkPhysicalDeviceSparseProperties(ctypes.Structure): 

231 # 5 ULONG32 booleans, also part of the Vulkan 1.0 ABI; see same header. 

232 _fields_ = [("_opaque", c_uint32 * 5)] 

233 

234 

235class _VkPhysicalDeviceProperties(ctypes.Structure): 

236 _fields_ = [ 

237 ("apiVersion", c_uint32), 

238 ("driverVersion", c_uint32), 

239 ("vendorID", c_uint32), 

240 ("deviceID", c_uint32), 

241 ("deviceType", c_uint32), 

242 ("deviceName", c_char * _VK_MAX_PHYSICAL_DEVICE_NAME_SIZE), 

243 ("pipelineCacheUUID", c_uint8 * _VK_UUID_SIZE), 

244 ("limits", _VkPhysicalDeviceLimits), 

245 ("sparseProperties", _VkPhysicalDeviceSparseProperties), 

246 ] 

247 

248 

249# VkPhysicalDeviceMemoryProperties layout (Vulkan 1.0 ABI, frozen). Array 

250# bounds and the device-local heap flag are from vulkan_core.h. The 

251# device-local heap size is the cross-vendor VRAM signal (the same heap 

252# nvidia-smi/rocm-smi report) used for placement bin-packing. 

253_VK_MAX_MEMORY_TYPES = 32 

254_VK_MAX_MEMORY_HEAPS = 16 

255_VK_MEMORY_HEAP_DEVICE_LOCAL_BIT = 0x00000001 

256 

257 

258class _VkMemoryType(ctypes.Structure): 

259 _fields_ = [("propertyFlags", c_uint32), ("heapIndex", c_uint32)] 

260 

261 

262class _VkMemoryHeap(ctypes.Structure): 

263 _fields_ = [("size", c_uint64), ("flags", c_uint32)] 

264 

265 

266class _VkPhysicalDevice16BitStorageFeatures(ctypes.Structure): 

267 # Promoted to core in Vulkan 1.1 from VK_KHR_16bit_storage. Chained onto 

268 # VkPhysicalDeviceFeatures2; only the first flag is read. 

269 _fields_ = [ 

270 ("sType", c_uint32), 

271 ("pNext", c_void_p), 

272 ("storageBuffer16BitAccess", c_uint32), 

273 ("uniformAndStorageBuffer16BitAccess", c_uint32), 

274 ("storagePushConstant16", c_uint32), 

275 ("storageInputOutput16", c_uint32), 

276 ] 

277 

278 

279class _VkPhysicalDeviceFeatures2(ctypes.Structure): 

280 # VkPhysicalDeviceFeatures is a flat run of VkBool32s whose count grows with 

281 # no version of the spec but is easy to miscount, and the driver writes the 

282 # whole thing into this buffer. Declared larger than the real struct so a 

283 # miscount cannot become a heap overrun the way the limits mirror once did; 

284 # the field sits last, so the extra words shift nothing the driver reads. 

285 _fields_ = [ 

286 ("sType", c_uint32), 

287 ("pNext", c_void_p), 

288 ("features", c_uint32 * 128), 

289 ] 

290 

291 

292class _VkExtensionProperties(ctypes.Structure): 

293 _fields_ = [ 

294 ("extensionName", c_char * _VK_MAX_EXTENSION_NAME_SIZE), 

295 ("specVersion", c_uint32), 

296 ] 

297 

298 

299class _VkPhysicalDeviceIDProperties(ctypes.Structure): 

300 # VkPhysicalDeviceIDProperties, promoted to core in Vulkan 1.1. Chained onto 

301 # VkPhysicalDeviceProperties2 via pNext; the driver fills every field, so the 

302 # trailing ones are declared for layout even though only deviceUUID is read. 

303 _fields_ = [ 

304 ("sType", c_uint32), 

305 ("pNext", c_void_p), 

306 ("deviceUUID", c_uint8 * _VK_UUID_SIZE), 

307 ("driverUUID", c_uint8 * _VK_UUID_SIZE), 

308 ("deviceLUID", c_uint8 * 8), 

309 ("deviceNodeMask", c_uint32), 

310 ("deviceLUIDValid", c_uint32), 

311 ] 

312 

313 

314class _VkPhysicalDeviceProperties2(ctypes.Structure): 

315 _fields_ = [ 

316 ("sType", c_uint32), 

317 ("pNext", c_void_p), 

318 ("properties", _VkPhysicalDeviceProperties), 

319 ] 

320 

321 

322class _VkPhysicalDeviceMemoryProperties(ctypes.Structure): 

323 _fields_ = [ 

324 ("memoryTypeCount", c_uint32), 

325 ("memoryTypes", _VkMemoryType * _VK_MAX_MEMORY_TYPES), 

326 ("memoryHeapCount", c_uint32), 

327 ("memoryHeaps", _VkMemoryHeap * _VK_MAX_MEMORY_HEAPS), 

328 ] 

329 

330 

331class _VkPhysicalDeviceMemoryProperties2(ctypes.Structure): 

332 _fields_ = [ 

333 ("sType", c_uint32), 

334 ("pNext", c_void_p), 

335 ("memoryProperties", _VkPhysicalDeviceMemoryProperties), 

336 ] 

337 

338 

339class _VkPhysicalDeviceMemoryBudgetPropertiesEXT(ctypes.Structure): 

340 # heapBudget is what this process may still allocate from each heap and 

341 # heapUsage what it already has; the difference across the device-local heaps 

342 # is the only cross-vendor figure that moves when another process takes VRAM. 

343 _fields_ = [ 

344 ("sType", c_uint32), 

345 ("pNext", c_void_p), 

346 ("heapBudget", c_uint64 * _VK_MAX_MEMORY_HEAPS), 

347 ("heapUsage", c_uint64 * _VK_MAX_MEMORY_HEAPS), 

348 ] 

349 

350 

351def enumerate_gpu_vram() -> list[tuple[int, int, int]] | None: 

352 """Return ``[(device_index, device_local_vram_bytes, free_bytes), ...]`` or ``None``. 

353 

354 Cross-vendor via the Vulkan probe (NVIDIA/AMD/Intel). ``None`` when the 

355 loader/probe is unavailable (macOS Metal, no Vulkan driver), so the 

356 placement planner can degrade to count-only or in-process. 

357 

358 Only discrete and integrated adapters are returned, the same rule ggml's 

359 Vulkan backend applies when it picks a device, so this cannot offer 

360 placement something the engine would refuse to run on. Matching that rule 

361 is the point: where the two disagree about which devices exist, placement 

362 sizes against a device llama-server never uses. 

363 

364 Two kinds are excluded. Mesa's llvmpipe is a software rasterizer that 

365 advertises itself through Vulkan and reports system RAM as its device 

366 memory, so beside integrated graphics it appears at an identical size and 

367 is indistinguishable by VRAM alone; planning against it splits the model 

368 across a real GPU and a CPU renderer. Paravirtual adapters (virgl, VMware, 

369 VirtIO-GPU) report as ``VIRTUAL_GPU`` and are typically compute-incapable 

370 or proxies that fail on allocation. 

371 

372 The device type is the only signal separating any of these, and the caller 

373 has no access to it: this returns sizes, and the ``--list-devices`` text it 

374 feeds carries no names on the fallback path. 

375 """ 

376 devices = _enumerate_vulkan_devices() 

377 if devices is None: 

378 return None 

379 return [ 

380 (d.index, d.vram_bytes, d.free_bytes if d.free_bytes is not None else d.vram_bytes) 

381 for d in devices 

382 if d.device_type in USABLE_VULKAN_TYPES 

383 ] 

384 

385 

386@lru_cache(maxsize=1) 

387def integrated_vulkan_indices() -> frozenset[int]: 

388 """Loader indices of adapters whose memory is the host's. 

389 

390 Empty when the loader is unavailable or the probe fails, which reads as 

391 "assume dedicated" and preserves the behaviour discrete hosts already have. 

392 

393 Cached because the device parser asks per device line: without it an 

394 N-device host paid N loader loads and N instance creations to answer the 

395 same question. Which adapters are integrated is a property of the machine, 

396 so one answer per process is right. 

397 """ 

398 devices = _enumerate_vulkan_devices() 

399 if not devices: 

400 return frozenset() 

401 return frozenset(d.index for d in devices if d.device_type == VkDeviceType.INTEGRATED_GPU) 

402 

403 

404def vulkan_free_bytes_by_name() -> dict[str, int]: 

405 """Device-local memory still free, keyed by the name the loader reports. 

406 

407 Deliberately not cached: free memory is a live number, and freezing it for 

408 the process lifetime would hand every later probe the first reading taken. 

409 Callers sample it once per parse rather than per device line. 

410 

411 Only devices whose driver exposes ``VK_EXT_memory_budget`` appear; the rest 

412 have no live figure to offer and are absent rather than guessed at. A name 

413 two adapters share is also absent: two identical cards have their own free 

414 figures, and nothing in the engine's text says which line is which. Guessing 

415 would report one card's headroom for the other. 

416 """ 

417 devices = _enumerate_vulkan_devices() 

418 if not devices: 

419 return {} 

420 seen = Counter(d.device_name for d in devices) 

421 return { 

422 d.device_name: d.free_bytes 

423 for d in devices 

424 if d.free_bytes is not None and seen[d.device_name] == 1 

425 } 

426 

427 

428@lru_cache(maxsize=1) 

429def vulkan_device_types_by_name() -> dict[str, VkDeviceType]: 

430 """Adapter type keyed by the name the loader reports, empty when unavailable. 

431 

432 Keyed by name rather than index because the engine's ``--list-devices`` 

433 ordinals are assigned after ggml has filtered and deduplicated the loader's 

434 list, so ``Vulkan0`` is only the loader's device 0 when nothing ahead of it 

435 was dropped. The name is the one field both views print verbatim from 

436 ``VkPhysicalDeviceProperties``, so it correlates the two without either 

437 side having to replicate the other's filtering. 

438 

439 Two adapters of the same model share a name, which is harmless: they share 

440 a type too, and the type is all this answers. 

441 """ 

442 devices = _enumerate_vulkan_devices() 

443 if not devices: 

444 return {} 

445 return { 

446 d.device_name: device_type 

447 for d in devices 

448 if (device_type := _known_device_type(d.device_type)) is not None 

449 } 

450 

451 

452def discrete_gpu_from_vendor(vendor_id: int) -> bool | None: 

453 """Whether the loader reports a discrete adapter from *vendor_id*. 

454 

455 ``None`` when the loader cannot be reached, which is a different answer from 

456 "no": a caller deciding whether to fail loud must not read silence as proof 

457 that a card is absent, nor as proof that one is present. 

458 """ 

459 devices = _enumerate_vulkan_devices() 

460 if not devices: 

461 return None 

462 return any( 

463 d.vendor_id == vendor_id and d.device_type == VkDeviceType.DISCRETE_GPU for d in devices 

464 ) 

465 

466 

467# Vendors whose PCI display controllers are always dedicated cards. AMD and 

468# Intel both ship integrated parts under their own IDs, so their presence says 

469# nothing about whether a discrete card exists; NVIDIA's desktop and laptop 

470# parts are discrete without exception here. 

471_DISCRETE_ONLY_VENDORS: frozenset[int] = frozenset({PCIVendorID.NVIDIA}) 

472 

473 

474def host_has_no_discrete_gpu() -> bool: 

475 """Whether the Vulkan loader can see adapters and none of them is discrete. 

476 

477 The vendor-neutral answer to a question CUDA and ROCm cannot be asked 

478 through text: their ``--list-devices`` lines carry no device type, so an AMD 

479 APU and a Jetson enumerate exactly like a discrete card while reporting 

480 system RAM as their memory. Every such part also ships a Vulkan driver, and 

481 a machine whose loader reports adapters but no discrete one has no discrete 

482 GPU for CUDA or ROCm to be enumerating. 

483 

484 The verdict rests on an integrated adapter actually being there. Software 

485 rasterizers report through the loader on any host with mesa installed, even 

486 with no vendor ICD present at all, which is ordinary on headless CUDA boxes 

487 and in containers; concluding from a list that holds only those would mark a 

488 real discrete card as sharing the host's memory and shrink its budget. 

489 

490 False when the loader is unreachable, when any discrete adapter exists, or 

491 when nothing but rasterizers answered, so a host with a real card is never 

492 talked into the shared-memory budget. A host holding both a discrete card 

493 and an APU also answers False, which leaves the APU sized as dedicated; 

494 correlating individual devices across two backends' naming needs more than 

495 the type. 

496 

497 The loader is not the only witness, and on a hybrid laptop it is the wrong 

498 one. Optimus and its equivalents leave the discrete card powered down until 

499 something asks for it through prime-run, so the loader enumerates the 

500 integrated adapter alone while a dedicated card sits on the PCI bus. Taking 

501 that list at its word marked a real 4 GB card as sharing system memory, 

502 which is the exact outcome the paragraph above promises never to reach. PCI 

503 settles it: a vendor that only ever ships discrete parts, present in the 

504 device tree but absent from the loader's list, means the loader is telling 

505 an incomplete story rather than a complete one. 

506 """ 

507 types = set(vulkan_device_types_by_name().values()) 

508 if VkDeviceType.DISCRETE_GPU in types: 

509 return False 

510 if _DISCRETE_ONLY_VENDORS & installed_gpu_vendor_ids(): 

511 return False 

512 return VkDeviceType.INTEGRATED_GPU in types 

513 

514 

515def _known_device_type(value: int) -> VkDeviceType | None: 

516 """The enum member for a raw ``deviceType``, ``None`` for a value vk.h doesn't define.""" 

517 try: 

518 return VkDeviceType(value) 

519 except ValueError: 

520 return None 

521 

522 

523def enumerate_in_process() -> list[VulkanDevice] | None: 

524 """Open libvulkan, create a throwaway instance, enumerate adapters. 

525 

526 Returns ``None`` if the loader can't be found or any Vulkan call 

527 fails; empty list ("loader present, no adapters") is a distinct 

528 outcome and propagates back. 

529 

530 Runs the loader in whatever process calls it, which is why 

531 :func:`_enumerate_vulkan_devices` calls it in a child rather than directly: 

532 ``vkCreateInstance`` loads every vendor ICD on the host, and a faulting one 

533 raises no exception, it raises a signal. 

534 """ 

535 lib = _load_vulkan_loader() 

536 if lib is None: 

537 return None 

538 try: 

539 devices = _list_devices_with_instance(lib) 

540 return _deduplicate_by_uuid(_drop_devices_the_engine_refuses(devices)) 

541 except OSError: 

542 # ctypes argument / call-site errors land here; treat as 

543 # "probe failed" rather than crashing the host process. 

544 return None 

545 

546 

547def _run_probe_child() -> tuple[str, int, str]: 

548 """Run the enumeration in a child; returns ``(stdout, returncode, stderr)``.""" 

549 from lilbee.providers.fleet.proc import run_bounded 

550 

551 argv = [sys.executable, "-m", _PROBE_MODULE] 

552 stdout, returncode = run_bounded( 

553 argv, 

554 timeout_s=_PROBE_TIMEOUT_S, 

555 kill_wait_s=_PROBE_KILL_WAIT_S, 

556 label="vulkan-probe", 

557 ) 

558 return stdout, returncode, "" 

559 

560 

561def _enumerate_vulkan_devices() -> list[VulkanDevice] | None: 

562 """The adapters the Vulkan loader reports, or ``None`` for no opinion. 

563 

564 Asked of a short-lived child. ``vkCreateInstance`` pre-loads every vendor ICD 

565 on the host, and a broken or conflicting one faults inside the loader; a 

566 fault is a signal, so no ``except`` here could keep the daemon alive. In a 

567 child, dying is simply an answer. Every failure reads the same as an 

568 unreachable loader, which is the state the callers were written for. 

569 

570 An empty list still means "loader present, no adapters", which is a 

571 different fact and propagates back intact. 

572 

573 Uncached, and each caller decides for itself whether to hold the answer: the 

574 device types are a property of the machine and are cached, while free memory 

575 is a live number that is read fresh every time it is asked for. 

576 """ 

577 from lilbee.providers.base import ProviderError 

578 from lilbee.providers.fleet.vulkan_probe import from_json 

579 

580 try: 

581 stdout, returncode, stderr = _run_probe_child() 

582 except (ProviderError, OSError) as exc: 

583 log.debug("Vulkan probe child could not be run: %s", exc) 

584 return None 

585 if returncode != 0: 

586 log.debug("Vulkan probe child exited %s: %s", returncode, stderr.strip() or stdout.strip()) 

587 return None 

588 try: 

589 return from_json(json.loads(stdout)) 

590 except (ValueError, KeyError, TypeError) as exc: 

591 log.debug("Vulkan probe child printed no usable device list: %s", exc) 

592 return None 

593 

594 

595def _drop_devices_the_engine_refuses(devices: list[VulkanDevice]) -> list[VulkanDevice]: 

596 """Drop adapters ggml's Vulkan backend would exclude from its device pool. 

597 

598 ``ggml_vk_device_is_supported`` gates on exactly one feature, 

599 ``storageBuffer16BitAccess``, and excludes devices without it silently, with 

600 no error anywhere. Some Adreno parts are the documented case. Keeping such a 

601 device means placement sizes a fleet against VRAM the engine will never 

602 touch, and the engine quietly runs on the CPU or another adapter instead. 

603 

604 Only a definite ``False`` drops a device: a loader too old to be asked 

605 reports ``None``, and that is not a refusal. 

606 """ 

607 return [d for d in devices if d.storage_buffer_16bit is not False] 

608 

609 

610def _deduplicate_by_uuid(devices: list[VulkanDevice]) -> list[VulkanDevice]: 

611 """Collapse adapters that share a ``deviceUUID`` into one, keeping the first. 

612 

613 Two ICDs able to drive the same card (RADV beside AMDVLK is the case ggml's 

614 own dedup names) enumerate it twice. ggml counts it once, so without this 

615 lilbee plans a two-GPU fleet on one piece of silicon and tensor-splits a 

616 model across a card and itself. 

617 

618 ggml breaks the same tie with a driver-priority table, picking which 

619 driver's entry survives. Lowest index is enough here because nothing lilbee 

620 reads off a device tells the two entries apart: the type, the name and the 

621 device-local heap size describe the silicon, not the driver, and no caller 

622 pins by the raw enumeration index any more. 

623 

624 Devices with no UUID are all kept, since "the loader would not say" is not 

625 evidence that two adapters are one. 

626 """ 

627 seen: set[bytes] = set() 

628 unique: list[VulkanDevice] = [] 

629 for device in devices: 

630 if device.device_uuid and device.device_uuid in seen: 

631 log.debug( 

632 "Vulkan device %d (%s) is device %s under a second driver; ignoring the duplicate", 

633 device.index, 

634 device.device_name, 

635 next(d.index for d in unique if d.device_uuid == device.device_uuid), 

636 ) 

637 continue 

638 if device.device_uuid: 

639 seen.add(device.device_uuid) 

640 unique.append(device) 

641 return unique 

642 

643 

644def _load_vulkan_loader() -> ctypes.CDLL | None: 

645 """Locate and load the Vulkan loader for the current platform. 

646 

647 Returns ``None`` when the loader isn't installed, which is the 

648 expected outcome on stock macOS (we ship a Metal wheel there) and 

649 on hosts without a Vulkan-capable driver. 

650 """ 

651 candidates: tuple[str, ...] 

652 if sys.platform == "win32": 

653 candidates = ("vulkan-1.dll",) 

654 elif sys.platform == "darwin": 

655 # MoltenVK exposes a different ABI than libvulkan; lilbee's 

656 # macOS wheel uses Metal directly, so skipping the probe on 

657 # Darwin is correct. 

658 return None 

659 else: 

660 candidates = ("libvulkan.so.1", "libvulkan.so") 

661 

662 for name in candidates: 

663 try: 

664 return ctypes.CDLL(name) 

665 except OSError: 

666 continue 

667 # ctypes.util.find_library is a last-resort fallback for distros 

668 # where the soname isn't directly loadable. 

669 resolved = ctypes.util.find_library("vulkan") 

670 if resolved is not None: 

671 try: 

672 return ctypes.CDLL(resolved) 

673 except OSError: 

674 return None 

675 return None 

676 

677 

678def _create_probe_instance(create_instance: ctypes._FuncPointer) -> tuple[c_void_p | None, int]: 

679 """Create the throwaway instance, asking for 1.1 and settling for 1.0. 

680 

681 Returns the instance and the API version it was created with. 1.1 makes 

682 ``vkGetPhysicalDeviceProperties2`` core, which is where the device UUID 

683 lives; a 1.0-only loader rejects the request outright, so the 1.0 retry is 

684 what keeps the probe working there at all rather than silently reporting no 

685 adapters. 

686 """ 

687 for api_version in (_VK_API_VERSION_1_1, _VK_API_VERSION_1_0): 

688 app_info = _VkApplicationInfo( 

689 sType=_VK_STRUCTURE_TYPE_APPLICATION_INFO, 

690 pNext=None, 

691 pApplicationName=b"lilbee-gpu-probe", 

692 applicationVersion=0, 

693 pEngineName=b"lilbee", 

694 engineVersion=0, 

695 apiVersion=api_version, 

696 ) 

697 create_info = _VkInstanceCreateInfo( 

698 sType=_VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO, 

699 pNext=None, 

700 flags=0, 

701 pApplicationInfo=ctypes.pointer(app_info), 

702 enabledLayerCount=0, 

703 ppEnabledLayerNames=None, 

704 enabledExtensionCount=0, 

705 ppEnabledExtensionNames=None, 

706 ) 

707 instance = c_void_p() 

708 result = create_instance(byref(create_info), None, byref(instance)) 

709 if result == _VK_SUCCESS and instance.value: 

710 return instance, api_version 

711 return None, 0 

712 

713 

714def _resolve_properties2(lib: ctypes.CDLL) -> ctypes._FuncPointer | None: 

715 """``vkGetPhysicalDeviceProperties2`` with argtypes stamped, ``None`` if absent.""" 

716 try: 

717 get_properties2 = lib.vkGetPhysicalDeviceProperties2 

718 except AttributeError: 

719 return None 

720 get_properties2.argtypes = [c_void_p, POINTER(_VkPhysicalDeviceProperties2)] 

721 get_properties2.restype = None 

722 return get_properties2 

723 

724 

725def _resolve_memory_budget( 

726 lib: ctypes.CDLL, 

727) -> tuple[ctypes._FuncPointer, ctypes._FuncPointer] | None: 

728 """``(vkGetPhysicalDeviceMemoryProperties2, vkEnumerateDeviceExtensionProperties)``.""" 

729 try: 

730 get_memory2 = lib.vkGetPhysicalDeviceMemoryProperties2 

731 enum_extensions = lib.vkEnumerateDeviceExtensionProperties 

732 except AttributeError: 

733 return None 

734 get_memory2.argtypes = [c_void_p, POINTER(_VkPhysicalDeviceMemoryProperties2)] 

735 get_memory2.restype = None 

736 enum_extensions.argtypes = [ 

737 c_void_p, 

738 c_char_p, 

739 POINTER(c_uint32), 

740 POINTER(_VkExtensionProperties), 

741 ] 

742 enum_extensions.restype = c_uint32 

743 return get_memory2, enum_extensions 

744 

745 

746def _supports_memory_budget(handle: c_void_p, enum_extensions: ctypes._FuncPointer) -> bool: 

747 """Whether the device advertises ``VK_EXT_memory_budget``. 

748 

749 Asked rather than assumed: chaining the budget struct onto a device that 

750 does not support it leaves it zeroed, and zero budget is indistinguishable 

751 from a full card. 

752 """ 

753 count = c_uint32(0) 

754 if enum_extensions(handle, None, byref(count), None) != _VK_SUCCESS or count.value == 0: 

755 return False 

756 props = (_VkExtensionProperties * count.value)() 

757 if enum_extensions(handle, None, byref(count), props) != _VK_SUCCESS: 

758 return False 

759 return any(props[i].extensionName == _VK_EXT_MEMORY_BUDGET for i in range(count.value)) 

760 

761 

762def _free_device_local_bytes( 

763 handle: c_void_p, memory_budget: tuple[ctypes._FuncPointer, ctypes._FuncPointer] | None 

764) -> int | None: 

765 """Device-local memory still available, or ``None`` when it cannot be asked. 

766 

767 ``None`` rather than the heap size on purpose. Reporting capacity as free is 

768 how a desktop holding gigabytes of compositor and browser VRAM was planned 

769 as an empty card. 

770 """ 

771 if memory_budget is None: 

772 return None 

773 get_memory2, enum_extensions = memory_budget 

774 if not _supports_memory_budget(handle, enum_extensions): 

775 return None 

776 budget = _VkPhysicalDeviceMemoryBudgetPropertiesEXT( 

777 sType=_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT, pNext=None 

778 ) 

779 props2 = _VkPhysicalDeviceMemoryProperties2( 

780 sType=_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PROPERTIES_2, 

781 pNext=ctypes.cast(ctypes.pointer(budget), c_void_p), 

782 ) 

783 get_memory2(handle, byref(props2)) 

784 mem = props2.memoryProperties 

785 free = 0 

786 for i in range(mem.memoryHeapCount): 

787 if mem.memoryHeaps[i].flags & _VK_MEMORY_HEAP_DEVICE_LOCAL_BIT: 

788 free += max(0, int(budget.heapBudget[i]) - int(budget.heapUsage[i])) 

789 return free 

790 

791 

792def _resolve_features2(lib: ctypes.CDLL) -> ctypes._FuncPointer | None: 

793 """``vkGetPhysicalDeviceFeatures2`` with argtypes stamped, ``None`` if absent.""" 

794 try: 

795 get_features2 = lib.vkGetPhysicalDeviceFeatures2 

796 except AttributeError: 

797 return None 

798 get_features2.argtypes = [c_void_p, POINTER(_VkPhysicalDeviceFeatures2)] 

799 get_features2.restype = None 

800 return get_features2 

801 

802 

803def _storage_buffer_16bit( 

804 handle: c_void_p, get_features2: ctypes._FuncPointer | None 

805) -> bool | None: 

806 """Whether the adapter supports ``storageBuffer16BitAccess``, ``None`` if unasked. 

807 

808 The one feature ggml's Vulkan backend requires before it will put a device 

809 in its pool, and it drops devices that lack it silently. Some Adreno parts 

810 expose ``uniformAndStorageBuffer16BitAccess`` without it, so the two are not 

811 interchangeable and only the first flag answers the question. 

812 """ 

813 if get_features2 is None: 

814 return None 

815 storage = _VkPhysicalDevice16BitStorageFeatures( 

816 sType=_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_16BIT_STORAGE_FEATURES, pNext=None 

817 ) 

818 features2 = _VkPhysicalDeviceFeatures2( 

819 sType=_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, 

820 pNext=ctypes.cast(ctypes.pointer(storage), c_void_p), 

821 ) 

822 get_features2(handle, byref(features2)) 

823 return bool(storage.storageBuffer16BitAccess) 

824 

825 

826def _device_uuid(handle: c_void_p, get_properties2: ctypes._FuncPointer | None) -> bytes: 

827 """The adapter's ``deviceUUID``, empty when it cannot be asked for. 

828 

829 An all-zero UUID is returned as empty too: it is what a driver leaves behind 

830 when it ignores the chained struct, and treating it as a real identity would 

831 collapse every such adapter into one. 

832 """ 

833 if get_properties2 is None: 

834 return b"" 

835 id_props = _VkPhysicalDeviceIDProperties( 

836 sType=_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ID_PROPERTIES, pNext=None 

837 ) 

838 props2 = _VkPhysicalDeviceProperties2( 

839 sType=_VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, 

840 pNext=ctypes.cast(ctypes.pointer(id_props), c_void_p), 

841 ) 

842 get_properties2(handle, byref(props2)) 

843 uuid = bytes(id_props.deviceUUID) 

844 return b"" if not any(uuid) else uuid 

845 

846 

847def _list_devices_with_instance(lib: ctypes.CDLL) -> list[VulkanDevice]: 

848 """Create a temporary VkInstance, enumerate physical devices, destroy. 

849 

850 Mirrors what ``vulkaninfo --summary`` does internally. The 

851 instance is short-lived (created and destroyed in the same call) 

852 so the probe leaves no driver state behind. 

853 """ 

854 ( 

855 create_instance, 

856 destroy_instance, 

857 enum_physical, 

858 get_properties, 

859 get_memory, 

860 ) = _resolve_vk_symbols(lib) 

861 

862 instance, api_version = _create_probe_instance(create_instance) 

863 if instance is None: 

864 return [] 

865 core_1_1 = api_version >= _VK_API_VERSION_1_1 

866 get_properties2 = _resolve_properties2(lib) if core_1_1 else None 

867 get_features2 = _resolve_features2(lib) if core_1_1 else None 

868 memory_budget = _resolve_memory_budget(lib) if core_1_1 else None 

869 

870 try: 

871 count = c_uint32(0) 

872 result = enum_physical(instance, byref(count), None) 

873 if result != _VK_SUCCESS or count.value == 0: 

874 return [] 

875 handles = (c_void_p * count.value)() 

876 result = enum_physical(instance, byref(count), handles) 

877 if result != _VK_SUCCESS: 

878 return [] 

879 devices: list[VulkanDevice] = [] 

880 for i in range(count.value): 

881 # Indexing the array yields a bare address; the queries below take a 

882 # handle, so make it one rather than widen every signature to int. 

883 handle = c_void_p(handles[i]) 

884 props = _VkPhysicalDeviceProperties() 

885 get_properties(handle, byref(props)) 

886 mem = _VkPhysicalDeviceMemoryProperties() 

887 get_memory(handle, byref(mem)) 

888 devices.append( 

889 VulkanDevice( 

890 index=i, 

891 device_type=int(props.deviceType), 

892 device_name=props.deviceName.decode("utf-8", errors="replace"), 

893 vendor_id=int(props.vendorID), 

894 vram_bytes=_device_local_vram(mem), 

895 device_uuid=_device_uuid(handle, get_properties2), 

896 storage_buffer_16bit=_storage_buffer_16bit(handle, get_features2), 

897 free_bytes=_free_device_local_bytes(handle, memory_budget), 

898 ) 

899 ) 

900 return devices 

901 finally: 

902 destroy_instance(instance, None) 

903 

904 

905def _resolve_vk_symbols( 

906 lib: ctypes.CDLL, 

907) -> tuple[ 

908 ctypes._FuncPointer, 

909 ctypes._FuncPointer, 

910 ctypes._FuncPointer, 

911 ctypes._FuncPointer, 

912 ctypes._FuncPointer, 

913]: 

914 """Look up the five Vulkan symbols this probe needs and stamp argtypes. 

915 

916 All argtypes / restypes are set here so ctypes uses the same 

917 calling convention as the C ABI; missing this on Windows produces 

918 silent stack corruption. 

919 """ 

920 create_instance = lib.vkCreateInstance 

921 create_instance.argtypes = [ 

922 POINTER(_VkInstanceCreateInfo), 

923 c_void_p, 

924 POINTER(c_void_p), 

925 ] 

926 create_instance.restype = c_uint32 

927 

928 destroy_instance = lib.vkDestroyInstance 

929 destroy_instance.argtypes = [c_void_p, c_void_p] 

930 destroy_instance.restype = None 

931 

932 enum_physical = lib.vkEnumeratePhysicalDevices 

933 enum_physical.argtypes = [c_void_p, POINTER(c_uint32), POINTER(c_void_p)] 

934 enum_physical.restype = c_uint32 

935 

936 get_properties = lib.vkGetPhysicalDeviceProperties 

937 get_properties.argtypes = [c_void_p, POINTER(_VkPhysicalDeviceProperties)] 

938 get_properties.restype = None 

939 

940 get_memory = lib.vkGetPhysicalDeviceMemoryProperties 

941 get_memory.argtypes = [c_void_p, POINTER(_VkPhysicalDeviceMemoryProperties)] 

942 get_memory.restype = None 

943 

944 return create_instance, destroy_instance, enum_physical, get_properties, get_memory 

945 

946 

947def _device_local_vram(mem_props: _VkPhysicalDeviceMemoryProperties) -> int: