Coverage for src/lilbee/catalog/download.py: 100%
285 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-11-06 08:17 +0000
1"""GGUF download, mmproj resolution, post-download hooks."""
3import fnmatch
4import logging
5import shutil
6import time
7from collections.abc import Callable, Iterable
8from http import HTTPStatus
9from pathlib import Path
10from typing import Any, NamedTuple, TypeVar
12import httpx
13import httpx2
14from pydantic import BaseModel
16from lilbee.catalog.compat import UnsupportedQuantError, classify, file_header
17from lilbee.catalog.download_progress import ProgressCallback, _ProgressTracker
18from lilbee.catalog.hf_client import (
19 DEFAULT_TIMEOUT,
20 HF_API_URL,
21 hf_headers,
22 hf_token,
23 repo_has_mmproj,
24)
25from lilbee.catalog.models import CatalogModel
26from lilbee.catalog.refs import (
27 DEFAULT_MMPROJ_PATTERN,
28 FLOAT_QUANTS,
29 WILDCARD,
30 quant_label,
31 rank_gguf_candidates,
32 split_shard_filenames,
33)
34from lilbee.catalog.types import ModelCompat, ModelTask
35from lilbee.runtime.cancellation import CancelSignal, TaskCancelledError
37_T = TypeVar("_T")
39CompleteCallback = Callable[[CatalogModel, Path], None]
40# Raises UnsupportedQuantError when the engine cannot decode the named file.
41LoadCheck = Callable[[str, str], None]
42# Called once per file resolved with the Hub, before any bytes transfer, with
43# the cache blob that file occupies (None when the Hub reports no blob).
44ProbeCallback = Callable[[str | None], None]
46log = logging.getLogger(__name__)
49class RemoteFile(NamedTuple):
50 """The Hub's answer about one repo file.
52 *blob* is the name the file takes in the cache's blob directory, and it is
53 None when the Hub reports no entity tag for the file.
54 """
56 size: int
57 blob: str | None
60def _models_dir() -> Path:
61 """Deferred cfg read: a module-level cfg import is circular via Config()'s
62 model-ref validator (config -> model_ref -> catalog -> here -> config)."""
63 from lilbee.core.config.model import cfg
65 return cfg.models_dir
68class DownloadConfig(BaseModel):
69 model_config = {"arbitrary_types_allowed": True}
71 repo_id: str
72 filename: str
73 token: str | None
74 force_download: bool = False
75 cache_dir: str | None = None
76 tqdm_class: Any = None
79_BYTES_PER_GB = 1024**3
82def _free_bytes(path: Path) -> int | None:
83 """Free space on the volume that will hold *path*, which need not exist yet.
85 Measured at the nearest existing ancestor, since shutil.disk_usage raises on
86 a missing path.
87 """
88 probe = path.resolve()
89 while True:
90 try:
91 return shutil.disk_usage(probe).free
92 except OSError:
93 if probe.parent == probe:
94 return None
95 probe = probe.parent
98def disk_shortfall(models_dir: Path, hf_repo: str, needed: int) -> str | None:
99 """Describe why *needed* bytes will not fit, or None when they will.
101 A partial blob from an interrupted attempt is not counted: huggingface_hub
102 writes each transfer to a fresh temporary file, so those bytes are spent.
103 """
104 if needed == _SIZE_UNKNOWN:
105 return None # offline or unresolvable; nothing to compare against
106 free = _free_bytes(models_dir)
107 if free is None:
108 return None # unmeasurable volume; let the download report the truth
109 if needed <= free:
110 return None
111 return (
112 f"Not enough disk space for {hf_repo}: needs "
113 f"{needed / _BYTES_PER_GB:.1f} GB, {free / _BYTES_PER_GB:.1f} GB free."
114 )
117def discard_partial_blobs(models_dir: Path, hf_repo: str, blobs: Iterable[str]) -> None:
118 """Delete the temporary files of *blobs*, which no later download reads.
120 huggingface_hub writes every transfer to a fresh ``<blob>.<unique>.incomplete``
121 name and unlinks it on the way out, so a leftover belongs to an attempt that
122 never unwound: a terminated child, a power loss, a build that predates this
123 sweep. Left alone the bytes are lost for the life of the cache.
125 Scoped to the blobs the caller resolved, because another quant of the same
126 repo downloads into the same directory at the same time and its temporary
127 file is live.
128 """
129 from huggingface_hub.file_download import repo_folder_name
131 repo_dir = models_dir / repo_folder_name(repo_id=hf_repo, repo_type="model")
132 for blob in blobs:
133 for partial in repo_dir.glob(f"blobs/{blob}.*.incomplete"):
134 try:
135 partial.unlink()
136 except OSError:
137 log.warning("Left a partial download behind: %s", partial)
140def _require_disk_space(entry: CatalogModel, models_dir: Path, needed: int) -> None:
141 """Refuse a download the disk cannot hold, naming the shortfall.
143 huggingface_hub only warns, and the xet path reports a full disk as a
144 reconstruction error naming neither the disk nor the file.
145 """
146 message = disk_shortfall(models_dir, entry.hf_repo, needed)
147 if message is not None:
148 raise RuntimeError(message)
151_LOW_DISK_FLOOR = 512 * 1024**2
152"""Free bytes below which a failed download is reported as a full disk.
154Catches a volume that filled mid-transfer, which the pre-flight cannot see."""
157def _raise_if_disk_exhausted(
158 entry: CatalogModel, config: DownloadConfig, cause: BaseException
159) -> None:
160 """Re-raise a failed download as a disk problem when the volume is full.
162 Low free space is a heuristic, not a diagnosis, so *cause* stays in the
163 message.
164 """
165 if config.cache_dir is None:
166 return
167 try:
168 free = shutil.disk_usage(config.cache_dir).free
169 except OSError:
170 return # the path went away with the failure; leave the original error
171 if free >= _LOW_DISK_FLOOR:
172 return
173 raise RuntimeError(
174 f"Ran out of disk space downloading {entry.hf_repo}: "
175 f"{free / _BYTES_PER_GB:.1f} GB free. {type(cause).__name__}: {cause}"
176 ) from None
179_TRANSFER_RETRIES = 2
180_RETRY_BACKOFF_SECONDS = 5
183class _TransientDownloadError(RuntimeError):
184 """A transfer fault another attempt can clear: a network or an I/O error."""
187def _retry_transient(work: Callable[[], _T]) -> _T:
188 """Run *work*, retrying only the faults another attempt can clear.
190 Every other error propagates on the first attempt, cancellation included,
191 so a defect or a configuration error is never hidden behind a retry. A
192 transfer that goes quiet raises nothing here, so it is the parent process
193 that ends it, not this loop.
194 """
195 last_error: Exception | None = None
196 for attempt in range(_TRANSFER_RETRIES + 1):
197 try:
198 return work()
199 except _TransientDownloadError as exc:
200 last_error = exc
201 log.warning("%s (attempt %d/%d)", exc, attempt + 1, _TRANSFER_RETRIES + 1)
202 if attempt < _TRANSFER_RETRIES:
203 time.sleep(_RETRY_BACKOFF_SECONDS * (attempt + 1))
204 raise RuntimeError(
205 f"{last_error}. The transfer failed {_TRANSFER_RETRIES + 1} times. Check the "
206 "network connection and retry; the files that finished are kept."
207 ) from last_error
210def _download_with_retry(entry: CatalogModel, config: DownloadConfig) -> Path:
211 """Run one file's transfer, retrying the faults another attempt can clear."""
212 return _retry_transient(lambda: _hf_download_or_translate(entry, config))
215def _hf_download_or_translate(entry: CatalogModel, config: DownloadConfig) -> Path:
216 """Run the HF download and translate every error class into a clean exception."""
217 from huggingface_hub import hf_hub_download
218 from huggingface_hub.utils import EntryNotFoundError, GatedRepoError, RepositoryNotFoundError
220 try:
221 return Path(hf_hub_download(**config.model_dump(exclude_none=True)))
222 except TaskCancelledError:
223 raise
224 except GatedRepoError:
225 raise PermissionError(
226 f"{entry.hf_repo} requires HuggingFace authentication. "
227 "Set HF_TOKEN env var or visit the repo page to request access."
228 ) from None
229 except RepositoryNotFoundError:
230 raise RuntimeError(f"Repository {entry.hf_repo!r} not found on HuggingFace.") from None
231 except EntryNotFoundError:
232 raise RuntimeError(_missing_file_message(entry.hf_repo, config.filename)) from None
233 except httpx2.TransportError as exc:
234 raise _TransientDownloadError(f"Network error downloading {entry.hf_repo}: {exc}") from None
235 except OSError as exc:
236 raise _TransientDownloadError(f"I/O error downloading {entry.hf_repo}: {exc}") from None
237 except Exception as exc:
238 _raise_if_disk_exhausted(entry, config, exc)
239 raise RuntimeError(
240 f"Failed to download {entry.hf_repo}: {type(exc).__name__}: {exc}"
241 ) from None
244class _NeverCancelled:
245 """Cancel signal for a caller that has no way to stop the download."""
247 def is_set(self) -> bool:
248 """Always False; nothing holds a handle that could set this."""
249 return False
252_NEVER_CANCELLED = _NeverCancelled()
255def download_model(
256 entry: CatalogModel,
257 *,
258 on_progress: ProgressCallback | None = None,
259 on_complete: CompleteCallback | None = None,
260 cancel: CancelSignal | None = None,
261) -> Path:
262 """Download a GGUF model from HuggingFace to the models dir.
263 Uses huggingface_hub for caching and auth; a stalled file starts again
264 from the top, and files already finished are kept.
265 The optional *on_progress(downloaded, total)* callback receives byte counts.
266 The optional *on_complete(entry, file_path)* callback runs after every file
267 is on disk; modelhub uses it to write a registry manifest. For vision
268 models, also downloads the mmproj (CLIP projection) file.
270 The transfer always runs in its own child process, because terminating that
271 process is the only stop a wedged transfer cannot refuse. A caller that
272 passes no *cancel* signal gets one that is never set, so the watchdog that
273 ends a quiet child covers every download.
275 A split GGUF has every shard fetched before the model is finalized, so the
276 registry manifest (and thus "installed") only lands once the full set is on
277 disk; an interrupted multi-part pull leaves the model not-installed and
278 re-pullable rather than registered-but-unloadable.
280 Raises:
281 PermissionError: gated repo requiring authentication
282 RuntimeError: repo not found or download failure with details
283 TaskCancelledError: the cancel signal was set
284 """
285 models_dir = _models_dir()
286 models_dir.mkdir(parents=True, exist_ok=True)
287 token = hf_token()
288 # circular: download -> download_process via fetch_model_files
289 from lilbee.catalog.download_process import download_in_subprocess
291 dest = download_in_subprocess(
292 entry,
293 models_dir,
294 token,
295 on_progress=on_progress,
296 cancel=_NEVER_CANCELLED if cancel is None else cancel,
297 )
298 if on_complete is not None:
299 on_complete(entry, dest)
300 return dest
303def fetch_model_files(
304 entry: CatalogModel,
305 models_dir: Path,
306 token: str | None,
307 *,
308 on_progress: ProgressCallback | None = None,
309 on_probe: ProbeCallback | None = None,
310) -> Path:
311 """Fetch *entry*'s GGUF shards, plus its projector when the repo ships one.
313 Takes the models dir and token as arguments so a download child process
314 can run it without reading cfg. Writes no registry state. *on_probe* fires
315 once per file resolved with the Hub, which is the only sign of life a
316 caller gets before the first bytes, and it names the cache blob so the
317 caller can clear that file's leftovers.
318 """
319 filename = resolve_filename(entry)
320 shards = split_shard_filenames(filename)
321 paths = _cached_shards(entry, models_dir, shards, on_probe)
322 if paths is not None:
323 dest = paths[0]
324 log.info("Model already downloaded: %s", dest)
325 if on_progress is not None:
326 size = sum(path.stat().st_size for path in paths)
327 on_progress(size, size) # Report 100% immediately (every shard)
328 _ensure_projector(entry, models_dir, token, on_progress=on_progress, on_probe=on_probe)
329 return dest
331 remote = [_probed_shard(entry, shard, on_probe) for shard in shards]
332 shard_sizes = [file.size for file in remote]
333 sizes_known = all(size != _SIZE_UNKNOWN for size in shard_sizes)
334 # Reclaim first: a leftover partial is unreadable bytes that still occupy
335 # the volume the space check is about to measure.
336 discard_partial_blobs(models_dir, entry.hf_repo, _blobs(remote))
337 _require_disk_space(entry, models_dir, sum(shard_sizes) if sizes_known else 0)
339 # Sum the shard sizes up front so a multi-shard pull reports one monotonic
340 # 0->100% against the real total, not N separate per-shard cycles. Only use
341 # the sum when every shard size is known (0 = unresolved/offline); a partial
342 # sum would undercount the total and let progress run past 100%.
343 grand_total = sum(shard_sizes) if len(shards) > 1 and sizes_known else 0
344 tracker = _ProgressTracker(on_progress, grand_total=grand_total) if on_progress else None
345 shard_paths: list[Path] = []
346 for shard in shards:
347 log.info("Downloading %s/%s → %s", entry.hf_repo, shard, models_dir)
348 config = DownloadConfig(
349 repo_id=entry.hf_repo,
350 filename=shard,
351 token=token,
352 cache_dir=str(models_dir),
353 tqdm_class=tracker.make_tqdm_class() if tracker else None,
354 )
355 shard_path = _download_with_retry(entry, config)
356 shard_paths.append(shard_path)
357 if tracker is not None:
358 tracker.shard_done(shard_path.stat().st_size)
359 first_shard_path = shard_paths[0] # the 00001-of-N shard llama.cpp loads from
361 if on_progress:
362 total_size = sum(path.stat().st_size for path in shard_paths)
363 if not tracker or not tracker.was_used:
364 log.info("Model found in HuggingFace cache: %s", first_shard_path)
365 on_progress(total_size, total_size)
366 _ensure_projector(entry, models_dir, token, on_progress=on_progress, on_probe=on_probe)
367 return first_shard_path
370def _blobs(files: Iterable[RemoteFile]) -> list[str]:
371 """The cache blob names among *files*, dropping the ones the Hub did not report."""
372 return [file.blob for file in files if file.blob is not None]
375def _probe(on_probe: ProbeCallback | None, file: RemoteFile) -> RemoteFile:
376 """Report *file* as resolved, then hand it back to the caller."""
377 if on_probe is not None:
378 on_probe(file.blob)
379 return file
382def _cached_shard(
383 entry: CatalogModel, models_dir: Path, shard: str, on_probe: ProbeCallback | None
384) -> Path | None:
385 """The on-disk copy of *shard* at the size the Hub reports for it, or None.
387 Looks in the Hub cache layout first, then at the flat path an older
388 install used.
389 """
390 from huggingface_hub import try_to_load_from_cache
392 file = _probe(on_probe, fetch_remote_file(entry.hf_repo, shard))
393 hit = try_to_load_from_cache(entry.hf_repo, shard, cache_dir=str(models_dir))
394 for path in (Path(hit) if isinstance(hit, str) else None, models_dir / shard):
395 if path is not None and path.exists() and _size_matches(path, file.size):
396 return path
397 return None
400def _cached_shards(
401 entry: CatalogModel, models_dir: Path, shards: list[str], on_probe: ProbeCallback | None
402) -> list[Path] | None:
403 """Every shard's on-disk copy, or None at the first one that is missing."""
404 paths: list[Path] = []
405 for shard in shards:
406 path = _cached_shard(entry, models_dir, shard, on_probe)
407 if path is None:
408 return None
409 paths.append(path)
410 return paths
413def _probed_shard(entry: CatalogModel, shard: str, on_probe: ProbeCallback | None) -> RemoteFile:
414 """The Hub's answer about *shard*, reported to *on_probe* as it arrives."""
415 return _probe(on_probe, fetch_remote_file(entry.hf_repo, shard))
418def _ensure_projector(
419 entry: CatalogModel,
420 models_dir: Path,
421 token: str | None,
422 *,
423 on_progress: ProgressCallback | None = None,
424 on_probe: ProbeCallback | None = None,
425) -> None:
426 """Fetch the projector whenever the repo ships one, not only for VISION entries.
428 Dual-use VL repos (Qwen-VL, InternVL, gemma-3) classify as chat by
429 name and arch, and without their projector the vision role dies at plan
430 time with a missing-mmproj warning a re-pull cannot cure.
431 """
432 if entry.task == ModelTask.VISION or repo_has_mmproj(entry.hf_repo):
433 _fetch_mmproj(entry, models_dir, token, on_progress=on_progress, on_probe=on_probe)
436def download_mmproj(
437 entry: CatalogModel,
438 *,
439 on_progress: ProgressCallback | None = None,
440) -> Path | None:
441 """Download the mmproj (CLIP projection) file for a vision model.
442 Returns the path to the downloaded file, or None if no mmproj is configured.
443 The optional ``on_progress`` callback receives ``(downloaded, total)`` byte
444 counts and is wired through the same tqdm hook used by the main download.
445 """
446 return _fetch_mmproj(entry, _models_dir(), hf_token(), on_progress=on_progress)
449def _fetch_mmproj(
450 entry: CatalogModel,
451 models_dir: Path,
452 token: str | None,
453 *,
454 on_progress: ProgressCallback | None = None,
455 on_probe: ProbeCallback | None = None,
456) -> Path | None:
457 """Fetch *entry*'s mmproj into *models_dir*, or None when the repo names none."""
458 mmproj_filename = _resolve_mmproj_filename(entry.hf_repo, DEFAULT_MMPROJ_PATTERN)
459 if not mmproj_filename:
460 log.warning("Could not resolve mmproj file for %s", entry.hf_repo)
461 return None
463 tracker = _ProgressTracker(on_progress) if on_progress else None
464 log.info("Downloading mmproj %s/%s → %s", entry.hf_repo, mmproj_filename, models_dir)
465 projector = _probe(on_probe, fetch_remote_file(entry.hf_repo, mmproj_filename))
466 discard_partial_blobs(models_dir, entry.hf_repo, _blobs([projector]))
467 _require_disk_space(entry, models_dir, projector.size)
468 # The projector gets the same error translation and retry as the GGUF.
469 path = _download_with_retry(
470 entry,
471 DownloadConfig(
472 repo_id=entry.hf_repo,
473 filename=mmproj_filename,
474 token=token,
475 cache_dir=str(models_dir),
476 tqdm_class=tracker.make_tqdm_class() if tracker else None,
477 ),
478 )
479 if on_progress is not None and (not tracker or not tracker.was_used):
480 # Cache hit: HF returned the cached path without invoking tqdm.
481 size = path.stat().st_size
482 on_progress(size, size)
483 return path
486def _repo_sibling_files(hf_repo: str) -> list[str]:
487 """Every filename the HuggingFace API lists for *hf_repo*.
489 Raises:
490 PermissionError: the repo is gated and needs authentication.
491 RuntimeError: the listing could not be fetched.
492 """
493 try:
494 resp = httpx.get(
495 f"{HF_API_URL}/{hf_repo}",
496 timeout=DEFAULT_TIMEOUT,
497 headers=hf_headers(),
498 )
499 if resp.status_code == HTTPStatus.UNAUTHORIZED:
500 raise PermissionError(
501 f"{hf_repo} requires HuggingFace authentication. "
502 "Set HF_TOKEN env var or visit the repo page to request access."
503 )
504 resp.raise_for_status()
505 siblings = resp.json().get("siblings", [])
506 except PermissionError:
507 raise
508 except Exception as exc:
509 raise RuntimeError(f"Cannot query files for {hf_repo}: {exc}") from exc
510 return [s.get("rfilename", "") for s in siblings]
513def _mmproj_rank(filename: str) -> tuple[bool, str]:
514 """Sort key preferring an unquantized projector, ties broken by name."""
515 return (quant_label(filename) not in FLOAT_QUANTS, filename)
518def _resolve_mmproj_filename(hf_repo: str, pattern: str) -> str | None:
519 """Resolve an mmproj filename pattern to a concrete filename via the HF API."""
520 if WILDCARD not in pattern:
521 return pattern
522 try:
523 names = _repo_sibling_files(hf_repo)
524 except (PermissionError, RuntimeError) as exc:
525 log.warning("Cannot query mmproj files for %s: %s", hf_repo, exc)
526 return None
527 matches = [name for name in names if fnmatch.fnmatch(name, pattern)]
528 return min(matches, key=_mmproj_rank) if matches else None
531def resolve_filename(entry: CatalogModel, *, can_load: LoadCheck | None = None) -> str:
532 """The repo file a pull of *entry* fetches, gated on each candidate's GGUF header.
534 Quant labels only order the candidates. A file whose header calls it a
535 projector or an adapter is never the model, so a repo that labels its
536 projector ``Q8_0`` cannot install it as one.
538 *can_load* is the engine's verdict on one file, supplied by the layer that
539 owns the engine. It runs last because it is the expensive question, and it
540 decides between candidates rather than judging the one already chosen: a
541 repo publishing the same weights in several packings holds files the engine
542 reads and files it cannot, and only one of them is worth downloading.
544 Where no architecture is supported the best-ranked weights still come back.
545 Refusing here would report a generic error and disable ``--allow-unsupported``,
546 so that verdict belongs to the architecture guard.
548 Raises:
549 PermissionError: the repo is gated and needs authentication.
550 UnsupportedQuantError: every candidate carries weights the engine cannot
551 decode; the first such refusal is re-raised, naming its file.
552 RuntimeError: the repo listing failed, or it holds no model weights.
553 """
554 named = entry.gguf_filename
555 if WILDCARD not in named and file_header(entry.hf_repo, named).is_model:
556 return named
557 unsupported: str | None = None
558 refused: UnsupportedQuantError | None = None
559 for candidate in rank_gguf_candidates(_repo_sibling_files(entry.hf_repo)):
560 header = file_header(entry.hf_repo, candidate)
561 if not header.is_model:
562 continue
563 if classify(header.architecture) is ModelCompat.UNSUPPORTED:
564 unsupported = unsupported or candidate
565 continue
566 if can_load is not None:
567 try:
568 can_load(entry.hf_repo, candidate)
569 except UnsupportedQuantError as exc:
570 refused = refused or exc
571 continue
572 return candidate
573 if unsupported is not None:
574 return unsupported
575 if refused is not None:
576 raise refused
577 raise RuntimeError(f"No GGUF model weights found in {entry.hf_repo}")
580_SIZE_UNKNOWN = 0
583def _size_matches(dest: Path, expected: int) -> bool:
584 """Whether *dest* holds the *expected* byte count, accepting an unknown one.
586 An unknown size is accepted because there is nothing to verify against and
587 refusing would block every offline reuse. A size that disagrees is a
588 truncated or corrupt file, so the caller fetches it again.
589 """
590 if expected == _SIZE_UNKNOWN:
591 return True
592 actual = dest.stat().st_size
593 if actual == expected:
594 return True
595 log.warning(
596 "Cached %s is %d bytes but HuggingFace reports %d; re-downloading",
597 dest,
598 actual,
599 expected,
600 )
601 return False
604def _hf_file_metadata(hf_repo: str, filename: str) -> tuple[int | None, str | None]:
605 """Byte size and cache blob huggingface_hub resolves for *filename*."""
606 from huggingface_hub import get_hf_file_metadata, hf_hub_url
608 metadata = get_hf_file_metadata(hf_hub_url(hf_repo, filename), token=hf_token())
609 return metadata.size, metadata.etag
612def _missing_file_message(hf_repo: str, filename: str) -> str:
613 """User-facing error for a file the Hub reports as nonexistent."""
614 return (
615 f"File {filename!r} does not exist in {hf_repo} on HuggingFace. "
616 "Check the filename on the repo page."
617 )
620def fetch_remote_file(hf_repo: str, filename: str) -> RemoteFile:
621 """Return the size and cache blob huggingface_hub reports for *filename*.
623 Resolves via hf_hub's own file metadata (correct revision, redirects, and
624 LFS/Xet handled uniformly) instead of scraping the repo tree. Reports an
625 unknown size when offline or unresolvable, in which case the caller keeps
626 the cached file. A file the Hub reports as nonexistent raises instead: that
627 answer is definitive, and treating it as unknown let a pull of a mistyped
628 filename accept a stale local file and report success without downloading
629 anything.
630 """
631 from huggingface_hub.errors import RemoteEntryNotFoundError
633 try:
634 size, blob = _hf_file_metadata(hf_repo, filename)
635 except RemoteEntryNotFoundError:
636 raise RuntimeError(_missing_file_message(hf_repo, filename)) from None
637 except Exception:
638 return RemoteFile(size=_SIZE_UNKNOWN, blob=None)
639 return RemoteFile(size=size or _SIZE_UNKNOWN, blob=blob)
642def download_bytes(hf_repo: str, filename: str) -> int:
643 """Bytes a pull of *filename* fetches, every shard summed, or 0 when unknown.
645 The exact figure HuggingFace reports, not the catalog row's approximation:
646 a disk check refuses a real download, so it asks about the real file. A
647 single unresolvable shard makes the sum unknown rather than short.
648 """
649 sizes = [fetch_remote_file(hf_repo, shard).size for shard in split_shard_filenames(filename)]
650 return sum(sizes) if all(sizes) else _SIZE_UNKNOWN