Coverage for src/lilbee/catalog/download.py: 100%
328 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-09-04 17:08 +0000
1"""GGUF download, mmproj resolution, post-download hooks."""
3import fnmatch
4import logging
5import os
6import shutil
7import sys
8import threading
9import time
10from collections.abc import Callable
11from http import HTTPStatus
12from pathlib import Path
13from typing import Any
15import httpx
16from pydantic import BaseModel
18from lilbee.catalog.compat import UnsupportedQuantError, classify, file_header
19from lilbee.catalog.download_progress import ProgressCallback, _ProgressTracker
20from lilbee.catalog.hf_client import (
21 DEFAULT_TIMEOUT,
22 HF_API_URL,
23 hf_headers,
24 hf_token,
25 repo_has_mmproj,
26)
27from lilbee.catalog.models import CatalogModel
28from lilbee.catalog.refs import (
29 DEFAULT_MMPROJ_PATTERN,
30 FLOAT_QUANTS,
31 WILDCARD,
32 quant_label,
33 rank_gguf_candidates,
34 split_shard_filenames,
35)
36from lilbee.catalog.types import ModelCompat, ModelTask
37from lilbee.runtime.cancellation import CancelSignal, TaskCancelledError
39CompleteCallback = Callable[[CatalogModel, Path], None]
40# Raises UnsupportedQuantError when the engine cannot decode the named file.
41LoadCheck = Callable[[str, str], None]
43log = logging.getLogger(__name__)
46def _models_dir() -> Path:
47 """Deferred cfg read: a module-level cfg import is circular via Config()'s
48 model-ref validator (config -> model_ref -> catalog -> here -> config)."""
49 from lilbee.core.config.model import cfg
51 return cfg.models_dir
54class DownloadConfig(BaseModel):
55 model_config = {"arbitrary_types_allowed": True}
57 repo_id: str
58 filename: str
59 token: str | None
60 force_download: bool = False
61 cache_dir: str | None = None
62 tqdm_class: Any = None
65_BYTES_PER_GB = 1024**3
68def _repo_partial_bytes(models_dir: Path, hf_repo: str) -> int:
69 """Bytes an interrupted attempt at *hf_repo* already holds on disk.
71 A resume needs only the remainder, so these count toward available space.
72 """
73 from huggingface_hub.file_download import repo_folder_name
75 repo_dir = models_dir / repo_folder_name(repo_id=hf_repo, repo_type="model")
76 if not repo_dir.is_dir():
77 return 0
78 return sum(f.stat().st_size for f in repo_dir.glob("blobs/*.incomplete") if f.is_file())
81def _free_bytes(path: Path) -> int | None:
82 """Free space on the volume that will hold *path*, which need not exist yet.
84 Measured at the nearest existing ancestor, since shutil.disk_usage raises on
85 a missing path.
86 """
87 probe = path.resolve()
88 while True:
89 try:
90 return shutil.disk_usage(probe).free
91 except OSError:
92 if probe.parent == probe:
93 return None
94 probe = probe.parent
97def disk_shortfall(models_dir: Path, hf_repo: str, needed: int) -> str | None:
98 """Describe why *needed* bytes will not fit, or None when they will."""
99 if needed == _SIZE_UNKNOWN:
100 return None # offline or unresolvable; nothing to compare against
101 free = _free_bytes(models_dir)
102 if free is None:
103 return None # unmeasurable volume; let the download report the truth
104 available = free + _repo_partial_bytes(models_dir, hf_repo)
105 if needed <= available:
106 return None
107 return (
108 f"Not enough disk space for {hf_repo}: needs "
109 f"{needed / _BYTES_PER_GB:.1f} GB, {available / _BYTES_PER_GB:.1f} GB free."
110 )
113def _require_disk_space(entry: CatalogModel, models_dir: Path, needed: int) -> None:
114 """Refuse a download the disk cannot hold, naming the shortfall.
116 huggingface_hub only warns, and the xet path reports a full disk as a
117 reconstruction error naming neither the disk nor the file.
118 """
119 message = disk_shortfall(models_dir, entry.hf_repo, needed)
120 if message is not None:
121 raise RuntimeError(message)
124_LOW_DISK_FLOOR = 512 * 1024**2
125"""Free bytes below which a failed download is reported as a full disk.
127Catches a volume that filled mid-transfer, which the pre-flight cannot see."""
130def _raise_if_disk_exhausted(
131 entry: CatalogModel, config: DownloadConfig, cause: BaseException
132) -> None:
133 """Re-raise a failed download as a disk problem when the volume is full.
135 Low free space is a heuristic, not a diagnosis, so *cause* stays in the
136 message.
137 """
138 if config.cache_dir is None:
139 return
140 try:
141 free = shutil.disk_usage(config.cache_dir).free
142 except OSError:
143 return # the path went away with the failure; leave the original error
144 if free >= _LOW_DISK_FLOOR:
145 return
146 raise RuntimeError(
147 f"Ran out of disk space downloading {entry.hf_repo}: "
148 f"{free / _BYTES_PER_GB:.1f} GB free. {type(cause).__name__}: {cause}"
149 ) from None
152_XET_HIGH_PERFORMANCE_ENV = "HF_XET_HIGH_PERFORMANCE"
154_XET_DISABLE_ENV = "HF_HUB_DISABLE_XET"
157def _disable_xet_where_it_stalls() -> None:
158 """Fall back to the plain HTTP download path on Windows.
160 hf_xet transfers stall or deadlock on Windows (xet-core issues #446,
161 #789, #850), while the plain path downloads at line speed. Everywhere
162 else xet stays on deliberately: it is the fast path. A user who
163 exported the variable keeps whatever they chose. huggingface_hub
164 parses the variable once at import, so the hub constant must change
165 too; the environment write covers worker subprocesses, which parse it
166 fresh.
167 """
168 if sys.platform != "win32":
169 return
170 if _XET_DISABLE_ENV in os.environ:
171 return
172 from huggingface_hub import constants
174 os.environ[_XET_DISABLE_ENV] = "1"
175 constants.HF_HUB_DISABLE_XET = True
178def _apply_fast_download_mode() -> None:
179 """Publish the high-performance setting to xet before it builds a session.
181 hf_xet reads it from the environment in Rust and caches it when the session
182 is built, so a change lands on restart.
183 """
184 # circular: catalog.download -> core.config via cfg, the same cycle
185 # _models_dir documents (config -> model_ref -> catalog -> here).
186 from lilbee.core.config.model import cfg
188 if cfg.fast_model_downloads:
189 os.environ[_XET_HIGH_PERFORMANCE_ENV] = "1"
190 else:
191 os.environ.pop(_XET_HIGH_PERFORMANCE_ENV, None)
194_STALL_WINDOW_S = 60.0
195"""Seconds per measurement window; a transfer below the byte floor for a
196whole window counts as stalled.
198Well past the hub's own 10s read timeout and its resume retries, so the
199guard only fires on transfers those mechanisms cannot wake."""
201_STALL_FLOOR_BYTES = 256 * 1024
202"""Minimum bytes per window for a transfer to count as alive.
204A wedged connection can trickle a few bytes a minute, which an any-activity
205check reads as progress; ~4 KB/s is far below any usable model download."""
207_STALL_POLL_S = 5.0
209_STALL_RETRIES = 2
212def _abort_stalled_transfer() -> None:
213 """Break a wedged transfer so the blocked download thread raises.
215 Covers both transports: the xet session abort stops a deadlocked Rust
216 transfer (a no-op without one), and closing the hub's shared client
217 closes the plain path's socket under its blocked read. The next hub
218 call builds a fresh client. The session abort is safe here because a
219 process runs at most one download; concurrent downloads each run in
220 their own child process.
221 """
222 from huggingface_hub.utils._http import close_session
223 from huggingface_hub.utils._xet import abort_xet_session
225 abort_xet_session()
226 close_session()
229class _StallGuard:
230 """Aborts a transfer that reports no bytes for the stall window.
232 A wedged transfer blocks forever with the task showing active: hf_xet
233 can deadlock before its first byte, a dead socket never wakes the
234 plain path's read, and a dying connection can trickle bytes too slowly
235 to ever finish. The guard rides the same progress stream the task bar
236 shows; when a window passes under the byte floor, the abort makes the
237 blocked thread raise, and the caller resumes from the .incomplete file.
238 """
240 def __init__(
241 self,
242 window_s: float = _STALL_WINDOW_S,
243 poll_s: float = _STALL_POLL_S,
244 floor_bytes: int = _STALL_FLOOR_BYTES,
245 ) -> None:
246 self._window_s = window_s
247 self._poll_s = poll_s
248 self._floor_bytes = floor_bytes
249 self._window_start = time.monotonic()
250 self._window_bytes = 0
251 self._stop = threading.Event()
252 self._thread: threading.Thread | None = None
253 self.fired = False
255 def pulse(self, n: float = 0) -> None:
256 """Count transferred bytes; called from the download thread's tqdm."""
257 self._window_bytes += int(n)
259 def wrap_tqdm(self, tqdm_class: Any) -> Any:
260 """Subclass *tqdm_class* (or the hub's default) to pulse on every update.
262 ``update_transfer`` is only defined when the base has it: the hub
263 feature-detects the method, so adding it to a base that lacks it
264 would advertise a stream the base cannot aggregate.
265 """
266 from huggingface_hub.utils.tqdm import tqdm as hub_tqdm
268 guard = self
269 base = tqdm_class if tqdm_class is not None else hub_tqdm
271 class _Pulsing(base): # type: ignore[misc, valid-type]
272 def update(self, n: float = 1) -> bool | None:
273 guard.pulse(n)
274 super().update(n)
275 return None
277 if not hasattr(base, "update_transfer"):
278 return _Pulsing
280 class _PulsingTransfer(_Pulsing):
281 def update_transfer(self, n: float = 1) -> bool | None:
282 guard.pulse(n)
283 super().update_transfer(n)
284 return None
286 return _PulsingTransfer
288 def _watch(self) -> None:
289 while not self._stop.wait(self._poll_s):
290 if not self._keep_watching():
291 return
293 def _keep_watching(self) -> bool:
294 """One tick: True to keep watching, False once fired or stopped."""
295 now = time.monotonic()
296 if now - self._window_start < self._window_s:
297 return True
298 if self._stop.is_set():
299 return False # the transfer finished while this tick was deciding
300 if self._window_bytes >= self._floor_bytes:
301 self._window_start = now
302 self._window_bytes = 0
303 return True
304 self.fired = True
305 _abort_stalled_transfer()
306 return False
308 def __enter__(self) -> "_StallGuard":
309 self._thread = threading.Thread(
310 target=self._watch, name="download-stall-guard", daemon=True
311 )
312 self._thread.start()
313 return self
315 def __exit__(self, *exc_info: object) -> None:
316 self._stop.set()
317 if self._thread is not None:
318 self._thread.join(timeout=self._poll_s + 1)
321def _download_with_stall_guard(entry: CatalogModel, config: DownloadConfig) -> Path:
322 """Run the transfer under the stall guard, resuming after each stall.
324 huggingface_hub resumes from the .incomplete file, so a retry costs only
325 the bytes since the stall. A failure with the guard quiet is a real
326 error and propagates on the first attempt; cancellation always does.
327 """
328 last_error: Exception | None = None
329 for attempt in range(_STALL_RETRIES + 1):
330 guard = _StallGuard()
331 guarded = config.model_copy(update={"tqdm_class": guard.wrap_tqdm(config.tqdm_class)})
332 try:
333 with guard:
334 return _hf_download_or_translate(entry, guarded)
335 except TaskCancelledError:
336 raise
337 except Exception as exc:
338 if not guard.fired:
339 raise
340 last_error = exc
341 log.warning(
342 "Transfer of %s stalled (attempt %d/%d); resuming.",
343 entry.hf_repo,
344 attempt + 1,
345 _STALL_RETRIES + 1,
346 )
347 raise RuntimeError(
348 f"Download of {entry.hf_repo} stalled {_STALL_RETRIES + 1} times with almost "
349 "no data arriving. Check the network connection and retry; the finished part "
350 "is kept and the download resumes where it stopped."
351 ) from last_error
354def _hf_download_or_translate(entry: CatalogModel, config: DownloadConfig) -> Path:
355 """Run the HF download and translate every error class into a clean exception."""
356 from huggingface_hub import hf_hub_download
357 from huggingface_hub.utils import EntryNotFoundError, GatedRepoError, RepositoryNotFoundError
359 _disable_xet_where_it_stalls()
360 try:
361 return Path(hf_hub_download(**config.model_dump(exclude_none=True)))
362 except TaskCancelledError:
363 raise
364 except GatedRepoError:
365 raise PermissionError(
366 f"{entry.hf_repo} requires HuggingFace authentication. "
367 "Set HF_TOKEN env var or visit the repo page to request access."
368 ) from None
369 except RepositoryNotFoundError:
370 raise RuntimeError(f"Repository {entry.hf_repo!r} not found on HuggingFace.") from None
371 except EntryNotFoundError:
372 raise RuntimeError(_missing_file_message(entry.hf_repo, config.filename)) from None
373 except (httpx.TimeoutException, httpx.ConnectError) as exc:
374 raise RuntimeError(f"Network error downloading {entry.hf_repo}: {exc}") from None
375 except OSError as exc:
376 raise RuntimeError(f"I/O error downloading {entry.hf_repo}: {exc}") from None
377 except Exception as exc:
378 _raise_if_disk_exhausted(entry, config, exc)
379 raise RuntimeError(
380 f"Failed to download {entry.hf_repo}: {type(exc).__name__}: {exc}"
381 ) from None
384def download_model(
385 entry: CatalogModel,
386 *,
387 on_progress: ProgressCallback | None = None,
388 on_complete: CompleteCallback | None = None,
389 cancel: CancelSignal | None = None,
390) -> Path:
391 """Download a GGUF model from HuggingFace to the models dir.
392 Uses huggingface_hub for resumable downloads, caching, and auth.
393 The optional *on_progress(downloaded, total)* callback receives byte counts.
394 The optional *on_complete(entry, file_path)* callback runs after every file
395 is on disk; modelhub uses it to write a registry manifest. For vision
396 models, also downloads the mmproj (CLIP projection) file.
398 With a *cancel* signal the transfer runs in its own child process, and a
399 set signal terminates that process mid-transfer; without one the transfer
400 runs in this process and only ``on_progress`` raising can stop it.
402 A split GGUF has every shard fetched before the model is finalized, so the
403 registry manifest (and thus "installed") only lands once the full set is on
404 disk; an interrupted multi-part pull leaves the model not-installed and
405 re-pullable rather than registered-but-unloadable.
407 Raises:
408 PermissionError: gated repo requiring authentication
409 RuntimeError: repo not found or download failure with details
410 TaskCancelledError: the cancel signal was set
411 """
412 _apply_fast_download_mode()
413 models_dir = _models_dir()
414 models_dir.mkdir(parents=True, exist_ok=True)
415 token = hf_token()
416 if cancel is None:
417 dest = fetch_model_files(entry, models_dir, token, on_progress=on_progress)
418 else:
419 # circular: download -> download_process via fetch_model_files
420 from lilbee.catalog.download_process import download_in_subprocess
422 dest = download_in_subprocess(
423 entry, models_dir, token, on_progress=on_progress, cancel=cancel
424 )
425 if on_complete is not None:
426 on_complete(entry, dest)
427 return dest
430def fetch_model_files(
431 entry: CatalogModel,
432 models_dir: Path,
433 token: str | None,
434 *,
435 on_progress: ProgressCallback | None = None,
436) -> Path:
437 """Fetch *entry*'s GGUF shards, plus its projector when the repo ships one.
439 Takes the models dir and token as arguments so a download child process
440 can run it without reading cfg. Writes no registry state.
441 """
442 filename = resolve_filename(entry)
443 shards = split_shard_filenames(filename)
444 dest = models_dir / shards[0]
445 if all(
446 (models_dir / shard).exists()
447 and _cached_file_is_complete(entry.hf_repo, shard, models_dir / shard)
448 for shard in shards
449 ):
450 log.info("Model already downloaded: %s", dest)
451 if on_progress is not None:
452 size = sum((models_dir / shard).stat().st_size for shard in shards)
453 on_progress(size, size) # Report 100% immediately (every shard)
454 _ensure_projector(entry, models_dir, token, on_progress=on_progress)
455 return dest
457 shard_sizes = [fetch_expected_file_size(entry.hf_repo, shard) for shard in shards]
458 sizes_known = all(size != _SIZE_UNKNOWN for size in shard_sizes)
459 _require_disk_space(entry, models_dir, sum(shard_sizes) if sizes_known else 0)
461 # Sum the shard sizes up front so a multi-shard pull reports one monotonic
462 # 0->100% against the real total, not N separate per-shard cycles. Only use
463 # the sum when every shard size is known (0 = unresolved/offline); a partial
464 # sum would undercount the total and let progress run past 100%.
465 grand_total = sum(shard_sizes) if len(shards) > 1 and sizes_known else 0
466 tracker = _ProgressTracker(on_progress, grand_total=grand_total) if on_progress else None
467 shard_paths: list[Path] = []
468 for shard in shards:
469 log.info("Downloading %s/%s → %s", entry.hf_repo, shard, models_dir)
470 config = DownloadConfig(
471 repo_id=entry.hf_repo,
472 filename=shard,
473 token=token,
474 cache_dir=str(models_dir),
475 tqdm_class=tracker.make_tqdm_class() if tracker else None,
476 )
477 shard_path = _download_with_stall_guard(entry, config)
478 shard_paths.append(shard_path)
479 if tracker is not None:
480 tracker.shard_done(shard_path.stat().st_size)
481 first_shard_path = shard_paths[0] # the 00001-of-N shard llama.cpp loads from
483 if on_progress:
484 total_size = sum(path.stat().st_size for path in shard_paths)
485 if not tracker or not tracker.was_used:
486 log.info("Model found in HuggingFace cache: %s", first_shard_path)
487 on_progress(total_size, total_size)
488 _ensure_projector(entry, models_dir, token, on_progress=on_progress)
489 return first_shard_path
492def _ensure_projector(
493 entry: CatalogModel,
494 models_dir: Path,
495 token: str | None,
496 *,
497 on_progress: ProgressCallback | None = None,
498) -> None:
499 """Fetch the projector whenever the repo ships one, not only for VISION entries.
501 Dual-use VL repos (Qwen-VL, InternVL, SmolVLM, gemma-3) classify as chat by
502 name and arch, and without their projector the vision role dies at plan
503 time with a missing-mmproj warning a re-pull cannot cure.
504 """
505 if entry.task == ModelTask.VISION or repo_has_mmproj(entry.hf_repo):
506 _fetch_mmproj(entry, models_dir, token, on_progress=on_progress)
509def download_mmproj(
510 entry: CatalogModel,
511 *,
512 on_progress: ProgressCallback | None = None,
513) -> Path | None:
514 """Download the mmproj (CLIP projection) file for a vision model.
515 Returns the path to the downloaded file, or None if no mmproj is configured.
516 The optional ``on_progress`` callback receives ``(downloaded, total)`` byte
517 counts and is wired through the same tqdm hook used by the main download.
518 """
519 _apply_fast_download_mode()
520 return _fetch_mmproj(entry, _models_dir(), hf_token(), on_progress=on_progress)
523def _fetch_mmproj(
524 entry: CatalogModel,
525 models_dir: Path,
526 token: str | None,
527 *,
528 on_progress: ProgressCallback | None = None,
529) -> Path | None:
530 """Fetch *entry*'s mmproj into *models_dir*, or None when the repo names none."""
531 mmproj_filename = _resolve_mmproj_filename(entry.hf_repo, DEFAULT_MMPROJ_PATTERN)
532 if not mmproj_filename:
533 log.warning("Could not resolve mmproj file for %s", entry.hf_repo)
534 return None
536 tracker = _ProgressTracker(on_progress) if on_progress else None
537 log.info("Downloading mmproj %s/%s → %s", entry.hf_repo, mmproj_filename, models_dir)
538 _require_disk_space(entry, models_dir, fetch_expected_file_size(entry.hf_repo, mmproj_filename))
539 # The projector gets the same error translation and stall guard as the GGUF.
540 path = _download_with_stall_guard(
541 entry,
542 DownloadConfig(
543 repo_id=entry.hf_repo,
544 filename=mmproj_filename,
545 token=token,
546 cache_dir=str(models_dir),
547 tqdm_class=tracker.make_tqdm_class() if tracker else None,
548 ),
549 )
550 if on_progress is not None and (not tracker or not tracker.was_used):
551 # Cache hit: HF returned the cached path without invoking tqdm.
552 size = path.stat().st_size
553 on_progress(size, size)
554 return path
557def _repo_sibling_files(hf_repo: str) -> list[str]:
558 """Every filename the HuggingFace API lists for *hf_repo*.
560 Raises:
561 PermissionError: the repo is gated and needs authentication.
562 RuntimeError: the listing could not be fetched.
563 """
564 try:
565 resp = httpx.get(
566 f"{HF_API_URL}/{hf_repo}",
567 timeout=DEFAULT_TIMEOUT,
568 headers=hf_headers(),
569 )
570 if resp.status_code == HTTPStatus.UNAUTHORIZED:
571 raise PermissionError(
572 f"{hf_repo} requires HuggingFace authentication. "
573 "Set HF_TOKEN env var or visit the repo page to request access."
574 )
575 resp.raise_for_status()
576 siblings = resp.json().get("siblings", [])
577 except PermissionError:
578 raise
579 except Exception as exc:
580 raise RuntimeError(f"Cannot query files for {hf_repo}: {exc}") from exc
581 return [s.get("rfilename", "") for s in siblings]
584def _mmproj_rank(filename: str) -> tuple[bool, str]:
585 """Sort key preferring an unquantized projector, ties broken by name."""
586 return (quant_label(filename) not in FLOAT_QUANTS, filename)
589def _resolve_mmproj_filename(hf_repo: str, pattern: str) -> str | None:
590 """Resolve an mmproj filename pattern to a concrete filename via the HF API."""
591 if WILDCARD not in pattern:
592 return pattern
593 try:
594 names = _repo_sibling_files(hf_repo)
595 except (PermissionError, RuntimeError) as exc:
596 log.warning("Cannot query mmproj files for %s: %s", hf_repo, exc)
597 return None
598 matches = [name for name in names if fnmatch.fnmatch(name, pattern)]
599 return min(matches, key=_mmproj_rank) if matches else None
602def resolve_filename(entry: CatalogModel, *, can_load: LoadCheck | None = None) -> str:
603 """The repo file a pull of *entry* fetches, gated on each candidate's GGUF header.
605 Quant labels only order the candidates. A file whose header calls it a
606 projector or an adapter is never the model, so a repo that labels its
607 projector ``Q8_0`` cannot install it as one.
609 *can_load* is the engine's verdict on one file, supplied by the layer that
610 owns the engine. It runs last because it is the expensive question, and it
611 decides between candidates rather than judging the one already chosen: a
612 repo publishing the same weights in several packings holds files the engine
613 reads and files it cannot, and only one of them is worth downloading.
615 Where no architecture is supported the best-ranked weights still come back.
616 Refusing here would report a generic error and disable ``--allow-unsupported``,
617 so that verdict belongs to the architecture guard.
619 Raises:
620 PermissionError: the repo is gated and needs authentication.
621 UnsupportedQuantError: every candidate carries weights the engine cannot
622 decode; the first such refusal is re-raised, naming its file.
623 RuntimeError: the repo listing failed, or it holds no model weights.
624 """
625 named = entry.gguf_filename
626 if WILDCARD not in named and file_header(entry.hf_repo, named).is_model:
627 return named
628 unsupported: str | None = None
629 refused: UnsupportedQuantError | None = None
630 for candidate in rank_gguf_candidates(_repo_sibling_files(entry.hf_repo)):
631 header = file_header(entry.hf_repo, candidate)
632 if not header.is_model:
633 continue
634 if classify(header.architecture) is ModelCompat.UNSUPPORTED:
635 unsupported = unsupported or candidate
636 continue
637 if can_load is not None:
638 try:
639 can_load(entry.hf_repo, candidate)
640 except UnsupportedQuantError as exc:
641 refused = refused or exc
642 continue
643 return candidate
644 if unsupported is not None:
645 return unsupported
646 if refused is not None:
647 raise refused
648 raise RuntimeError(f"No GGUF model weights found in {entry.hf_repo}")
651_SIZE_UNKNOWN = 0
654def _cached_file_is_complete(hf_repo: str, filename: str, dest: Path) -> bool:
655 """Decide whether an existing cached file may be accepted as complete.
657 Verifies the on-disk byte size against the size HuggingFace reports for
658 *filename*. A mismatch means a truncated / corrupt download, so the file
659 is rejected and re-fetched. When the size can't be fetched (offline, API
660 error) it stays unknown and the cached file is accepted: there's nothing
661 to verify against and refusing would block all offline reuse.
662 """
663 expected = fetch_expected_file_size(hf_repo, filename)
664 if expected == _SIZE_UNKNOWN:
665 return True
666 actual = dest.stat().st_size
667 if actual == expected:
668 return True
669 log.warning(
670 "Cached %s is %d bytes but HuggingFace reports %d; re-downloading",
671 dest,
672 actual,
673 expected,
674 )
675 return False
678def _hf_file_size(hf_repo: str, filename: str) -> int | None:
679 """Byte size huggingface_hub resolves for *filename* (None if unreported)."""
680 from huggingface_hub import get_hf_file_metadata, hf_hub_url
682 return get_hf_file_metadata(hf_hub_url(hf_repo, filename), token=hf_token()).size
685def _missing_file_message(hf_repo: str, filename: str) -> str:
686 """User-facing error for a file the Hub reports as nonexistent."""
687 return (
688 f"File {filename!r} does not exist in {hf_repo} on HuggingFace. "
689 "Check the filename on the repo page."
690 )
693def fetch_expected_file_size(hf_repo: str, filename: str) -> int:
694 """Return the byte size huggingface_hub reports for *filename*, or _SIZE_UNKNOWN.
696 Resolves via hf_hub's own file metadata (correct revision, redirects, and
697 LFS/Xet handled uniformly) instead of scraping the repo tree. Returns 0 when
698 offline or unresolvable, in which case the caller keeps the cached file. A
699 file the Hub reports as nonexistent raises instead: that answer is
700 definitive, and treating it as unknown let a pull of a mistyped filename
701 accept a stale local file and report success without downloading anything.
702 """
703 from huggingface_hub.errors import RemoteEntryNotFoundError
705 try:
706 return _hf_file_size(hf_repo, filename) or _SIZE_UNKNOWN
707 except RemoteEntryNotFoundError:
708 raise RuntimeError(_missing_file_message(hf_repo, filename)) from None
709 except Exception:
710 return _SIZE_UNKNOWN