Coverage for src/lilbee/data/ingest/skip_marker.py: 100%

1"""Sidecar records of files that produced no chunks, so a sync can skip them. 

2 

3A file that yields zero chunks (Tesseract timeout, decode failure, no usable 

4text) gets a marker here keyed by the file hash that failed. 

5``_plan_file_changes`` treats a file whose current hash matches its marker as 

6unchanged, so the per-file extract cost (30-60s for a stubborn scanned PDF) is 

7paid once, not on every sync. The marker is a small JSON file in 

8``cfg.data_root``; editing the file changes its hash and re-arms it, and 

9``retry_skipped`` / ``force_rebuild`` drop the file from the marker set. 

10 

11A second sidecar (``skip_reasons.json``) records filename → human-readable 

12reason, so a report can say WHY a file was skipped (the exception message, or 

13"no text extracted"), not just that it was. It is informational only -- the 

14hash-keyed markers above drive the resume logic -- and is cleared alongside them. 

15""" 

16 

17from __future__ import annotations 

18 

19import contextlib 

20import json 

21import logging 

22import os 

23from collections.abc import Iterable 

24from pathlib import Path 

25 

26from lilbee.data.types import SkippedSource 

27 

28log = logging.getLogger(__name__) 

29 

30SKIP_MARKER_FILENAME = "skipped_sources.json" 

31SKIP_REASON_FILENAME = "skip_reasons.json" 

32DEFAULT_SKIP_REASON = "held out by an earlier sync" 

33 

34 

35def _load_str_map(path: Path) -> dict[str, str]: 

36 """Load a ``{str: str}`` JSON file, or empty dict on any read/parse error.""" 

37 if not path.exists(): 

38 return {} 

39 try: 

40 raw = json.loads(path.read_text(encoding="utf-8")) 

41 except (OSError, json.JSONDecodeError, UnicodeDecodeError) as exc: 

42 log.debug("Sidecar %s unreadable, treating as empty: %s", path.name, exc) 

43 return {} 

44 if not isinstance(raw, dict): 

45 return {} 

46 return {str(k): str(v) for k, v in raw.items() if isinstance(v, str)} 

47 

48 

49def _write_str_map(path: Path, data: dict[str, str]) -> None: 

50 """Replace *path* atomically with a ``{str: str}`` JSON map. Best-effort.""" 

51 tmp = path.with_suffix(path.suffix + ".tmp") 

52 try: 

53 path.parent.mkdir(parents=True, exist_ok=True) 

54 tmp.write_text(json.dumps(data, sort_keys=True), encoding="utf-8") 

55 os.replace(tmp, path) 

56 except OSError as exc: 

57 log.warning("Failed to persist %s: %s", path, exc) 

58 with contextlib.suppress(OSError): 

59 tmp.unlink() 

60 

61 

62def _unlink(path: Path) -> None: 

63 try: 

64 path.unlink(missing_ok=True) 

65 except OSError as exc: 

66 log.debug("Could not remove %s: %s", path, exc) 

67 

68 

69def load_skip_markers(data_root: Path) -> dict[str, str]: 

70 """Load the filename → failed-hash map, or empty dict on any read error.""" 

71 return _load_str_map(data_root / SKIP_MARKER_FILENAME) 

72 

73 

74def write_skip_markers(data_root: Path, markers: dict[str, str]) -> None: 

75 """Replace the marker file atomically. Best-effort: errors are logged, not raised.""" 

76 _write_str_map(data_root / SKIP_MARKER_FILENAME, markers) 

77 

78 

79def load_skip_reasons(data_root: Path) -> dict[str, str]: 

80 """Load the filename → skip-reason map (informational), empty on any read error.""" 

81 return _load_str_map(data_root / SKIP_REASON_FILENAME) 

82 

83 

84def write_skip_reasons(data_root: Path, reasons: dict[str, str]) -> None: 

85 """Replace the reasons sidecar atomically. Best-effort: errors are logged, not raised.""" 

86 _write_str_map(data_root / SKIP_REASON_FILENAME, reasons) 

87 

88 

89def describe_skips(data_root: Path, names: Iterable[str]) -> list[SkippedSource]: 

90 """Pair each name with its recorded reason, in order; ``DEFAULT_SKIP_REASON`` when none.""" 

91 reasons = load_skip_reasons(data_root) 

92 return [ 

93 SkippedSource(filename=name, reason=reasons.get(name, DEFAULT_SKIP_REASON)) 

94 for name in names 

95 ] 

96 

97 

98def clear_skip_markers(data_root: Path) -> None: 

99 """Delete both the marker file and the reasons sidecar. No-op if absent.""" 

100 _unlink(data_root / SKIP_MARKER_FILENAME) 

101 _unlink(data_root / SKIP_REASON_FILENAME)