Coverage for src/lilbee/data/ingest/skip_marker.py: 100%

1"""Sidecar records of files that produced no chunks, so a sync can skip them. 

2 

3A file that yields zero chunks (Tesseract timeout, decode failure, no usable 

4text) gets a marker here keyed by the file hash that failed. 

5``_plan_file_changes`` treats a file whose current hash matches its marker as 

6unchanged, so the per-file extract cost (30-60s for a stubborn scanned PDF) is 

7paid once, not on every sync. The marker is a small JSON file in 

8``cfg.data_root``; editing the file changes its hash and re-arms it, and 

9``retry_skipped`` / ``force_rebuild`` drop the file from the marker set. 

10 

11A second sidecar (``skip_reasons.json``) records filename → human-readable 

12reason, so a report can say WHY a file was skipped (the exception message, or 

13"no text extracted"), not just that it was. It is informational only -- the 

14hash-keyed markers above drive the resume logic -- and is cleared alongside them. 

15""" 

16 

17from __future__ import annotations 

18 

19import contextlib 

20import json 

21import logging 

22import os 

23from pathlib import Path 

24 

25log = logging.getLogger(__name__) 

26 

27SKIP_MARKER_FILENAME = "skipped_sources.json" 

28SKIP_REASON_FILENAME = "skip_reasons.json" 

29 

30 

31def _load_str_map(path: Path) -> dict[str, str]: 

32 """Load a ``{str: str}`` JSON file, or empty dict on any read/parse error.""" 

33 if not path.exists(): 

34 return {} 

35 try: 

36 raw = json.loads(path.read_text(encoding="utf-8")) 

37 except (OSError, json.JSONDecodeError, UnicodeDecodeError) as exc: 

38 log.debug("Sidecar %s unreadable, treating as empty: %s", path.name, exc) 

39 return {} 

40 if not isinstance(raw, dict): 

41 return {} 

42 return {str(k): str(v) for k, v in raw.items() if isinstance(v, str)} 

43 

44 

45def _write_str_map(path: Path, data: dict[str, str]) -> None: 

46 """Replace *path* atomically with a ``{str: str}`` JSON map. Best-effort.""" 

47 tmp = path.with_suffix(path.suffix + ".tmp") 

48 try: 

49 path.parent.mkdir(parents=True, exist_ok=True) 

50 tmp.write_text(json.dumps(data, sort_keys=True), encoding="utf-8") 

51 os.replace(tmp, path) 

52 except OSError as exc: 

53 log.warning("Failed to persist %s: %s", path, exc) 

54 with contextlib.suppress(OSError): 

55 tmp.unlink() 

56 

57 

58def _unlink(path: Path) -> None: 

59 try: 

60 path.unlink(missing_ok=True) 

61 except OSError as exc: 

62 log.debug("Could not remove %s: %s", path, exc) 

63 

64 

65def load_skip_markers(data_root: Path) -> dict[str, str]: 

66 """Load the filename → failed-hash map, or empty dict on any read error.""" 

67 return _load_str_map(data_root / SKIP_MARKER_FILENAME) 

68 

69 

70def write_skip_markers(data_root: Path, markers: dict[str, str]) -> None: 

71 """Replace the marker file atomically. Best-effort: errors are logged, not raised.""" 

72 _write_str_map(data_root / SKIP_MARKER_FILENAME, markers) 

73 

74 

75def load_skip_reasons(data_root: Path) -> dict[str, str]: 

76 """Load the filename → skip-reason map (informational), empty on any read error.""" 

77 return _load_str_map(data_root / SKIP_REASON_FILENAME) 

78 

79 

80def write_skip_reasons(data_root: Path, reasons: dict[str, str]) -> None: 

81 """Replace the reasons sidecar atomically. Best-effort: errors are logged, not raised.""" 

82 _write_str_map(data_root / SKIP_REASON_FILENAME, reasons) 

83 

84 

85def clear_skip_markers(data_root: Path) -> None: 

86 """Delete both the marker file and the reasons sidecar. No-op if absent.""" 

87 _unlink(data_root / SKIP_MARKER_FILENAME) 

88 _unlink(data_root / SKIP_REASON_FILENAME)