Coverage for src/lilbee/data/store/held_import.py: 100%

1"""The held-out records an older lilbee kept in JSON files at the data root.""" 

2 

3from __future__ import annotations 

4 

5import contextlib 

6import json 

7import logging 

8from pathlib import Path 

9 

10from .types import REMOVED_REASON, HeldOut, HeldRecord 

11 

12log = logging.getLogger(__name__) 

13 

14_MARKERS = "skipped_sources.json" 

15_REASONS = "skip_reasons.json" 

16_KINDS = "skip_kinds.json" 

17_PENDING = "skip_records.pending.json" 

18_RECORDS = (_MARKERS, _REASONS, _KINDS) 

19# Every file the older format kept, the lock of its writers included. 

20LEGACY_FILES = (*_RECORDS, _PENDING, f"{_MARKERS}.lock") 

21# The reason an older lilbee wrote for a removal. 

22_OLD_REMOVED_REASON = "removed via remove (re-add the source or run retry-skipped to restore)" 

23_DEFAULT_REASON = "held out by an earlier sync" 

24 

25 

26def legacy_files(data_root: Path) -> list[Path]: 

27 """The files of the older format that are still at *data_root*.""" 

28 return [path for name in LEGACY_FILES if (path := data_root / name).exists()] 

29 

30 

31def _text(path: Path) -> str | None: 

32 """The text of *path*, or None when it is absent or cannot be read.""" 

33 try: 

34 return path.read_text(encoding="utf-8") 

35 except (OSError, UnicodeDecodeError) as exc: 

36 log.debug("Cannot read %s: %s", path.name, exc) 

37 return None 

38 

39 

40def _object(text: str | None) -> dict[str, object]: 

41 """The JSON object *text* holds, or an empty one.""" 

42 if text is None: 

43 return {} 

44 try: 

45 raw = json.loads(text) 

46 except json.JSONDecodeError: 

47 return {} 

48 # the files are untyped JSON 

49 return {str(key): value for key, value in raw.items()} if isinstance(raw, dict) else {} 

50 

51 

52def _texts(data_root: Path) -> dict[str, str | None]: 

53 """The text of each record file, from a write that did not end when one is recorded.""" 

54 after = _object(_text(data_root / _PENDING)).get("after") 

55 # the pending file is untyped JSON 

56 if isinstance(after, dict) and all(isinstance(after.get(name), str) for name in _RECORDS): 

57 return {name: after[name] for name in _RECORDS} 

58 return {name: _text(data_root / name) for name in _RECORDS} 

59 

60 

61def _kind(stored: object, marker: str, reason: str | None) -> HeldOut: 

62 """The stored kind while it names this marker and reason; else the kind its reason gives.""" 

63 # the kinds file is untyped JSON 

64 if isinstance(stored, dict) and (stored.get("hash"), stored.get("reason")) == (marker, reason): 

65 with contextlib.suppress(ValueError): 

66 return HeldOut(stored.get("kind", "")) 

67 return HeldOut.REMOVED if reason == _OLD_REMOVED_REASON else HeldOut.FAILED 

68 

69 

70def read_legacy_records(data_root: Path) -> dict[str, HeldRecord]: 

71 """The record of each file the JSON files at *data_root* hold out, by path.""" 

72 texts = _texts(data_root) 

73 markers, reasons, kinds = (_object(texts[name]) for name in _RECORDS) 

74 records: dict[str, HeldRecord] = {} 

75 for name, marker in markers.items(): 

76 if not isinstance(marker, str): 

77 continue 

78 reason = reasons.get(name) 

79 text = reason if isinstance(reason, str) else None 

80 kind = _kind(kinds.get(name), marker, text) 

81 shown = REMOVED_REASON if kind is HeldOut.REMOVED else text or _DEFAULT_REASON 

82 records[name] = HeldRecord(kind, marker, shown) 

83 return records 

84 

85 

86def remove_legacy_files(data_root: Path) -> None: 

87 """Delete the files of the older format; one that cannot go is read again next open.""" 

88 for path in legacy_files(data_root): 

89 try: 

90 path.unlink(missing_ok=True) 

91 except OSError as exc: 

92 log.warning("Could not delete %s: %s", path, exc)