Coverage for src/lilbee/data/title.py: 100%

1"""Document title and source-level metadata derivation for ingest.""" 

2 

3from __future__ import annotations 

4 

5import re 

6from pathlib import PurePath 

7from typing import Protocol 

8 

9from lilbee.data.store import SourceMeta 

10 

11# Filename-stem separators flattened to spaces when no extracted title exists. 

12_STEM_SEPARATOR_RE = re.compile(r"[_\-\s]+") 

13 

14# Stems with no searchable words: camera/scanner counters, generic names, bare 

15# numbers/dates, hex ids. Indexing these gives the title arm noise at full weight. 

16_COUNTER_STEM_RE = re.compile( 

17 r"(?:img|image|dsc[nf]?|pxl|mvimg|vid|video|scan|screenshot|photo|pic|picture" 

18 r"|untitled|unnamed|noname|new|document|doc|file|page)?(?:\s*\d+)*", 

19 re.IGNORECASE, 

20) 

21_NUMERIC_STEM_RE = re.compile(r"[\d\s.]+") 

22_HEX_ID_RE = re.compile(r"[0-9a-f]{8,}", re.IGNORECASE) 

23 

24# Below this many characters a stem cannot form a searchable word. 

25_MIN_TITLE_CHARS = 3 

26 

27 

28def is_junk_stem(stem: str) -> bool: 

29 """True when a filename stem carries no searchable title words.""" 

30 flat = _STEM_SEPARATOR_RE.sub(" ", stem).strip() 

31 if len(flat) < _MIN_TITLE_CHARS: 

32 return True 

33 if _NUMERIC_STEM_RE.fullmatch(flat) or _COUNTER_STEM_RE.fullmatch(flat): 

34 return True 

35 return bool(_HEX_ID_RE.fullmatch(flat.replace(" ", ""))) 

36 

37 

38class ExtractionMetadata(Protocol): 

39 """xberg metadata fields, typed ``object``: xberg annotates but does not enforce 

40 them (a PDF /Author arrives as a bare str; a non-str title is accepted).""" 

41 

42 @property 

43 def title(self) -> object: ... 

44 

45 @property 

46 def authors(self) -> object: ... 

47 

48 @property 

49 def created_at(self) -> object: ... 

50 

51 

52def derive_title(source_name: str, metadata_title: object = None) -> str: 

53 """Human-readable document title: the extracted title, else the cleaned filename stem. 

54 

55 The stem cleanup flattens underscore/hyphen separators to spaces so BM25 

56 tokenizes ``survey_214.pdf`` into the same terms a query would use. Junk 

57 stems (``IMG 1234``, bare numbers, hex ids) yield "" so no title is stored. 

58 """ 

59 if isinstance(metadata_title, str) and metadata_title.strip(): 

60 return metadata_title.strip() 

61 stem = PurePath(source_name).stem 

62 if is_junk_stem(stem): 

63 return "" 

64 return _STEM_SEPARATOR_RE.sub(" ", stem).strip() 

65 

66 

67def source_meta_from_extraction( 

68 metadata: ExtractionMetadata | None, source_name: str 

69) -> SourceMeta: 

70 """Fold xberg extraction metadata into a :class:`SourceMeta`. 

71 

72 The title falls back to the filename stem; authors and creation date stay 

73 empty (persisted NULL) when the extractor reports none. xberg annotates these 

74 fields but does not enforce them: a PDF ``/Author`` arrives as a bare ``str`` 

75 where ``list[str]`` is declared, so a string is treated as one author rather 

76 than split into its characters, and non-string entries are coerced. 

77 """ 

78 raw_authors = metadata.authors if metadata is not None else None 

79 if isinstance(raw_authors, str): 

80 authors: list[str] = [raw_authors] 

81 elif isinstance(raw_authors, (list, tuple)): 

82 authors = [str(a) for a in raw_authors if a] 

83 else: 

84 authors = [] 

85 return SourceMeta( 

86 title=derive_title(source_name, metadata.title if metadata is not None else None), 

87 authors=", ".join(a for a in authors if a), 

88 created_at=str((metadata.created_at if metadata is not None else None) or ""), 

89 )