Coverage for src/lilbee/data/title.py: 100%

1"""Document title and source-level metadata derivation for ingest.""" 

2 

3from __future__ import annotations 

4 

5import re 

6from pathlib import PurePath 

7from typing import Protocol 

8 

9from lilbee.core.web_names import is_web_source, page_stem 

10from lilbee.data.store import SourceMeta 

11 

12# Filename-stem separators flattened to spaces when no extracted title exists. 

13_STEM_SEPARATOR_RE = re.compile(r"[_\-\s]+") 

14 

15# Stems with no searchable words: camera/scanner counters, generic names, bare 

16# numbers/dates, hex ids. Indexing these gives the title arm noise at full weight. 

17_COUNTER_STEM_RE = re.compile( 

18 r"(?:img|image|dsc[nf]?|pxl|mvimg|vid|video|scan|screenshot|photo|pic|picture" 

19 r"|untitled|unnamed|noname|new|document|doc|file|page)?(?:\s*\d+)*", 

20 re.IGNORECASE, 

21) 

22_NUMERIC_STEM_RE = re.compile(r"[\d\s.]+") 

23_HEX_ID_RE = re.compile(r"[0-9a-f]{8,}", re.IGNORECASE) 

24# A crawled page named by one number or one hex string is titled with that name. 

25_BARE_ID_RE = re.compile(r"\d+|[0-9a-f]{8,}", re.IGNORECASE) 

26 

27# Below this many characters a stem cannot form a searchable word. 

28_MIN_TITLE_CHARS = 3 

29 

30 

31def is_junk_stem(stem: str) -> bool: 

32 """True when a filename stem carries no searchable title words.""" 

33 flat = _STEM_SEPARATOR_RE.sub(" ", stem).strip() 

34 if len(flat) < _MIN_TITLE_CHARS: 

35 return True 

36 if _NUMERIC_STEM_RE.fullmatch(flat) or _COUNTER_STEM_RE.fullmatch(flat): 

37 return True 

38 return bool(_HEX_ID_RE.fullmatch(flat.replace(" ", ""))) 

39 

40 

41class ExtractionMetadata(Protocol): 

42 """xberg metadata fields, typed ``object``: xberg annotates but does not enforce 

43 them (a PDF /Author arrives as a bare str; a non-str title is accepted).""" 

44 

45 @property 

46 def title(self) -> object: ... 

47 

48 @property 

49 def authors(self) -> object: ... 

50 

51 @property 

52 def created_at(self) -> object: ... 

53 

54 

55def derive_title(source_name: str, metadata_title: object = None) -> str: 

56 """Human-readable document title: the extracted title, else the cleaned filename stem. 

57 

58 The stem cleanup flattens underscore/hyphen separators to spaces so BM25 

59 tokenizes ``survey_214.pdf`` into the same terms a query would use. Junk 

60 stems (``IMG 1234``, bare numbers, hex ids) yield "" so no title is stored. 

61 A crawled page's stem is its readable name, without the hash of its address; 

62 a page named by a bare number or hex id takes that name as its title. 

63 """ 

64 if isinstance(metadata_title, str) and metadata_title.strip(): 

65 return metadata_title.strip() 

66 name = PurePath(source_name).name 

67 stem = page_stem(name) if is_web_source(source_name) else PurePath(name).stem 

68 if is_junk_stem(stem): 

69 return stem if is_web_source(source_name) and _BARE_ID_RE.fullmatch(stem) else "" 

70 return _STEM_SEPARATOR_RE.sub(" ", stem).strip() 

71 

72 

73def source_meta_from_extraction( 

74 metadata: ExtractionMetadata | None, source_name: str 

75) -> SourceMeta: 

76 """Fold xberg extraction metadata into a :class:`SourceMeta`. 

77 

78 The title falls back to the filename stem; authors and creation date stay 

79 empty (persisted NULL) when the extractor reports none. xberg annotates these 

80 fields but does not enforce them: a PDF ``/Author`` arrives as a bare ``str`` 

81 where ``list[str]`` is declared, so a string is treated as one author rather 

82 than split into its characters, and non-string entries are coerced. 

83 """ 

84 raw_authors = metadata.authors if metadata is not None else None 

85 if isinstance(raw_authors, str): 

86 authors: list[str] = [raw_authors] 

87 elif isinstance(raw_authors, (list, tuple)): 

88 authors = [str(a) for a in raw_authors if a] 

89 else: 

90 authors = [] 

91 return SourceMeta( 

92 title=derive_title(source_name, metadata.title if metadata is not None else None), 

93 authors=", ".join(a for a in authors if a), 

94 created_at=str((metadata.created_at if metadata is not None else None) or ""), 

95 )