Coverage for src/lilbee/retrieval/query/structural.py: 100%

1"""Detect document-structure chunks that dilute retrieval precision. 

2 

3Flags two classes: tables of contents (generic), and classification-banner 

4cover/title pages. Both carry a document's title and section words but never 

5*answer* a substantive question. The detector is deliberately conservative: 

6it only flags chunks that are unambiguously structural, so a false negative 

7(some noise slips through) is preferred to a false positive (dropping real 

8content). 

9""" 

10 

11from __future__ import annotations 

12 

13import itertools 

14import re 

15 

16# A TOC line ends in dot leaders followed by a page number: "Geographic Trends ....... 9". 

17_TOC_LINE = re.compile(r"\.{3,}\s*(\d{1,4})\s*$") 

18 

19# Leader variants normalized to plain dots before matching: ellipsis and 

20# spaced dots (". . . .") are common PDF extractions of the same leaders. 

21_ELLIPSIS = "…" 

22_SPACED_DOTS = re.compile(r"\.(?:[ \t]+\.){2,}") 

23 

24# Classification banners head cover/title pages and running headers alike, so the 

25# banner alone is not enough -- it is combined with low prose density below. 

26_CLASSIFICATION = re.compile(r"\b(UNCLASSIFIED|CONFIDENTIAL|SECRET|FOR OFFICIAL USE ONLY|FOUO)\b") 

27 

28# A chunk needs at least this many non-empty lines before the TOC ratio means anything. 

29_MIN_TOC_LINES = 3 

30# And at least this many dot-leader lines, so a page with one stray "... 42" is not a TOC. 

31_MIN_TOC_HITS = 3 

32_TOC_RATIO = 0.30 

33 

34# Cover/title-page gates: a title page is very short with essentially no prose. 

35# Deliberately tight -- looser gates fire on short banner-carrying body pages and 

36# drop content the answer needs, so a real body page's word count or its first 

37# full sentence must take it out of scope. 

38_COVER_MAX_WORDS = 60 

39_COVER_MAX_SENTENCES = 1 

40# Ratio of fully upper-case LINES, not words: acronym-dense prose ("NATO", 

41# "GDP") stays mixed-case at line level while cover banners are whole lines. 

42# Real covers mix caps org lines with title-case title lines, hence 0.4. 

43_COVER_CAPS_LINE_RATIO = 0.4 

44 

45 

46def _normalize_leaders(line: str) -> str: 

47 """Fold ellipsis and spaced-dot leaders into plain dots.""" 

48 line = line.replace(_ELLIPSIS, "...") 

49 return _SPACED_DOTS.sub(lambda m: "." * (m.group().count(".")), line) 

50 

51 

52def _is_toc(nonempty: list[str]) -> bool: 

53 """A table of contents: several dot-leader lines with non-decreasing page numbers. 

54 

55 The monotonic check separates a TOC from dot-leader data pages (price 

56 lists, log output), whose trailing numbers are not ordered. 

57 """ 

58 if len(nonempty) < _MIN_TOC_LINES: 

59 return False 

60 pages = [ 

61 int(m.group(1)) for line in nonempty if (m := _TOC_LINE.search(_normalize_leaders(line))) 

62 ] 

63 if len(pages) < _MIN_TOC_HITS or len(pages) / len(nonempty) < _TOC_RATIO: 

64 return False 

65 return all(a <= b for a, b in itertools.pairwise(pages)) 

66 

67 

68def _is_cover_page(text: str) -> bool: 

69 """A cover/title page: short, almost no sentences, banner-line dominated, 

70 and carrying a classification banner.""" 

71 if not _CLASSIFICATION.search(text): 

72 return False 

73 words = text.split() 

74 if not words or len(words) > _COVER_MAX_WORDS: 

75 return False 

76 sentences = text.count(".") + text.count("!") + text.count("?") 

77 if sentences > _COVER_MAX_SENTENCES: 

78 return False 

79 lines = [line.strip() for line in text.splitlines() if line.strip()] 

80 # The banner satisfies its own gate; the caps ratio measures the title 

81 # lines beyond it, so one banner cannot tip a page with real content. 

82 content = [line for line in lines if not _CLASSIFICATION.search(line)] 

83 if not content: 

84 return True 

85 caps_lines = sum(1 for line in content if len(line) > 1 and line.isupper()) 

86 return caps_lines / len(content) >= _COVER_CAPS_LINE_RATIO 

87 

88 

89def is_structural_chunk(text: str) -> bool: 

90 """True when *text* is a table of contents or a cover/title page -- a 

91 document-structure chunk that should not compete as an answer passage.""" 

92 if not text or not text.strip(): 

93 return False 

94 nonempty = [line for line in text.splitlines() if line.strip()] 

95 return _is_toc(nonempty) or _is_cover_page(text)