Coverage for src/lilbee/retrieval/language.py: 100%

1"""Language packs for query understanding. 

2 

3ALL language-specific patterns for intent detection and noun matching live 

4here, mirroring ``cli/tui/messages.py``: parsing logic consumes the active 

5pack instead of hardcoding English, so supporting another language means 

6adding a pack, not editing parsers. English is the only shipped pack; the 

7accessor grows config-driven selection when a second pack exists. 

8""" 

9 

10from __future__ import annotations 

11 

12import re 

13from collections.abc import Callable 

14from dataclasses import dataclass 

15 

16 

17@dataclass(frozen=True) 

18class QueryLanguage: 

19 """Everything language-specific the query-understanding layer consumes. 

20 

21 The patterns encode question *shapes*, not vocabulary lists: a pack for 

22 another language supplies its own shapes (word order included), and the 

23 parsing logic in ``query.intent`` stays untouched. 

24 """ 

25 

26 code: str 

27 # "document 214", "doc #47": a document noun followed by a short identifier. 

28 doc_ref_pattern: re.Pattern[str] 

29 # Generic nouns that follow a document noun in topical questions ("the 

30 # documents mention...", "which document says..."); never identifiers. 

31 ref_stopwords: frozenset[str] 

32 # Count-question shapes; routing rules live in ``query.intent.parse_aggregate``. 

33 how_many_pattern: re.Pattern[str] 

34 total_pattern: re.Pattern[str] 

35 association_pattern: re.Pattern[str] 

36 per_pattern: re.Pattern[str] 

37 distinct_pattern: re.Pattern[str] 

38 term_mention_pattern: re.Pattern[str] 

39 # Leading article stripped from a counted term ("the observatory"). 

40 leading_article_pattern: re.Pattern[str] 

41 # Known-item question shapes ("summarize X", "what is X about"): each 

42 # captures a candidate document title, resolved token-exactly against 

43 # source names by the caller. Shapes, not vocabulary: a wrong candidate 

44 # costs one lookup, never a wrong route. 

45 known_item_patterns: tuple[re.Pattern[str], ...] 

46 # Spelling variants of a noun phrase (singular/plural) for entity-type 

47 # matching; morphology is the most language-specific piece of all. 

48 noun_variants: Callable[[str], set[str]] 

49 

50 

51# Plural forms the suffix rules below can't produce, mapped both ways. 

52_EN_IRREGULAR_PLURALS = { 

53 "person": "people", 

54 "man": "men", 

55 "woman": "women", 

56 "child": "children", 

57 "foot": "feet", 

58 "tooth": "teeth", 

59 "mouse": "mice", 

60 "goose": "geese", 

61} 

62_EN_IRREGULAR_SINGULARS = {plural: singular for singular, plural in _EN_IRREGULAR_PLURALS.items()} 

63 

64 

65def _english_noun_variants(noun: str) -> set[str]: 

66 """Normalized spelling variants of a noun phrase: itself plus 

67 singular/plural forms of its last word ("tail numbers" ~ "tail number", 

68 "people" ~ "person"). Over-generated junk forms match nothing; a missed 

69 form only fails to resolve, never resolves wrongly. 

70 """ 

71 normalized = " ".join(noun.strip().lower().split()) 

72 if not normalized: 

73 return set() 

74 head, _, last = normalized.rpartition(" ") 

75 prefix = head + " " if head else "" 

76 forms = {last} 

77 if last in _EN_IRREGULAR_PLURALS: 

78 forms.add(_EN_IRREGULAR_PLURALS[last]) 

79 if last in _EN_IRREGULAR_SINGULARS: 

80 forms.add(_EN_IRREGULAR_SINGULARS[last]) 

81 if last.endswith("ies") and len(last) > len("ies"): 

82 forms.add(last[:-3] + "y") 

83 if last.endswith("y"): 

84 forms.add(last[:-1] + "ies") 

85 if last.endswith(("ses", "xes", "zes", "ches", "shes")): 

86 forms.add(last[:-2]) 

87 forms.add(last[:-1] if last.endswith("s") else last + "s") 

88 return {prefix + form for form in forms} 

89 

90 

91# Nouns that name the corpus's units rather than entities within them. A 

92# count over these is a document scan; a count over an entity noun (people, 

93# aircraft) needs extracted records and must NOT route to the scan, because 

94# the scan answers "N documents", a different question than the one asked. 

95_EN_CORPUS_NOUNS = ( 

96 r"(?:documents|sources|files|pages|chunks|passages|books|novels|texts|works" 

97 r"|articles|papers|reports|letters|stories|entries|records|notes|emails" 

98 r"|posts|volumes|manuscripts)" 

99) 

100# "how many of these books ...", "how many of the stories ...". 

101_EN_OF_THESE = r"(?:of\s+(?:these|those|the|my|our)\s+)?" 

102 

103ENGLISH = QueryLanguage( 

104 code="en", 

105 doc_ref_pattern=re.compile( 

106 r"\b(?:document|doc|file|exhibit|attachment|report)\s+#?([\w][\w.-]{0,23})\b", 

107 re.IGNORECASE, 

108 ), 

109 ref_stopwords=frozenset( 

110 {"that", "which", "the", "this", "these", "those", "it", "them", "was", "is", "are"} 

111 ), 

112 how_many_pattern=re.compile( 

113 r"^\s*(?:roughly\s+|approximately\s+|about\s+)?how\s+many\b", re.IGNORECASE 

114 ), 

115 # "how many documents/books/sources are there/indexed": corpus totals. 

116 total_pattern=re.compile( 

117 r"how\s+many\s+" + _EN_OF_THESE + _EN_CORPUS_NOUNS + r"\s*" 

118 r"(?:are\s+(?:there|indexed|in\s+the\s+index)|do(?:es)?\s+.*\b(?:index|corpus|vault)\b.*)?[?\s]*$", 

119 re.IGNORECASE, 

120 ), 

121 # "how many X is each Y associated with" / "how many X per Y": typed 

122 # association counts over extracted entities. 

123 association_pattern=re.compile( 

124 r"how\s+many\s+(.+?)\s+(?:is|are)\s+each\s+(.+?)\s+" 

125 r"(?:associated\s+with|linked\s+to|recorded\s+(?:for|against))", 

126 re.IGNORECASE, 

127 ), 

128 per_pattern=re.compile(r"how\s+many\s+(.+?)\s+per\s+(.+?)[?.\s]*$", re.IGNORECASE), 

129 # "how many distinct/unique X ...": typed distinct counts. 

130 distinct_pattern=re.compile( 

131 r"how\s+many\s+(?:distinct|unique|different)\s+(.+?)" 

132 r"(?:\s+(?:are|were|is|exist)\b.*)?[?.\s]*$", 

133 re.IGNORECASE, 

134 ), 

135 # "how many <corpus noun> mention/contain/reference X": term-mention 

136 # counts, including "of these/those" phrasing ("how many of these books 

137 # mention blood"). Entity nouns deliberately fall through (see 

138 # _EN_CORPUS_NOUNS). 

139 term_mention_pattern=re.compile( 

140 r"how\s+many\s+" + _EN_OF_THESE + _EN_CORPUS_NOUNS + r"\s+" 

141 r"(?:mention|mentions|mentioning|contain|contains|containing|reference|references|referencing|discuss|discussing)\s+" 

142 r"(.+?)[?.\s]*$", 

143 re.IGNORECASE, 

144 ), 

145 leading_article_pattern=re.compile(r"^(?:the|a|an)\s+", re.IGNORECASE), 

146 known_item_patterns=( 

147 re.compile(r"^\s*(?:please\s+)?summari[sz]e\s+(.+?)[?.!\s]*$", re.IGNORECASE), 

148 re.compile(r"^\s*(?:please\s+)?describe\s+(.+?)[?.!\s]*$", re.IGNORECASE), 

149 re.compile(r"^\s*what\s+is\s+(.+?)\s+about[?.!\s]*$", re.IGNORECASE), 

150 re.compile( 

151 r"^\s*(?:give\s+me\s+)?(?:a\s+|an\s+)?(?:summary|overview)\s+of\s+(.+?)[?.!\s]*$", 

152 re.IGNORECASE, 

153 ), 

154 ), 

155 noun_variants=_english_noun_variants, 

156) 

157 

158_PACKS = {ENGLISH.code: ENGLISH} 

159 

160 

161def query_language() -> QueryLanguage: 

162 """The active language pack. 

163 

164 English is the only shipped pack; when more exist this resolves from 

165 the configured language instead of a constant. 

166 """ 

167 return _PACKS["en"] 

168 

169 

170def noun_variants(noun: str) -> set[str]: 

171 """Spelling variants of *noun* under the active language pack.""" 

172 return query_language().noun_variants(noun)