Coverage for src/lilbee/retrieval/language.py: 100%

1"""Language packs for query understanding. 

2 

3ALL language-specific patterns for intent detection and noun matching live 

4here, mirroring ``cli/tui/messages.py``: parsing logic consumes the active 

5pack instead of hardcoding English, so supporting another language means 

6adding a pack, not editing parsers. English is the only shipped pack; the 

7accessor grows config-driven selection when a second pack exists. 

8""" 

9 

10from __future__ import annotations 

11 

12import re 

13from collections.abc import Callable 

14from dataclasses import dataclass 

15 

16 

17@dataclass(frozen=True) 

18class QueryLanguage: 

19 """Everything language-specific the query-understanding layer consumes. 

20 

21 The patterns encode question *shapes*, not vocabulary lists: a pack for 

22 another language supplies its own shapes (word order included), and the 

23 parsing logic in ``query.intent`` stays untouched. 

24 """ 

25 

26 code: str 

27 # "document 214", "doc #47": a document noun followed by a short identifier. 

28 doc_ref_pattern: re.Pattern[str] 

29 # Generic nouns that follow a document noun in topical questions ("the 

30 # documents mention...", "which document says..."); never identifiers. 

31 ref_stopwords: frozenset[str] 

32 # Count-question shapes; routing rules live in ``query.intent.parse_aggregate``. 

33 how_many_pattern: re.Pattern[str] 

34 total_pattern: re.Pattern[str] 

35 association_pattern: re.Pattern[str] 

36 per_pattern: re.Pattern[str] 

37 distinct_pattern: re.Pattern[str] 

38 term_mention_pattern: re.Pattern[str] 

39 # Leading article stripped from a counted term ("the observatory"). 

40 leading_article_pattern: re.Pattern[str] 

41 # Known-item question shapes ("summarize X", "what is X about"): each 

42 # captures a candidate document title, resolved token-exactly against 

43 # source names by the caller. Shapes, not vocabulary: a wrong candidate 

44 # costs one lookup, never a wrong route. 

45 known_item_patterns: tuple[re.Pattern[str], ...] 

46 # A question that leans on earlier turns: a follow-up opener at the start 

47 # or a pronoun/reference word anywhere. A false positive costs one rewrite 

48 # call; a miss searches the question as typed. 

49 follow_up_pattern: re.Pattern[str] 

50 # Spelling variants of a noun phrase (singular/plural) for entity-type 

51 # matching; morphology is the most language-specific piece of all. 

52 noun_variants: Callable[[str], set[str]] 

53 

54 

55# Plural forms the suffix rules below can't produce, mapped both ways. 

56_EN_IRREGULAR_PLURALS = { 

57 "person": "people", 

58 "man": "men", 

59 "woman": "women", 

60 "child": "children", 

61 "foot": "feet", 

62 "tooth": "teeth", 

63 "mouse": "mice", 

64 "goose": "geese", 

65} 

66_EN_IRREGULAR_SINGULARS = {plural: singular for singular, plural in _EN_IRREGULAR_PLURALS.items()} 

67 

68 

69def _english_noun_variants(noun: str) -> set[str]: 

70 """Normalized spelling variants of a noun phrase: itself plus 

71 singular/plural forms of its last word ("tail numbers" ~ "tail number", 

72 "people" ~ "person"). Over-generated junk forms match nothing; a missed 

73 form only fails to resolve, never resolves wrongly. 

74 """ 

75 normalized = " ".join(noun.strip().lower().split()) 

76 if not normalized: 

77 return set() 

78 head, _, last = normalized.rpartition(" ") 

79 prefix = head + " " if head else "" 

80 forms = {last} 

81 if last in _EN_IRREGULAR_PLURALS: 

82 forms.add(_EN_IRREGULAR_PLURALS[last]) 

83 if last in _EN_IRREGULAR_SINGULARS: 

84 forms.add(_EN_IRREGULAR_SINGULARS[last]) 

85 if last.endswith("ies") and len(last) > len("ies"): 

86 forms.add(last[:-3] + "y") 

87 if last.endswith("y"): 

88 forms.add(last[:-1] + "ies") 

89 if last.endswith(("ses", "xes", "zes", "ches", "shes")): 

90 forms.add(last[:-2]) 

91 forms.add(last[:-1] if last.endswith("s") else last + "s") 

92 return {prefix + form for form in forms} 

93 

94 

95# Nouns that name the corpus's units rather than entities within them. A 

96# count over these is a document scan; a count over an entity noun (people, 

97# aircraft) needs extracted records and must NOT route to the scan, because 

98# the scan answers "N documents", a different question than the one asked. 

99_EN_CORPUS_NOUNS = ( 

100 r"(?:documents|sources|files|pages|chunks|passages|books|novels|texts|works" 

101 r"|articles|papers|reports|letters|stories|entries|records|notes|emails" 

102 r"|posts|volumes|manuscripts)" 

103) 

104# "how many of these books ...", "how many of the stories ...". 

105_EN_OF_THESE = r"(?:of\s+(?:these|those|the|my|our)\s+)?" 

106 

107ENGLISH = QueryLanguage( 

108 code="en", 

109 doc_ref_pattern=re.compile( 

110 r"\b(?:document|doc|file|exhibit|attachment|report)\s+#?([\w][\w.-]{0,23})\b", 

111 re.IGNORECASE, 

112 ), 

113 ref_stopwords=frozenset( 

114 {"that", "which", "the", "this", "these", "those", "it", "them", "was", "is", "are"} 

115 ), 

116 how_many_pattern=re.compile( 

117 r"^\s*(?:roughly\s+|approximately\s+|about\s+)?how\s+many\b", re.IGNORECASE 

118 ), 

119 # "how many documents/books/sources are there/indexed": corpus totals. 

120 total_pattern=re.compile( 

121 r"how\s+many\s+" + _EN_OF_THESE + _EN_CORPUS_NOUNS + r"\s*" 

122 r"(?:are\s+(?:there|indexed|in\s+the\s+index)|do(?:es)?\s+.*\b(?:index|corpus|vault)\b.*)?[?\s]*$", 

123 re.IGNORECASE, 

124 ), 

125 # "how many X is each Y associated with" / "how many X per Y": typed 

126 # association counts over extracted entities. 

127 association_pattern=re.compile( 

128 r"how\s+many\s+(.+?)\s+(?:is|are)\s+each\s+(.+?)\s+" 

129 r"(?:associated\s+with|linked\s+to|recorded\s+(?:for|against))", 

130 re.IGNORECASE, 

131 ), 

132 per_pattern=re.compile(r"how\s+many\s+(.+?)\s+per\s+(.+?)[?.\s]*$", re.IGNORECASE), 

133 # "how many distinct/unique X ...": typed distinct counts. 

134 distinct_pattern=re.compile( 

135 r"how\s+many\s+(?:distinct|unique|different)\s+(.+?)" 

136 r"(?:\s+(?:are|were|is|exist)\b.*)?[?.\s]*$", 

137 re.IGNORECASE, 

138 ), 

139 # "how many <corpus noun> mention/contain/reference X": term-mention 

140 # counts, including "of these/those" phrasing ("how many of these books 

141 # mention blood"). Entity nouns deliberately fall through (see 

142 # _EN_CORPUS_NOUNS). 

143 term_mention_pattern=re.compile( 

144 r"how\s+many\s+" + _EN_OF_THESE + _EN_CORPUS_NOUNS + r"\s+" 

145 r"(?:mention|mentions|mentioning|contain|contains|containing|reference|references|referencing|discuss|discussing)\s+" 

146 r"(.+?)[?.\s]*$", 

147 re.IGNORECASE, 

148 ), 

149 leading_article_pattern=re.compile(r"^(?:the|a|an)\s+", re.IGNORECASE), 

150 known_item_patterns=( 

151 re.compile(r"^\s*(?:please\s+)?summari[sz]e\s+(.+?)[?.!\s]*$", re.IGNORECASE), 

152 re.compile(r"^\s*(?:please\s+)?describe\s+(.+?)[?.!\s]*$", re.IGNORECASE), 

153 re.compile(r"^\s*what\s+is\s+(.+?)\s+about[?.!\s]*$", re.IGNORECASE), 

154 re.compile( 

155 r"^\s*(?:give\s+me\s+)?(?:a\s+|an\s+)?(?:summary|overview)\s+of\s+(.+?)[?.!\s]*$", 

156 re.IGNORECASE, 

157 ), 

158 ), 

159 follow_up_pattern=re.compile( 

160 r"^\s*(?:and|but|so|or|also|then|what about|how about)\b" 

161 r"|\b(?:it|its|they|them|their|theirs|he|him|his|she|her|hers|this|that" 

162 r"|these|those|one|ones|same|there|again|above|earlier|previous|former" 

163 r"|latter|else|another|other)\b", 

164 re.IGNORECASE, 

165 ), 

166 noun_variants=_english_noun_variants, 

167) 

168 

169_PACKS = {ENGLISH.code: ENGLISH} 

170 

171 

172def query_language() -> QueryLanguage: 

173 """The active language pack. 

174 

175 English is the only shipped pack; when more exist this resolves from 

176 the configured language instead of a constant. 

177 """ 

178 return _PACKS["en"] 

179 

180 

181def noun_variants(noun: str) -> set[str]: 

182 """Spelling variants of *noun* under the active language pack.""" 

183 return query_language().noun_variants(noun)