Coverage for src/lilbee/retrieval/clustering_embedding/types.py: 100%

1"""Type definitions for the chunk-level mutual-kNN clusterer.""" 

2 

3from __future__ import annotations 

4 

5from dataclasses import dataclass, field 

6 

7# Minimum token length for TF-IDF labeling. Shorter tokens are mostly 

8# articles, prepositions, and single letters: noise that inflates term 

9# counts without adding topic signal. Three characters keeps useful 

10# acronyms (api, xml, sql). 

11_MIN_TF_TOKEN_LEN = 3 

12 

13 

14def _tokenize_for_tf(text: str) -> list[str]: 

15 """Lowercase alphanumeric tokens for TF-IDF scoring. 

16 

17 Deliberately has NO stopword list: common words like "the" or "and" 

18 get an IDF near zero (they appear in almost every chunk) so TF-IDF 

19 filters them automatically. A hand-curated English stoplist would 

20 add maintenance burden and break on non-English corpora for no 

21 additional quality. 

22 

23 Lives beside :class:`ClusterChunk` because the record derives its 

24 tokens from it; importing it from the helpers module would be circular. 

25 """ 

26 result: list[str] = [] 

27 for raw in text.lower().split(): 

28 word = "".join(ch for ch in raw if ch.isalnum()) 

29 if len(word) >= _MIN_TF_TOKEN_LEN: 

30 result.append(word) 

31 return result 

32 

33 

34@dataclass(slots=True) 

35class ClusterChunk: 

36 """Lightweight view of one chunk row used by the clusterer. 

37 

38 Named for the clusterer rather than ``ChunkRecord`` so it cannot be 

39 confused with :class:`lilbee.data.types.ChunkRecord`, an 

40 unrelated TypedDict with a different field set used across ingest. 

41 

42 ``tokens`` is a derived cache of ``_tokenize_for_tf(text)``. It is 

43 computed here rather than at each construction site: TF-IDF labeling 

44 silently degrades to fallback labels when a record carries no tokens, 

45 so the invariant is structural instead of caller discipline. ``slots`` 

46 keeps the per-record footprint down, since one record is materialized 

47 per corpus chunk. 

48 """ 

49 

50 source: str 

51 chunk_index: int 

52 text: str 

53 tokens: list[str] = field(default_factory=list) 

54 

55 def __post_init__(self) -> None: 

56 if not self.tokens: 

57 self.tokens = _tokenize_for_tf(self.text)