Coverage for src/lilbee/retrieval/concepts/nlp.py: 100%

1"""spaCy-backed NLP helpers for the concept graph.""" 

2 

3from __future__ import annotations 

4 

5import functools 

6import logging 

7from typing import Any 

8 

9from lilbee.core.text import collapse_whitespace, is_valid_label 

10 

11log = logging.getLogger(__name__) 

12 

13 

14@functools.cache 

15def concepts_available() -> bool: 

16 """Whether the concept-graph dependencies (spacy, graspologic) are installed. 

17 

18 Fixed for the process lifetime (cached), like :func:`gpu_device_count`. 

19 Python caches only *successful* imports, so an absent extra re-walks 

20 ``sys.path`` on every call, and ingest calls this once per file. 

21 

22 Deliberately checks the *packages* only, not whether the ``en_core_web_sm`` 

23 model is downloaded: a missing model is a fixable user situation, so 

24 :func:`load_spacy_pipeline` raises with the download command rather than 

25 being silently reported unavailable here. Callers therefore still handle 

26 ``ImportError`` from the load even when this returns True. 

27 """ 

28 try: 

29 import graspologic_native # noqa: F401 

30 import spacy # noqa: F401 

31 

32 return True 

33 except ImportError: 

34 return False 

35 

36 

37def _ensure_spacy_model() -> Any: 

38 """Load the spaCy NER model; raise ImportError with an install hint if missing.""" 

39 import spacy 

40 

41 model_name = "en_core_web_sm" 

42 try: 

43 return spacy.load(model_name) 

44 except OSError as exc: 

45 raise ImportError( 

46 f"spaCy model {model_name!r} not installed. Run: python -m spacy download {model_name}" 

47 ) from exc 

48 

49 

50def load_spacy_pipeline() -> Any: 

51 """Public entry point for the shared spaCy NER + noun-chunk pipeline. 

52 

53 Raises ``ImportError`` if spaCy or the ``en_core_web_sm`` model is not 

54 installed; the message carries the download command. This is the seam 

55 other packages import -- ``_ensure_spacy_model`` stays private to this 

56 module and its own package. 

57 """ 

58 return _ensure_spacy_model() 

59 

60 

61def _filter_noun_chunks(doc: Any, max_concepts: int) -> list[str]: 

62 """Extract deduplicated, filtered noun chunks from a spaCy doc. 

63 

64 Applies the same :func:`is_valid_label` gate the wiki entity 

65 extractor uses, so structural-noise concepts (markdown table 

66 delimiters, page-number-prefixed tokens, sub-three-char fragments) 

67 never enter the co-occurrence graph and therefore never become a 

68 synthesis-page cluster label. 

69 

70 The gate runs on the lowercased form here while the NER extractor 

71 gates on the original-cased surface; the two decisions match 

72 because ``is_valid_label`` is case-agnostic today. Any future 

73 case-sensitive rule must land in both call sites together. 

74 """ 

75 seen: set[str] = set() 

76 concepts: list[str] = [] 

77 for chunk in doc.noun_chunks: 

78 # Collapse wrapped lines: a noun chunk spanning a line break is the 

79 # same concept as its unwrapped form and must fold into one node. 

80 concept = collapse_whitespace(chunk.text.lower()) 

81 if not is_valid_label(concept): 

82 continue 

83 if concept in seen: 

84 continue 

85 seen.add(concept) 

86 concepts.append(concept) 

87 if len(concepts) >= max_concepts: 

88 break 

89 return concepts