Coverage for src/lilbee/core/text.py: 100%

1"""Label sanity checks and slug formatting.""" 

2 

3from __future__ import annotations 

4 

5import re 

6 

7_SLUG_CLEAN_RE = re.compile(r"[^a-z0-9-]") 

8_WHITESPACE_RE = re.compile(r"\s+") 

9 

10 

11def collapse_whitespace(text: str) -> str: 

12 """Fold every whitespace run to one space and strip the ends. 

13 

14 The one place this is defined. A label reaches a heading, a slug and a 

15 single-line marker comment, so producers collapse the surface they got 

16 before anything downstream sees it. 

17 """ 

18 return _WHITESPACE_RE.sub(" ", text).strip() 

19 

20 

21# Trailing possessive clitic on an entity surface: 's with a straight or 

22# curly (U+2019) apostrophe, or the bare apostrophe of a plural possessive 

23# (Voyagers'). Anchored to the end so a mid-label clitic (McDonald's 

24# Corporation) stays. 

25_POSSESSIVE_RE = re.compile("['\u2019]s?$") 

26 

27 

28def strip_possessive(label: str) -> str: 

29 """Drop a trailing possessive clitic from an entity surface form.""" 

30 return _POSSESSIVE_RE.sub("", label) 

31 

32 

33# Characters that signal markdown-structural noise in a concept label. 

34# Single source of truth for both ``is_valid_label`` (membership check) 

35# and ``clean_label_for_display`` (regex strip). 

36_STRUCTURAL_CHARS = frozenset("|#>") 

37_DISPLAY_STRUCTURAL_RE = re.compile(f"[{re.escape(''.join(_STRUCTURAL_CHARS))}]+") 

38 

39LABEL_SANITY_MIN_LEN = 3 

40LABEL_SANITY_MIN_ALNUM_RATIO = 0.5 

41 

42 

43def make_slug(label: str) -> str: 

44 """Turn a concept label into a filesystem-safe slug. 

45 

46 Lowercases, maps whitespace to single hyphens and slashes to double 

47 hyphens (path encoding), strips anything outside ``[a-z0-9-]``, and 

48 trims leading and trailing hyphens. Returns ``""`` when no sluggable 

49 characters remain; callers must treat an empty slug as "skip this 

50 entity" so the generator never writes a file called ``.md``. 

51 

52 Internal hyphen runs from the ``/`` path encoding are preserved; 

53 only leading and trailing hyphens (e.g. ``--body`` from a stripped 

54 ``| | Body``) are removed. 

55 """ 

56 # ``--`` is the reserved encoding for ``/``, so collapse whitespace first: 

57 # a double space would produce it and collide two entities onto one page. 

58 slug = collapse_whitespace(label.lower()) 

59 slug = slug.replace("/", "--").replace(" ", "-") 

60 slug = _SLUG_CLEAN_RE.sub("", slug) 

61 return slug.strip("-") 

62 

63 

64def is_valid_label(label: str) -> bool: 

65 """Reject structural-noise labels before aggregation. 

66 

67 Catches the noise patterns observed in QA (bb-8b7s): 

68 

69 - empty or sub-three-char fragments, 

70 - markdown table delimiters (``| | designer``), 

71 - page-number-prefixed tokens (``158 vehicle``), 

72 - paren-prefixed numerics (``(7.0 l)``: would otherwise slug to 

73 ``70-l`` after punctuation cleanup), 

74 - hyphen-prefixed fragments (``-answers``: trailing text from 

75 markdown bracket-link extraction), 

76 - labels carrying a line break. 

77 

78 The line-break rule is defensive only: no current caller can trip it. 

79 Both extractors run :func:`collapse_whitespace` first, which folds every 

80 separator to a space rather than rejecting the label, because a spaCy span 

81 crossing a wrapped line is the same entity as its unwrapped form and 

82 dropping it would lose the mention. The third caller, the heading check in 

83 :mod:`lilbee.wiki.quality`, takes its heading out of ``splitlines`` and so 

84 cannot produce one either. The rule stays because this is a shared 

85 validator and a future caller may hand over raw text. 

86 

87 Requires the first non-whitespace character to be a Unicode letter 

88 so any non-alpha prefix (digit, bracket, hyphen, punctuation) is 

89 rejected up front. Legitimate labels like ``E-mail`` or ``iPhone`` 

90 pass. Still permissive on three-char fragments like ``cro`` / 

91 ``fus``; A3's entity-type filter and ``wiki_entity_min_mentions`` 

92 catch those downstream. 

93 """ 

94 stripped = label.strip() 

95 if len(stripped) < LABEL_SANITY_MIN_LEN: 

96 return False 

97 if not stripped[0].isalpha(): 

98 return False 

99 # A label is one line: it becomes a heading, a slug, and part of the 

100 # single-line marker comments the drafts surface classifies by. Tested with 

101 # splitlines because that is what those readers use, so the gate and they 

102 # agree on what counts as a second line. A non-breaking or thin space does 

103 # not, and PDF text is full of both. 

104 if len(stripped.splitlines()) > 1: 

105 return False 

106 if any(ch in _STRUCTURAL_CHARS for ch in stripped): 

107 return False 

108 alnum = sum(1 for ch in stripped if ch.isalnum()) 

109 return alnum / len(stripped) >= LABEL_SANITY_MIN_ALNUM_RATIO 

110 

111 

112def clean_label_for_display(label: str) -> str: 

113 """Return a prompt-safe version of *label* for the ``{topic}`` slot. 

114 

115 Defense-in-depth behind :func:`is_valid_label`: a concept or entity 

116 label that reached this function already passed the sanity gate 

117 and should not contain ``|#>`` in practice. The structural-char 

118 strip here guards against a future code path that bypasses the 

119 gate (synthesis cluster labels sourced from ``concept_nodes``, 

120 user-supplied topics, tests). The always-useful work is whitespace 

121 normalization: spaCy surface forms can carry internal runs of 

122 whitespace that would reach the H1 verbatim. 

123 

124 Preserves the original capitalization so proper nouns 

125 (``Chevrolet Caprice``, ``iPhone``) survive intact; the model 

126 title-cases lowercase common nouns on its own. 

127 """ 

128 clean = _DISPLAY_STRUCTURAL_RE.sub("", label) 

129 return collapse_whitespace(clean)