Coverage for src/lilbee/core/text.py: 100%
41 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-11-06 08:17 +0000
1"""Label sanity checks and slug formatting."""
3from __future__ import annotations
5import re
7_SLUG_CLEAN_RE = re.compile(r"[^a-z0-9-]")
8_WHITESPACE_RE = re.compile(r"\s+")
9# ECMA-48 escape sequences: CSI, string commands (OSC and the like) up to their terminator,
10# and two-character escapes. A C1 byte stands for its ESC form.
11_ESCAPE_SEQUENCE_RE = re.compile(
12 r"(?:\x1b\[|\x9b)[0-?]*[ -/]*[@-~]"
13 r"|(?:\x1b[\]PX^_]|[\x90\x98\x9d\x9e\x9f])[^\x07\x1b\x9c]*(?:\x07|\x1b\\|\x9c)?"
14 r"|\x1b[ -~]"
15)
16_CONTROL_RE = re.compile(r"[\x00-\x1f\x7f-\x9f]")
17# The same set without TAB (0x09) and LF (0x0a), for text that spans lines.
18_CONTROL_KEEP_LINES_RE = re.compile(r"[\x00-\x08\x0b-\x1f\x7f-\x9f]")
21def collapse_whitespace(text: str) -> str:
22 """Fold every whitespace run to one space and strip the ends.
24 The one place this is defined. A label reaches a heading, a slug and a
25 single-line marker comment, so producers collapse the surface they got
26 before anything downstream sees it.
27 """
28 return _WHITESPACE_RE.sub(" ", text).strip()
31def strip_control_characters(text: str, *, keep_lines: bool = False) -> str:
32 """*text* without terminal escape sequences or any other control character.
34 A title or label is one line, so by default line breaks and tabs go too. With
35 *keep_lines* the text may span lines: tabs and line feeds stay, and CRLF becomes LF.
36 """
37 stripped = _ESCAPE_SEQUENCE_RE.sub("", text)
38 if keep_lines:
39 return _CONTROL_KEEP_LINES_RE.sub("", stripped.replace("\r\n", "\n"))
40 return _CONTROL_RE.sub("", stripped)
43# Trailing possessive clitic on an entity surface: 's with a straight or
44# curly (U+2019) apostrophe, or the bare apostrophe of a plural possessive
45# (Voyagers'). Anchored to the end so a mid-label clitic (McDonald's
46# Corporation) stays.
47_POSSESSIVE_RE = re.compile("['\u2019]s?$")
50def strip_possessive(label: str) -> str:
51 """Drop a trailing possessive clitic from an entity surface form."""
52 return _POSSESSIVE_RE.sub("", label)
55# Characters that signal markdown-structural noise in a concept label.
56# Single source of truth for both ``is_valid_label`` (membership check)
57# and ``clean_label_for_display`` (regex strip).
58_STRUCTURAL_CHARS = frozenset("|#>")
59_DISPLAY_STRUCTURAL_RE = re.compile(f"[{re.escape(''.join(_STRUCTURAL_CHARS))}]+")
61LABEL_SANITY_MIN_LEN = 3
62LABEL_SANITY_MIN_ALNUM_RATIO = 0.5
65def make_slug(label: str) -> str:
66 """Turn a concept label into a filesystem-safe slug.
68 Lowercases, maps whitespace to single hyphens and slashes to double
69 hyphens (path encoding), strips anything outside ``[a-z0-9-]``, and
70 trims leading and trailing hyphens. Returns ``""`` when no sluggable
71 characters remain; callers must treat an empty slug as "skip this
72 entity" so the generator never writes a file called ``.md``.
74 Internal hyphen runs from the ``/`` path encoding are preserved;
75 only leading and trailing hyphens (e.g. ``--body`` from a stripped
76 ``| | Body``) are removed.
77 """
78 # ``--`` is the reserved encoding for ``/``, so collapse whitespace first:
79 # a double space would produce it and collide two entities onto one page.
80 slug = collapse_whitespace(label.lower())
81 slug = slug.replace("/", "--").replace(" ", "-")
82 slug = _SLUG_CLEAN_RE.sub("", slug)
83 return slug.strip("-")
86def is_valid_label(label: str) -> bool:
87 """Reject structural-noise labels before aggregation.
89 Catches the noise patterns observed in QA (bb-8b7s):
91 - empty or sub-three-char fragments,
92 - markdown table delimiters (``| | designer``),
93 - page-number-prefixed tokens (``158 vehicle``),
94 - paren-prefixed numerics (``(7.0 l)``: would otherwise slug to
95 ``70-l`` after punctuation cleanup),
96 - hyphen-prefixed fragments (``-answers``: trailing text from
97 markdown bracket-link extraction),
98 - labels carrying a line break.
100 The line-break rule is defensive only: no current caller can trip it.
101 Both extractors run :func:`collapse_whitespace` first, which folds every
102 separator to a space rather than rejecting the label, because a spaCy span
103 crossing a wrapped line is the same entity as its unwrapped form and
104 dropping it would lose the mention. The third caller, the heading check in
105 :mod:`lilbee.wiki.quality`, takes its heading out of ``splitlines`` and so
106 cannot produce one either. The rule stays because this is a shared
107 validator and a future caller may hand over raw text.
109 Requires the first non-whitespace character to be a Unicode letter
110 so any non-alpha prefix (digit, bracket, hyphen, punctuation) is
111 rejected up front. Legitimate labels like ``E-mail`` or ``iPhone``
112 pass. Still permissive on three-char fragments like ``cro`` /
113 ``fus``; A3's entity-type filter and ``wiki_entity_min_mentions``
114 catch those downstream.
115 """
116 stripped = label.strip()
117 if len(stripped) < LABEL_SANITY_MIN_LEN:
118 return False
119 if not stripped[0].isalpha():
120 return False
121 # A label is one line: it becomes a heading, a slug, and part of the
122 # single-line marker comments the drafts surface classifies by. Tested with
123 # splitlines because that is what those readers use, so the gate and they
124 # agree on what counts as a second line. A non-breaking or thin space does
125 # not, and PDF text is full of both.
126 if len(stripped.splitlines()) > 1:
127 return False
128 if any(ch in _STRUCTURAL_CHARS for ch in stripped):
129 return False
130 alnum = sum(1 for ch in stripped if ch.isalnum())
131 return alnum / len(stripped) >= LABEL_SANITY_MIN_ALNUM_RATIO
134def clean_label_for_display(label: str) -> str:
135 """Return a prompt-safe version of *label* for the ``{topic}`` slot.
137 Defense-in-depth behind :func:`is_valid_label`: a concept or entity
138 label that reached this function already passed the sanity gate
139 and should not contain ``|#>`` in practice. The structural-char
140 strip here guards against a future code path that bypasses the
141 gate (synthesis cluster labels sourced from ``concept_nodes``,
142 user-supplied topics, tests). The always-useful work is whitespace
143 normalization: spaCy surface forms can carry internal runs of
144 whitespace that would reach the H1 verbatim.
146 Preserves the original capitalization so proper nouns
147 (``Chevrolet Caprice``, ``iPhone``) survive intact; the model
148 title-cases lowercase common nouns on its own.
149 """
150 clean = _DISPLAY_STRUCTURAL_RE.sub("", label)
151 return collapse_whitespace(clean)