Coverage for src/lilbee/wiki/citations.py: 100%
223 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-08-14 11:46 +0000
1"""Citation grammar and resolution for wiki pages.
3The grammar half parses ``[^srcN]`` footnote definitions out of wiki
4markdown, renders them back, strips the block from a body, and checks a
5single record's excerpt against the text it claims. The resolution half
6builds :class:`CitationRecord` rows from parsed markers (single-source
7and multi-source variants), matches each excerpt back to the source
8chunk it came from, and renders the YAML provenance block written into a
9wiki page's frontmatter.
10"""
12from __future__ import annotations
14import logging
15from collections import defaultdict
16from dataclasses import dataclass
17from datetime import UTC, datetime
18from enum import Enum
20import yaml
22from lilbee.core.config import Config
23from lilbee.data.store import CitationRecord, SearchChunk
24from lilbee.wiki.entity_extractor.factory import effective_entity_mode
25from lilbee.wiki.grammar import (
26 CHUNK_MARKER_RE,
27 CITATION_BLOCK_COMMENT,
28 CITATION_BLOCK_SEP,
29 CITE_RE,
30 CODE_FENCE_RE,
31 FOOTNOTE_RE,
32)
34log = logging.getLogger(__name__)
36# JSON-style escape sequences that may appear inside quoted excerpts the
37# model emits. Any backslash-prefixed character not in this map stays
38# verbatim (e.g. ``\\x`` passes through unchanged).
39_EXCERPT_ESCAPES: dict[str, str] = {"n": "\n", "t": "\t", '"': '"', "\\": "\\"}
41# Encoding side of the same table. A rendered footnote definition must stay on
42# one line and must re-parse to the excerpt it was rendered from.
43_EXCERPT_ESCAPE_INVERSE: dict[str, str] = {
44 char: f"\\{escape}" for escape, char in _EXCERPT_ESCAPES.items()
45}
48class CitationStatus(Enum):
49 """Result of verifying a citation against its source."""
51 VALID = "valid"
52 EXCERPT_MISSING = "excerpt_missing"
53 UNVERIFIABLE = "unverifiable"
56@dataclass(frozen=True)
57class ParsedCitation:
58 """A citation anchor extracted from wiki markdown."""
60 citation_key: str # e.g. "src1"
61 source_ref: str # human-readable ref, e.g. "python-docs/typing.md, lines 12-45"
62 line_number: int # 1-based line number in the markdown
65def _fence_flags(lines: list[str]) -> list[bool]:
66 """Per-line ``inside a code fence`` flags, fence delimiters included.
68 Footnote definitions and ``[^srcN]`` markers inside a fence are example
69 syntax on a page documenting the citation grammar, not citations.
70 """
71 flags: list[bool] = []
72 in_fence = False
73 for line in lines:
74 if CODE_FENCE_RE.match(line):
75 in_fence = not in_fence
76 flags.append(True)
77 continue
78 flags.append(in_fence)
79 return flags
82def parse_wiki_citations(markdown: str) -> list[ParsedCitation]:
83 """Extract citation footnote definitions from wiki markdown.
85 Scans the whole document outside code fences: the ``[^srcN]: ...`` pattern
86 unambiguously identifies a citation footnote wherever a model put it, so a
87 mid-body definition is a citation too, while a fenced one is example syntax.
88 A key is taken once: a mid-body definition repeated in the trailing block is
89 a single citation.
90 """
91 lines = markdown.splitlines()
93 citations: list[ParsedCitation] = []
94 seen: set[str] = set()
95 for line_idx, fenced in enumerate(_fence_flags(lines)):
96 if fenced:
97 continue
98 match = FOOTNOTE_RE.match(lines[line_idx])
99 if match and match.group(1) not in seen:
100 seen.add(match.group(1))
101 citations.append(
102 ParsedCitation(
103 citation_key=match.group(1),
104 source_ref=match.group(2).strip(),
105 line_number=line_idx + 1, # 1-based
106 )
107 )
108 return citations
111def render_citation_block(citations: list[CitationRecord]) -> str:
112 """Generate the markdown footnote footer from CitationRecord objects.
113 Returns the full citation block including separator and comment,
114 or an empty string when there are no citations.
115 """
116 if not citations:
117 return ""
118 lines = [CITATION_BLOCK_SEP, CITATION_BLOCK_COMMENT]
119 for rec in citations:
120 lines.append(f"[^{rec['citation_key']}]: {_format_source_ref(rec)}")
121 return "\n".join(lines) + "\n"
124def footnote_marker_keys(body: str) -> set[str]:
125 """Citation keys the body's ``[^srcN]`` markers reference, fences excluded.
127 Shares :func:`_fence_flags` with the parser and the scrubber: a marker the
128 other two treat as example syntax must not count as a section's citation.
129 """
130 lines = body.splitlines()
131 return {
132 key
133 for line, fenced in zip(lines, _fence_flags(lines), strict=True)
134 if not fenced
135 for key in CITE_RE.findall(line)
136 }
139def scrub_unverified_markers(body: str, verified: list[CitationRecord]) -> str:
140 """Drop in-body footnote definitions, unverified ``[^srcN]`` markers, and
141 ``[Chunk N]`` labels.
143 The citation block is re-rendered from the verified records, so any
144 definition line still inside the body would either duplicate a verified
145 definition or publish an unverified excerpt as prose. A marker whose
146 definition was dropped renders as literal ``[^srcN]`` text and hides the
147 claim from ``find_unmarked_claims``. A ``[Chunk N]`` label names prompt
148 evidence no reader can see, and stripping it returns the claim to uncited
149 prose rather than dressing it as a citation nothing verified. Fenced lines
150 are example syntax and stay verbatim.
151 """
152 keys = {rec["citation_key"] for rec in verified}
153 # keepends so removing a definition line does not also rewrite the body's
154 # own line endings or drop its trailing newline.
155 lines = body.splitlines(keepends=True)
156 kept = [
157 line if fenced else _scrub_line(line, keys)
158 for line, fenced in zip(lines, _fence_flags(lines), strict=True)
159 if fenced or not FOOTNOTE_RE.match(line)
160 ]
161 return "".join(kept)
164def _scrub_line(line: str, verified_keys: set[str]) -> str:
165 """One unfenced body line with unverified markers and chunk labels removed."""
166 line = CITE_RE.sub(lambda m: m.group(0) if m.group(1) in verified_keys else "", line)
167 return CHUNK_MARKER_RE.sub("", line)
170def wiki_sourced_count(records: list[CitationRecord], config: Config) -> int:
171 """Number of *records* citing a wiki page rather than a raw source.
173 :func:`verify_citations` skips these, so they are neither rendered nor
174 dropped as unverified.
175 """
176 return sum(1 for rec in records if _is_wiki_sourced(rec, config))
179def _is_wiki_sourced(record: CitationRecord, config: Config) -> bool:
180 """Whether a citation names a wiki page as its source."""
181 return record["source_filename"].startswith(config.wiki_dir + "/")
184def excerpt_in_chunks(excerpt: str, chunk_texts: list[str]) -> bool:
185 """Single rule for excerpt presence, shared by generation and lint.
187 The excerpt must be present in ONE chunk: a quote stitched across a chunk
188 boundary belongs to no source passage. An empty excerpt is never present.
189 """
190 from xberg import verify_excerpt
192 return bool(excerpt) and any(verify_excerpt(excerpt, text) for text in chunk_texts)
195def verify_citation(citation: CitationRecord, chunk_texts: list[str]) -> CitationStatus:
196 """Check whether a citation's excerpt exists in the source's extracted chunks.
198 Returns ``UNVERIFIABLE`` when the source has no extracted text to check
199 against. Does not check hash staleness or source existence: caller handles
200 those by comparing ``citation.source_hash`` against the current file hash
201 and checking file presence.
202 """
203 if not chunk_texts:
204 return CitationStatus.UNVERIFIABLE
205 if excerpt_in_chunks(citation["excerpt"], chunk_texts):
206 return CitationStatus.VALID
207 return CitationStatus.EXCERPT_MISSING
210def find_unmarked_claims(markdown: str) -> list[str]:
211 """Find body statements that are neither cited ``[^srcN]`` nor marked ``[*inference*]``.
213 Delegates to xberg's footnote/citation API over the body (frontmatter and the
214 citation block stripped).
215 """
216 from xberg import find_unmarked_claims as _find_unmarked_claims
218 return _find_unmarked_claims(extract_body(markdown))
221def strip_citation_block(markdown: str) -> str:
222 """Remove the citation block (separator + comment + footnotes) from markdown."""
223 lines = markdown.splitlines()
224 body_end = _body_end(lines)
225 if body_end == len(lines):
226 return markdown
227 return "\n".join(lines[:body_end]).rstrip() + "\n"
230def _find_citation_block_start(lines: list[str]) -> int | None:
231 """Return the 0-based line index where the citation block begins, or None."""
232 for i, line in enumerate(lines):
233 if line.strip() == CITATION_BLOCK_COMMENT:
234 return i
235 return None
238def _body_end(lines: list[str]) -> int:
239 """Return the line index the body ends at, before any citation block.
241 Without the auto-generated comment only the document's trailing run of
242 ``[^srcN]:`` definitions is a block; definitions followed by prose stay
243 in the body.
244 """
245 block_start = _find_citation_block_start(lines)
246 if block_start is None:
247 return _body_end_before_trailing_footnotes(lines)
248 return _drop_separator(lines, block_start)
251def _body_end_before_trailing_footnotes(lines: list[str]) -> int:
252 """Return the line index before the document's trailing run of ``[^srcN]:`` definitions."""
253 body_end = len(lines)
254 found = False
255 while body_end > 0:
256 line = lines[body_end - 1]
257 if FOOTNOTE_RE.match(line):
258 found = True
259 elif line.strip():
260 break
261 body_end -= 1
262 if not found:
263 return len(lines)
264 return _drop_separator(lines, body_end)
267def _drop_separator(lines: list[str], body_end: int) -> int:
268 """Return *body_end* less a preceding ``---`` separator line, if there is one."""
269 if body_end > 0 and lines[body_end - 1].strip() == CITATION_BLOCK_SEP:
270 return body_end - 1
271 return body_end
274def extract_body(markdown: str) -> str:
275 """Return markdown body: strip YAML frontmatter and citation block."""
276 text = _strip_frontmatter(markdown)
277 lines = text.splitlines()
278 body_end = _body_end(lines)
279 if body_end == len(lines):
280 return text
281 return "\n".join(lines[:body_end])
284def _strip_frontmatter(markdown: str) -> str:
285 """Remove YAML frontmatter delimited by ``---`` at the start."""
286 if not markdown.startswith("---"):
287 return markdown
288 lines = markdown.splitlines()
289 for i in range(1, len(lines)):
290 if lines[i].strip() == "---":
291 return "\n".join(lines[i + 1 :])
292 return markdown
295def _format_source_ref(rec: CitationRecord) -> str:
296 """Format a CitationRecord into a human-readable footnote reference."""
297 ref = rec["source_filename"]
298 has_page = rec["page_start"] is not None and rec["page_start"] > 0
299 has_page_end = rec["page_end"] is not None and rec["page_end"] > 0
300 has_line = rec["line_start"] is not None and rec["line_start"] > 0
301 has_line_end = rec["line_end"] is not None and rec["line_end"] > 0
302 if has_page or has_page_end:
303 if rec["page_start"] == rec["page_end"]:
304 ref += f", page {rec['page_start']}"
305 else:
306 ref += f", pages {rec['page_start']}-{rec['page_end']}"
307 elif has_line or has_line_end:
308 ref += f", lines {rec['line_start']}-{rec['line_end']}"
309 if rec["excerpt"]:
310 ref += f', excerpt: "{_encode_excerpt_escapes(rec["excerpt"])}"'
311 return ref
314def _extract_excerpt(source_ref: str) -> str:
315 """Extract the quoted excerpt from a citation source_ref string.
316 e.g. 'doc.md, excerpt: "Python supports typing."' → 'Python supports typing.'
318 Common JSON-style escape sequences inside the quoted span (``\\n``,
319 ``\\t``, ``\\"``, ``\\\\``) are decoded to their literal characters so
320 they round-trip against the source text. Some models "helpfully"
321 encode real newlines as ``\\n`` when emitting a quoted excerpt; the
322 source chunk they came from has real newlines, so skipping this
323 step leaves otherwise-faithful citations unverifiable.
324 """
325 marker = 'excerpt: "'
326 idx = source_ref.find(marker)
327 if idx == -1:
328 return ""
329 start = idx + len(marker)
330 end = _find_closing_quote(source_ref, start)
331 raw = source_ref[start:].strip() if end == -1 else source_ref[start:end].strip()
332 return _decode_excerpt_escapes(raw)
335def _find_closing_quote(text: str, start: int) -> int:
336 """Index of the first unescaped ``"`` at or after *start*, or -1.
338 An escaped quote belongs to the excerpt, so the scan steps over it rather
339 than ending the quoted span there.
340 """
341 i = start
342 while i < len(text):
343 if text[i] == "\\":
344 i += 2
345 continue
346 if text[i] == '"':
347 return i
348 i += 1
349 return -1
352def _encode_excerpt_escapes(text: str) -> str:
353 """Escape *text* so :func:`_decode_excerpt_escapes` returns it unchanged."""
354 return "".join(_EXCERPT_ESCAPE_INVERSE.get(char, char) for char in text)
357def _decode_excerpt_escapes(raw: str) -> str:
358 """Decode the JSON-style escapes models commonly emit inside quoted strings."""
359 if "\\" not in raw:
360 return raw
361 result: list[str] = []
362 i = 0
363 while i < len(raw):
364 ch = raw[i]
365 mapped = _EXCERPT_ESCAPES.get(raw[i + 1]) if ch == "\\" and i + 1 < len(raw) else None
366 if mapped is not None:
367 result.append(mapped)
368 i += 2
369 else:
370 result.append(ch)
371 i += 1
372 return "".join(result)
375def _find_excerpt_location(
376 excerpt: str,
377 chunks: list[SearchChunk],
378) -> tuple[int, int, int, int]:
379 """Find page/line location of an excerpt within chunks.
381 Matches with the verification rule, so a citation that verifies keeps
382 its location.
383 """
384 for chunk in chunks:
385 if excerpt_in_chunks(excerpt, [chunk.chunk]):
386 return chunk.page_start, chunk.page_end, chunk.line_start, chunk.line_end
387 return 0, 0, 0, 0
390def _build_citation_record(
391 citation_key: str,
392 excerpt: str,
393 source_filename: str,
394 source_hash: str,
395 page_start: int,
396 page_end: int,
397 line_start: int,
398 line_end: int,
399 created_at: str,
400) -> CitationRecord:
401 """Build a single CitationRecord with consistent defaults."""
402 return CitationRecord(
403 wiki_source="", # filled by caller
404 wiki_chunk_index=0,
405 citation_key=citation_key,
406 # A footnote definition is always a fact claim; an inference carries none.
407 claim_type="fact",
408 source_filename=source_filename,
409 source_hash=source_hash,
410 page_start=page_start,
411 page_end=page_end,
412 line_start=line_start,
413 line_end=line_end,
414 excerpt=excerpt,
415 created_at=created_at,
416 )
419def verify_citations(
420 citation_records: list[CitationRecord],
421 chunks: list[SearchChunk],
422 label: str,
423 config: Config,
424) -> list[CitationRecord]:
425 """Filter citation records, keeping only those whose excerpts are in their own source.
427 Each record is checked against the chunks of the source it names, the rule
428 lint and draft-accept apply. Checking against the whole pool would pass a
429 footnote that attributes source B a quote only source A carries, and publish
430 it with B's hash and no location. A footnote left without a quotable excerpt
431 is unverified and dropped.
432 """
433 chunk_texts_by_source: dict[str, list[str]] = defaultdict(list)
434 for chunk in chunks:
435 chunk_texts_by_source[chunk.source].append(chunk.chunk)
436 verified: list[CitationRecord] = []
437 for rec in citation_records:
438 if _is_wiki_sourced(rec, config):
439 log.debug("Skipping wiki-sourced citation %s", rec["citation_key"])
440 continue
441 if excerpt_in_chunks(rec["excerpt"], chunk_texts_by_source[rec["source_filename"]]):
442 verified.append(rec)
443 else:
444 log.debug(
445 "Citation %s excerpt not found in %s (%s), dropping",
446 rec["citation_key"],
447 rec["source_filename"],
448 label,
449 )
450 return verified
453def render_provenance(config: Config, chunks: list[SearchChunk]) -> str:
454 """Render the provenance block: chunk references + extraction method.
456 Uses ``yaml.safe_dump`` so a chunk source containing a quote, backslash,
457 colon, or newline cannot produce invalid YAML that ``parse_frontmatter``
458 would silently drop on read.
459 """
460 block = {
461 "provenance": {
462 # Record the extractor that actually runs (config mode may fall back),
463 # so the audit reflects reality, not the requested setting.
464 "extraction_method": effective_entity_mode(config.wiki_entity_mode).value,
465 "chunks": [{"source": c.source, "chunk_index": c.chunk_index} for c in chunks],
466 }
467 }
468 return yaml.safe_dump(block, sort_keys=False)
471def resolve_multi_source_citations(
472 parsed_citations: list[ParsedCitation],
473 source_names: list[str],
474 source_hashes: dict[str, str],
475 chunks_by_source: dict[str, list[SearchChunk]],
476) -> list[CitationRecord]:
477 """Resolve citations from a synthesis page that cites multiple sources.
479 Each citation's source_ref is matched against the source list to determine
480 which source document it references. A citation that names no listed source
481 and whose excerpt is in none of them is dropped, not attributed to an
482 arbitrary source.
483 """
484 records: list[CitationRecord] = []
485 now = datetime.now(UTC).isoformat()
487 for parsed in parsed_citations:
488 excerpt = _extract_excerpt(parsed.source_ref)
490 matched_source = _match_citation_source(
491 parsed.source_ref, source_names
492 ) or _find_excerpt_source(excerpt, chunks_by_source)
493 if not matched_source:
494 log.warning(
495 "Dropping citation %s: no source matches %r",
496 parsed.citation_key,
497 parsed.source_ref,
498 )
499 continue
501 search_chunks = chunks_by_source.get(matched_source, [])
502 page_start, page_end, line_start, line_end = _find_excerpt_location(excerpt, search_chunks)
503 records.append(
504 _build_citation_record(
505 parsed.citation_key,
506 excerpt,
507 matched_source,
508 source_hashes.get(matched_source, ""),
509 page_start,
510 page_end,
511 line_start,
512 line_end,
513 now,
514 )
515 )
516 return records
519def _match_citation_source(source_ref: str, source_names: list[str]) -> str:
520 """Find which source a citation references by matching filenames in the ref.
522 Checks longest names first so a filename that is a substring of another
523 (e.g. ``doc.md`` within ``mydoc.md``) can't shadow the more specific match.
524 """
525 for name in sorted(source_names, key=len, reverse=True):
526 if name in source_ref:
527 return name
528 return ""
531def _find_excerpt_source(excerpt: str, chunks_by_source: dict[str, list[SearchChunk]]) -> str:
532 """Find which source contains a given excerpt, matching as verification does."""
533 for source, chunks in chunks_by_source.items():
534 if excerpt_in_chunks(excerpt, [c.chunk for c in chunks]):
535 return source
536 return ""