Coverage for src/lilbee/wiki/drafts.py: 100%
215 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-08-14 11:46 +0000
1"""Draft review surface. List, diff, accept, reject wiki drafts.
3Wiki generation routes pages to ``wiki/drafts/`` when the content
4drift against an existing page exceeds the configured threshold or
5when the faithfulness score falls below it. Without a review
6surface drafts accumulate with no exit ramp, so this module exposes
7the four operations a reviewer needs: see what is pending, diff
8against the published version, accept (publish the page, register its
9citations, re-index its chunks), or reject (delete the draft file).
10"""
12from __future__ import annotations
14import difflib
15import logging
16import re
17from dataclasses import dataclass
18from pathlib import Path
19from typing import Any
21from lilbee.core.config import Config, cfg
22from lilbee.core.security import validate_path_within
23from lilbee.data.store import CitationRecord, SearchChunk, Store
24from lilbee.wiki.batch import hash_existing_sources
25from lilbee.wiki.citations import (
26 CitationStatus,
27 parse_wiki_citations,
28 render_citation_block,
29 resolve_multi_source_citations,
30 scrub_unverified_markers,
31 strip_citation_block,
32 verify_citation,
33)
34from lilbee.wiki.generation import rewrite_links_across_wiki
35from lilbee.wiki.index import update_wiki_index
36from lilbee.wiki.page import index_wiki_page, indexable_chunks
37from lilbee.wiki.shared import (
38 PENDING_COLLISION_MARKER_RE,
39 PENDING_PARSE_MARKER_RE,
40 WIKI_BUILD_LOCK,
41 WIKI_CONTENT_SUBDIRS,
42 PendingKind,
43 WikiSubdir,
44 atomic_write_text,
45 parse_frontmatter,
46)
48__all__ = [
49 "AcceptResult",
50 "BodylessDraftError",
51 "DraftAcceptError",
52 "DraftInfo",
53 "PendingKind",
54 "StaleDraftError",
55 "UnverifiedDraftError",
56 "accept_draft",
57 "diff_draft",
58 "list_drafts",
59 "reject_draft",
60]
62log = logging.getLogger(__name__)
64_DRIFT_MARKER_RE = re.compile(
65 # Keeps the stricter tail: this marker interpolates only a percentage, a
66 # subdir name and a hex hash, never a raw source name that could hold ">".
67 r"<!--\s*DRIFT:\s*(?P<pct>\d+)%\s*content changed[^>]*-->",
68 re.IGNORECASE,
69)
71# Pending-marker patterns come from wiki.shared, which owns the wording.
73# Published wiki subdirs searched in priority order when pairing a
74# draft slug with its counterpart. Summaries and synthesis come first
75# because they are the subdirs most drafts originate from (drift
76# detection runs on regen of an existing source or cluster page).
77_PUBLISHED_SUBDIRS: tuple[str, ...] = (
78 WikiSubdir.SUMMARIES,
79 WikiSubdir.SYNTHESIS,
80 WikiSubdir.CONCEPTS,
81 WikiSubdir.ENTITIES,
82)
85class DraftAcceptError(ValueError):
86 """Base for the refusals that stop a draft from being published."""
89class StaleDraftError(DraftAcceptError):
90 """Raised when a draft's published counterpart is newer than the draft itself."""
93class UnverifiedDraftError(DraftAcceptError):
94 """Raised when none of a draft's citations survive verification."""
97class UnindexedDraftError(DraftAcceptError):
98 """Raised when publishing a draft produced no chunk rows."""
101class BodylessDraftError(DraftAcceptError):
102 """Raised when a draft's body would index nothing.
104 Covers an empty body and a body whose markup produces no indexable text,
105 such as a bare "#", a horizontal rule, or an image. A heading with words in
106 it indexes normally.
107 """
110@dataclass
111class DraftInfo:
112 """Metadata about a single draft, surfaced in ``wiki drafts list``.
114 ``pending_kind`` distinguishes drift drafts (None) from
115 batched-generation markers (``"parse"``, ``"collision"``). Callers
116 can render the kind in the list view and branch on it when
117 deciding how to surface the draft (e.g. a collision needs the
118 winning-source context, a parse marker just needs a rerun).
119 """
121 slug: str
122 path: Path
123 drift_ratio: float | None
124 faithfulness_score: float | None
125 bad_title: bool
126 published_path: Path | None
127 mtime: float
128 pending_kind: str | None = None
130 @property
131 def published_exists(self) -> bool:
132 """True when a matching published page exists for this draft."""
133 return self.published_path is not None
135 def to_dict(self) -> dict[str, Any]:
136 """Serialize to a JSON-friendly dict."""
137 return {
138 "slug": self.slug,
139 "path": str(self.path),
140 "drift_ratio": self.drift_ratio,
141 "faithfulness_score": self.faithfulness_score,
142 "bad_title": self.bad_title,
143 "published_path": str(self.published_path) if self.published_path else None,
144 "published_exists": self.published_exists,
145 "mtime": self.mtime,
146 "pending_kind": self.pending_kind,
147 }
150@dataclass
151class AcceptResult:
152 """Outcome of accepting a draft. Returned so callers can confirm.
154 ``requested_slug`` is always the slug the caller asked to accept
155 (for PENDING-COLLISION drafts this looks like
156 ``brakes-collision-abc12345``). ``slug`` is where the content
157 landed (the de-collisioned base slug, so ``brakes``). For
158 non-collision drafts the two match. HTTP clients that round-trip
159 accept→list-refresh can compare both fields to track the rename.
160 """
162 slug: str
163 requested_slug: str
164 moved_to: Path
165 reindexed_chunks: int
167 def to_dict(self) -> dict[str, Any]:
168 """Serialize to a JSON-friendly dict for HTTP/MCP/CLI responses."""
169 return {
170 "slug": self.slug,
171 "requested_slug": self.requested_slug,
172 "moved_to": self.moved_to.as_posix(),
173 "reindexed_chunks": self.reindexed_chunks,
174 }
177def _draft_path(wiki_root: Path, slug: str) -> Path:
178 """Resolve a draft slug to a path, rejecting traversal outside the drafts dir.
180 The slug reaches here straight from a ``{slug:path}`` HTTP route and the
181 MCP tool, so an unvalidated ``..`` would let accept/reject/diff read,
182 overwrite, or delete arbitrary ``.md`` files. Mirrors browse.find_page.
183 """
184 drafts_root = wiki_root / WikiSubdir.DRAFTS
185 candidate = drafts_root / f"{slug}.md"
186 validate_path_within(candidate, drafts_root)
187 return candidate
190def _find_published(wiki_root: Path, slug: str) -> Path | None:
191 """Return the first published page matching *slug*, or None.
193 Checks summaries, synthesis, concepts, and entities subdirs in
194 priority order so a draft regenerated from an existing summary
195 page pairs with its original rather than the same slug under a
196 different page type. Rejects a traversal slug rather than reading
197 a matching file outside the wiki tree.
198 """
199 for subdir in _PUBLISHED_SUBDIRS:
200 candidate = wiki_root / subdir / f"{slug}.md"
201 validate_path_within(candidate, wiki_root)
202 if candidate.is_file():
203 return candidate
204 return None
207_ORIGIN_MARKER_RE = re.compile(
208 # The head stays greedy so the LAST "origin:" wins: a collision marker
209 # interpolates raw source names ahead of the real field, and a filename can
210 # contain the literal "origin:". The tail is non-greedy so a ">" inside the
211 # comment does not stop the match.
212 r"<!--.*origin:\s*(?P<subdir>\w+).*?-->",
213 re.IGNORECASE,
214)
216_CONTENT_SUBDIR_BY_VALUE = {s.value: s for s in WIKI_CONTENT_SUBDIRS}
219def _parse_drift_ratio(text: str) -> float | None:
220 """Extract the drift percentage from a draft's leading marker."""
221 match = _DRIFT_MARKER_RE.search(text)
222 if match is None:
223 return None
224 return int(match.group("pct")) / 100.0
227def _parse_origin_subdir(text: str) -> WikiSubdir | None:
228 """Extract the origin page-type subdir from a marker run, if it names a valid one.
230 Drift and collision markers both carry ``origin: <subdir>`` so an unpaired
231 draft accepts back into its own page type. Returns None for drafts without
232 the field (markers written before this was recorded) or values outside the
233 content subdirs, so the caller keeps the summaries fallback.
234 """
235 match = _ORIGIN_MARKER_RE.search(text)
236 if match is None:
237 return None
238 return _CONTENT_SUBDIR_BY_VALUE.get(match.group("subdir").lower())
241def _parse_pending_kind(text: str) -> str | None:
242 """Classify *text* as a PENDING-PARSE, PENDING-COLLISION, or neither.
244 Returns ``None`` when the leading line is not a PENDING marker. Markers
245 are always written as the first line, so a draft body that quotes a
246 marker comment further down does not get mis-classified.
247 """
248 first_line = text.splitlines()[0] if text else ""
249 if PENDING_PARSE_MARKER_RE.match(first_line):
250 return PendingKind.PARSE
251 if PENDING_COLLISION_MARKER_RE.match(first_line):
252 return PendingKind.COLLISION
253 return None
256def _is_marker_line(line: str) -> bool:
257 return any(
258 pattern.match(line)
259 for pattern in (
260 PENDING_PARSE_MARKER_RE,
261 PENDING_COLLISION_MARKER_RE,
262 _DRIFT_MARKER_RE,
263 _ORIGIN_MARKER_RE,
264 )
265 )
268def _split_marker_line(text: str) -> tuple[str, str]:
269 """Split *text* into its leading run of marker lines and the untouched remainder.
271 A drift that also collides stacks a PENDING marker above the DRIFT note, so
272 the whole leading run is consumed: kind comes from the first marker line,
273 drift ratio and origin from any of them. The run stops at the first
274 non-marker content, so a marker comment quoted in the body is never parsed
275 or stripped.
276 """
277 lines = text.split("\n")
278 markers: list[str] = []
279 index = 0
280 while index < len(lines):
281 line = lines[index]
282 if _is_marker_line(line):
283 markers.append(line)
284 elif line.strip() or not markers:
285 break
286 index += 1
287 if not markers:
288 return "", text
289 return "\n".join(markers), "\n".join(lines[index:])
292def _classify_and_strip_markers(text: str) -> tuple[str | None, float | None, str]:
293 """Single-pass read: parse kind, drift ratio, and return marker-stripped body.
295 Classification, drift ratio, and origin all come from the leading run of
296 marker lines, and stripping removes only that run, so a marker comment
297 quoted in the body survives the accept untouched.
298 """
299 marker_line, remainder = _split_marker_line(text)
300 pending_kind = _parse_pending_kind(marker_line)
301 drift = _parse_drift_ratio(marker_line)
302 return pending_kind, drift, remainder.lstrip() if marker_line else remainder
305def list_drafts(wiki_root: Path) -> list[DraftInfo]:
306 """Return one ``DraftInfo`` per draft markdown file under ``drafts/``.
308 Recurses so per-source draft nesting (``drafts/<source>/page.md``)
309 is covered. Reads each draft's full text once, classifies any
310 pending marker and drift ratio, strips the markers, then parses
311 frontmatter on the stripped body (so frontmatter parsing works
312 uniformly whether or not a marker shifted it down).
313 """
314 drafts_dir = wiki_root / WikiSubdir.DRAFTS
315 if not drafts_dir.is_dir():
316 return []
317 infos: list[DraftInfo] = []
318 for path in sorted(drafts_dir.rglob("*.md")):
319 text = path.read_text(encoding="utf-8")
320 pending_kind, drift, stripped = _classify_and_strip_markers(text)
321 fm = parse_frontmatter(stripped)
322 slug = str(path.relative_to(drafts_dir).with_suffix("")).replace("\\", "/")
323 infos.append(
324 DraftInfo(
325 slug=slug,
326 path=path,
327 drift_ratio=drift,
328 faithfulness_score=_coerce_float(fm.get("faithfulness_score")),
329 bad_title=bool(fm.get("bad_title", False)),
330 published_path=_find_published(wiki_root, slug),
331 mtime=path.stat().st_mtime,
332 pending_kind=pending_kind,
333 )
334 )
335 return infos
338def diff_draft(slug: str, wiki_root: Path) -> str:
339 """Return a unified diff of the draft against its published counterpart.
341 Raises :class:`FileNotFoundError` when the draft does not exist.
342 When no published counterpart exists the diff shows the draft as
343 all-new (baseline empty), which is useful for reviewing drafts
344 that originated from a fresh low-faithfulness generation.
345 """
346 draft = _draft_path(wiki_root, slug)
347 if not draft.is_file():
348 raise FileNotFoundError(f"draft not found: {slug}")
349 draft_text = draft.read_text(encoding="utf-8")
350 published = _find_published(wiki_root, slug)
351 baseline = published.read_text(encoding="utf-8") if published else ""
352 diff = difflib.unified_diff(
353 baseline.splitlines(),
354 draft_text.splitlines(),
355 fromfile=str(published) if published else "(new draft)",
356 tofile=str(draft),
357 lineterm="",
358 )
359 return "\n".join(diff)
362_COLLISION_SUFFIX_RE = re.compile(r"-collision-[0-9a-f]{8}$")
365def _base_slug_for_collision(slug: str) -> str:
366 """Strip the ``-collision-<hash>`` suffix so accept lands on the winning slug."""
367 return _COLLISION_SUFFIX_RE.sub("", slug)
370def accept_draft(
371 slug: str, wiki_root: Path, store: Store, config: Config | None = None
372) -> AcceptResult:
373 """Publish the draft, register its citations, and re-index its chunks.
375 Behavior branches on the draft's pending kind:
377 - **Drift draft** (default): write the accepted body to its
378 published counterpart (or ``summaries/`` when unpaired),
379 re-index, delete the draft.
380 - **PENDING-PARSE** (batched-generation parser could not recover
381 a section): accepting is a no-op on the published side: the
382 marker has no body to accept. The marker is deleted and the
383 user is told to run ``wiki build`` to regenerate. Returns an
384 ``AcceptResult`` with ``reindexed_chunks=0`` and
385 ``moved_to`` pointing at the deleted marker.
386 - **PENDING-COLLISION** (two sources proposed the same concept
387 slug): strips the ``-collision-<hash>`` suffix to find the
388 winning slug, overwrites the winning page with this draft's
389 body, re-indexes, deletes the collision marker.
391 Drafts carry no store state, so accept is where a page's citation
392 rows are created: the citation block embedded in the draft body is
393 re-parsed, verified against the chunks of the sources its
394 frontmatter names, and written under the published ``wiki_source``.
395 The published body is rendered from that same set, so its footnotes
396 and its rows cannot disagree.
398 Sequence for drift/collision: write the published file first,
399 register citations and re-index next, delete the draft last. If a
400 later step raises (chunker, embedder, LanceDB contention), the
401 draft file stays on disk so the user can retry ``accept``: both
402 the citation replace and ``index_wiki_page`` are idempotent on the
403 same ``wiki_source``. A body that indexed nothing is a failed step
404 too: an accepted page always chunks to at least one row.
406 Raises :class:`FileNotFoundError` when the draft does not exist,
407 :class:`StaleDraftError` when the published counterpart is newer,
408 :class:`UnverifiedDraftError` when no cited excerpt is still in its source,
409 :class:`BodylessDraftError` when the draft's body would index nothing, and
410 :class:`UnindexedDraftError` when the store write landed no rows anyway.
412 Holds the wiki build mutex while publishing, so accepting a draft cannot
413 interleave with a build, synthesis, or prune from another surface.
414 """
415 if config is None:
416 config = cfg
417 with WIKI_BUILD_LOCK:
418 draft = _draft_path(wiki_root, slug)
419 if not draft.is_file():
420 raise FileNotFoundError(f"draft not found: {slug}")
421 raw = draft.read_text(encoding="utf-8")
422 # Single-pass classify + strip (kind plus the three-marker removal), instead
423 # of re-deriving the kind and re-stripping the markers separately.
424 pending_kind, _drift, clean = _classify_and_strip_markers(raw)
426 if pending_kind == PendingKind.PARSE:
427 draft.unlink()
428 log.info(
429 "Accepted PENDING-PARSE marker %s; run `lilbee wiki build` "
430 "to regenerate the missing section.",
431 slug,
432 )
433 return AcceptResult(slug=slug, requested_slug=slug, moved_to=draft, reindexed_chunks=0)
435 target_slug = (
436 _base_slug_for_collision(slug) if pending_kind == PendingKind.COLLISION else slug
437 )
438 target = _accept_target(wiki_root, target_slug, slug, raw)
439 wiki_source = _wiki_source_for(target, wiki_root, config)
440 records = _accepted_citations(clean, wiki_source, slug, store)
441 content = _render_accepted_page(clean, records)
442 _refuse_stale_draft(target, draft, slug, content)
443 chunks = _refuse_bodyless_draft(content, slug)
445 atomic_write_text(target, content)
446 store.replace_citations_for_wiki(wiki_source, records)
447 reindexed = index_wiki_page(content, wiki_source, store, config, chunks)
448 if not reindexed:
449 # Backstop on the store write. The accept-time guard already
450 # refused every body that chunks to nothing, so no production input
451 # reaches this branch; its only test mocks the indexer.
452 raise UnindexedDraftError(
453 f"draft {slug} published no searchable chunks: the page was "
454 "written but the index write did not land. The draft is kept; "
455 "re-run accept once the index is writable"
456 )
457 update_wiki_index(config)
458 # Same link pass a build runs. An accepted draft is a published page,
459 # and without this it arrives with no [[links]] and sits alone in the
460 # graph, the way a page written on request did before #710.
461 rewrite_links_across_wiki([], config, wiki_root)
462 draft.unlink()
463 log.info("Accepted draft %s -> %s (%d chunks indexed)", slug, target, reindexed)
464 return AcceptResult(
465 slug=target_slug,
466 requested_slug=slug,
467 moved_to=target,
468 reindexed_chunks=reindexed,
469 )
472def _accept_target(wiki_root: Path, target_slug: str, slug: str, raw: str) -> Path:
473 """Resolve where an accepted draft lands."""
474 published = _find_published(wiki_root, target_slug)
475 if published is not None:
476 return published
477 marker_line, _remainder = _split_marker_line(raw)
478 fallback_subdir = _parse_origin_subdir(marker_line) or WikiSubdir.SUMMARIES
479 log.info("Draft %s has no published counterpart; accepting into %s", slug, fallback_subdir)
480 return wiki_root / fallback_subdir / f"{target_slug}.md"
483def _refuse_bodyless_draft(content: str, slug: str) -> list[str]:
484 """Refuse a draft whose body would index nothing.
486 Checked before the published page is overwritten, and against what the
487 indexer actually chunks: a body can be non-empty and still chunk to nothing
488 ("#", "---"). Indexing such a page clears the rows of whatever it replaced,
489 and the old order reported that as an index failure, so every retry
490 destroyed the published page again and could never succeed.
492 Returns the chunks so the caller indexes without chunking the same body a
493 second time inside the build lock.
494 """
495 chunks = indexable_chunks(content)
496 if not chunks:
497 raise BodylessDraftError(
498 f"draft {slug} has nothing to index: its body produces no searchable "
499 "text; reject it or edit the draft to add content"
500 )
501 return chunks
504def _refuse_stale_draft(target: Path, draft: Path, slug: str, content: str) -> None:
505 """Refuse a draft a later build has already outrun.
507 A published counterpart newer than the draft is a regenerated page the
508 older proposal would overwrite. Identical content is accept's own earlier
509 write: a retry after a failed citation or index step, which must finish.
510 """
511 if not target.is_file():
512 return
513 if target.read_text(encoding="utf-8") == content:
514 return
515 if target.stat().st_mtime > draft.stat().st_mtime:
516 raise StaleDraftError(
517 f"draft {slug} is older than the published page it would overwrite; "
518 "reject it and re-run `lilbee wiki build`"
519 )
522def _accepted_citations(
523 content: str, wiki_source: str, slug: str, store: Store
524) -> list[CitationRecord]:
525 """Citation rows for an accepted draft, verified against the store's chunks.
527 Follows the same rule as lint: a cited source the store holds no chunks
528 for was verified at build time and keeps its records, while a record whose
529 excerpt is absent from chunks that ARE present is dropped. A draft whose
530 citations all fail would publish provenance the store cannot back, so
531 accept refuses it.
532 """
533 parsed = parse_wiki_citations(content)
534 source_names = _frontmatter_sources(content)
535 chunks_by_source = {name: store.get_chunks_by_source(name) for name in source_names}
536 records = resolve_multi_source_citations(
537 parsed,
538 source_names,
539 hash_existing_sources(source_names),
540 chunks_by_source,
541 )
542 kept = [rec for rec in records if _keeps_provenance(rec, chunks_by_source, slug)]
543 if parsed and not kept:
544 raise UnverifiedDraftError(
545 f"draft {slug} has no citation whose excerpt is still in its source; "
546 "reject it and re-run `lilbee wiki build`"
547 )
548 for rec in kept:
549 rec["wiki_source"] = wiki_source
550 return kept
553def _keeps_provenance(
554 rec: CitationRecord, chunks_by_source: dict[str, list[SearchChunk]], slug: str
555) -> bool:
556 """Whether an accepted draft's citation record survives re-verification."""
557 chunk_texts = [c.chunk for c in chunks_by_source.get(rec["source_filename"], [])]
558 if verify_citation(rec, chunk_texts) is not CitationStatus.EXCERPT_MISSING:
559 return True
560 log.warning(
561 "Dropping citation %s from draft %s: excerpt no longer in %s",
562 rec["citation_key"],
563 slug,
564 rec["source_filename"],
565 )
566 return False
569def _render_accepted_page(content: str, records: list[CitationRecord]) -> str:
570 """Rebuild the page body around the citations that persisted."""
571 body = scrub_unverified_markers(strip_citation_block(content), records)
572 block = render_citation_block(records)
573 return f"{body.rstrip()}\n\n{block}" if block else body
576def _frontmatter_sources(content: str) -> list[str]:
577 """Source filenames recorded in a page's ``sources`` frontmatter field."""
578 # Frontmatter is untyped YAML: a hand-edited page can carry anything here.
579 raw = parse_frontmatter(content).get("sources")
580 return [str(item) for item in raw] if isinstance(raw, list) else []
583def reject_draft(slug: str, wiki_root: Path) -> None:
584 """Delete the draft file without touching the published page or the index."""
585 with WIKI_BUILD_LOCK:
586 draft = _draft_path(wiki_root, slug)
587 if not draft.is_file():
588 raise FileNotFoundError(f"draft not found: {slug}")
589 draft.unlink()
590 log.info("Rejected draft %s", slug)
593def _wiki_source_for(target: Path, wiki_root: Path, config: Config) -> str:
594 """Build the ``wiki_source`` identifier used in the chunks table.
596 Shape matches :attr:`PageTarget.wiki_source`:
597 ``<wiki_dir>/<subdir>/<slug>.md``. Built from ``config.wiki_dir`` like
598 every other producer, so a nested wiki_dir (``notes/wiki``) resolves.
599 """
600 relative = target.relative_to(wiki_root)
601 return f"{config.wiki_dir}/{relative.as_posix()}"
604def _coerce_float(value: Any) -> float | None:
605 """Return *value* as a float, or None when conversion is not sensible."""
606 if value is None:
607 return None
608 try:
609 return float(value)
610 except (TypeError, ValueError):
611 return None