Coverage for src/lilbee/wiki/persistence.py: 100%
147 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-11-06 08:17 +0000
1"""Disk-write side effects for wiki page generation.
3Owns the orchestrator that lands a generated page on disk plus the
4draft-routing helpers (drift redirects, PENDING markers for parse
5failures, collision markers for duplicate concept slugs). Higher-level
6code in :mod:`lilbee.wiki.page` calls into here for the publish step;
7the actual ``write_page`` lives there to keep file-handling close to
8content assembly.
9"""
11from __future__ import annotations
13import json
14import logging
15from collections.abc import Mapping
16from pathlib import Path
18from lilbee.core.config import Config
19from lilbee.data.store import CitationRecord, SourceType, Store
20from lilbee.wiki.index import append_wiki_log, update_wiki_index
21from lilbee.wiki.shared import (
22 PENDING_COLLISION_MARKER_PREFIX,
23 PageTarget,
24 WikiLogAction,
25 WikiSubdir,
26 atomic_write_text,
27 frontmatter_span,
28 is_pending_marker_text,
29 parse_frontmatter,
30)
31from lilbee.wiki.stats import BuildStats
33log = logging.getLogger(__name__)
35# Leading comment on a drift-diverted draft. The drafts surface matches the
36# same wording with a regex.
37_DRIFT_MARKER_PREFIX = "<!-- DRIFT:"
40def origin_marker(subdir: str) -> str:
41 """Leading comment recording the page type a draft would have published as.
43 Drift and collision drafts carry ``origin:`` inside their own markers. A
44 draft held only by the faithfulness gate has no marker of its own, so
45 without this one, accepting it with no published counterpart files it
46 under ``summaries/`` instead of its own page type.
47 """
48 return f"<!-- origin: {subdir} -->"
51# Once ``<wiki_dir>/`` is stripped, a well-formed source leaves at least
52# ``<subdir>/<slug>.md``. Anything shorter has no subdir.
53_WIKI_SOURCE_MIN_PARTS = 2
55# Drift-marker field carrying the hash of the diverting page's sources, so a
56# later divert can tell its own draft from another source's.
57_DRIFT_SOURCE_FIELD = "source: "
59# Frontmatter field listing the sources a generated page was written from.
60_DRAFT_SOURCES_FIELD = "sources"
62# Frontmatter field of a draft naming the id of each source path it was written with.
63SOURCE_IDS_FIELD = "source_ids"
66def with_source_ids(content: str, source_ids: Mapping[str, str]) -> str:
67 """*content* with a frontmatter line naming the id of each source path it shows."""
68 lines = content.split("\n")
69 span = frontmatter_span(lines)
70 if span is None:
71 return content
72 _start, end = span
73 lines.insert(end, f"{SOURCE_IDS_FIELD}: {json.dumps(dict(sorted(source_ids.items())))}")
74 return "\n".join(lines)
77def without_source_ids(text: str) -> str:
78 """*text* without the frontmatter line naming the ids of its sources."""
79 lines = text.split("\n")
80 span = frontmatter_span(lines)
81 if span is None:
82 return text
83 start, end = span
84 field = f"{SOURCE_IDS_FIELD}:"
85 kept = [line for line in lines[start + 1 : end] if not line.startswith(field)]
86 return "\n".join([*lines[: start + 1], *kept, *lines[end:]])
89def recorded_source_ids(text: str) -> dict[str, str] | None:
90 """The id of each source path a draft records, or None when it records none."""
91 recorded = parse_frontmatter(text).get(SOURCE_IDS_FIELD)
92 # Frontmatter is untyped YAML: a hand-edited draft can hold anything here.
93 if not isinstance(recorded, dict):
94 return None
95 return {str(path): str(source_id) for path, source_id in recorded.items()}
98def _read_draft(draft_path: Path) -> str | None:
99 """Return the draft's text, or None when it is absent or unreadable."""
100 if not draft_path.is_file():
101 return None
102 try:
103 return draft_path.read_text(encoding="utf-8")
104 except OSError:
105 return None
108def draft_source_names(draft_path: Path) -> list[str] | None:
109 """Sorted source names in an existing draft's frontmatter, or None when it has none.
111 A PENDING marker carries no ``sources`` field and so reads as unowned: it is
112 a placeholder, not review content. Reading past the marker run is the
113 parser's job.
114 """
115 text = _read_draft(draft_path)
116 if text is None:
117 return None
118 sources = parse_frontmatter(text).get(_DRAFT_SOURCES_FIELD)
119 # Frontmatter is untyped YAML: a hand-edited draft can hold anything.
120 return sorted(sources) if isinstance(sources, list) else None
123def _draft_belongs_to_other_source(
124 draft_path: Path, source_key: str, source_names: list[str]
125) -> bool:
126 """Return whether an existing draft holds reviewable content from another source.
128 A PENDING marker or an empty file is a placeholder, not review content, so
129 neither blocks the write. A drift draft carries its source key in the
130 marker; a quality-gate draft carries its sources in frontmatter, so a
131 same-source draft of either kind is superseded in place rather than
132 treated as a collision.
133 """
134 text = _read_draft(draft_path)
135 if text is None or not text.strip() or is_pending_marker_text(text):
136 return False
137 first_line = text.splitlines()[0]
138 if f"{_DRIFT_SOURCE_FIELD}{source_key}" in first_line:
139 return False
140 return draft_source_names(draft_path) != sorted(source_names)
143def divert_to_drafts(
144 new_content: str,
145 drafts_dir: Path,
146 slug: str,
147 change_ratio: float,
148 diff_text: str,
149 origin_subdir: str,
150 source_names: list[str],
151 *,
152 source_ids: Mapping[str, str],
153) -> Path:
154 """Write new content to wiki/drafts/ with a drift note instead of overwriting.
156 ``origin_subdir`` is the published subdir the page would have landed in
157 (``concepts``, ``entities``, ...); it rides the drift marker so that
158 accepting an unpaired draft restores it to its own page type instead of
159 defaulting to ``summaries/``. The marker also carries a hash of
160 ``source_names``: when ``drafts/<slug>.md`` already holds another source's
161 diverted content, this one lands at a ``-collision-<hash>`` draft rather
162 than overwriting a page awaiting review.
163 """
164 # circular: persistence -> batch via short_source_hash (batch imports
165 # persist_and_finalize / divert_concept_collision from persistence).
166 from lilbee.wiki.batch import short_source_hash
168 sources_label = ", ".join(sorted(source_names))
169 source_key = short_source_hash(sources_label)
170 note = (
171 f"{_DRIFT_MARKER_PREFIX} {change_ratio:.0%} content changed; origin: {origin_subdir}; "
172 f"{_DRIFT_SOURCE_FIELD}{source_key} - flagged for human review -->\n\n"
173 )
174 log.warning(
175 "Drift detected for %s (%.0f%% changed), diverted to drafts. Diff:\n%s",
176 slug,
177 change_ratio * 100,
178 diff_text,
179 )
180 draft_path = drafts_dir / f"{slug}.md"
181 if _draft_belongs_to_other_source(draft_path, source_key, source_names):
182 return divert_concept_collision(
183 slug=slug,
184 source=sources_label,
185 first_source=f"{WikiSubdir.DRAFTS}/{slug}.md",
186 content=note + new_content,
187 drafts_dir=drafts_dir,
188 origin_subdir=origin_subdir,
189 source_ids=source_ids,
190 )
191 atomic_write_text(draft_path, note + with_source_ids(new_content, source_ids))
192 return draft_path
195def subdir_from_wiki_source(wiki_source: str, wiki_dir: str) -> str | None:
196 """Return the subdir component (``summaries``, ``concepts``, ...) of *wiki_source*.
198 ``wiki_source`` is the ``<wiki_dir>/<subdir>/<slug>.md`` path stored in
199 citations and chunks. ``wiki_dir`` is stripped as a prefix rather than
200 split positionally, so a nested wiki_dir (``notes/wiki``) still resolves
201 to its subdir. Returns None when the source is not under *wiki_dir* or
202 carries no subdir.
203 """
204 relative = wiki_source.removeprefix(wiki_dir + "/")
205 if relative == wiki_source:
206 return None
207 parts = relative.split("/")
208 return parts[0] if len(parts) >= _WIKI_SOURCE_MIN_PARTS else None
211def persist_and_finalize(
212 content: str,
213 target: PageTarget,
214 verified: list[CitationRecord],
215 source_names: list[str],
216 store: Store,
217 config: Config,
218 *,
219 stats: BuildStats | None = None,
220) -> Path:
221 """Write page to disk, persist citations, index body chunks, update index and log.
223 Only a published page carries store state: a page routed to ``drafts/``
224 (drift diversion or a failed quality gate) is written and logged, then
225 returns. Its citations, chunks, and the raw sources it would supersede stay
226 untouched until the draft is accepted. Every outcome is recorded on *stats*.
227 """
228 # circular: page -> persistence via persist_and_finalize
229 from lilbee.wiki.page import index_wiki_page, write_page
231 stats = BuildStats.ensure(stats)
232 page_path = write_page(
233 target.wiki_root,
234 target.subdir,
235 target.slug,
236 content,
237 config.wiki_drift_threshold,
238 source_names,
239 target.page_type,
240 source_ids=store.source_ids(source_names, SourceType.DOCUMENT),
241 )
242 published_path = target.wiki_root / target.subdir / f"{target.slug}.md"
243 if page_path != published_path:
244 # A drafts-target collision writes a PENDING marker, not review content;
245 # count it the same way the batch collision branch does.
246 if is_pending_marker_text(_read_draft(page_path) or ""):
247 stats.record_pending_marker()
248 else:
249 stats.record_drafted()
250 append_wiki_log(
251 WikiLogAction.GENERATED,
252 f"{target.page_type} page for {target.label} diverted to draft "
253 f"{page_path.name} (published page unchanged)",
254 config,
255 )
256 return page_path
258 if target.subdir == WikiSubdir.DRAFTS:
259 stats.record_drafted()
260 append_wiki_log(
261 WikiLogAction.GENERATED,
262 f"{target.page_type} page for {target.label} held in "
263 f"{WikiSubdir.DRAFTS}/{page_path.name} pending review",
264 config,
265 )
266 return page_path
268 stats.record_published(target.wiki_source, len(verified))
269 for rec in verified:
270 rec["wiki_source"] = target.wiki_source
271 store.replace_citations_for_wiki(target.wiki_source, verified)
273 index_wiki_page(content, target.wiki_source, store, config)
275 if config.wiki_prune_raw and target.supersedes_sources:
276 for name in source_names:
277 try:
278 store.delete_by_source(name, SourceType.DOCUMENT)
279 except Exception:
280 # Best-effort pruning of raw sources the new page supersedes; one
281 # failed delete must not abort the loop or fail the generated page.
282 log.warning("Failed to prune raw source %s", name, exc_info=True)
284 update_wiki_index(config)
285 append_wiki_log(
286 WikiLogAction.GENERATED,
287 f"{target.page_type} page for {target.label} -> {target.subdir}/{target.slug}.md",
288 config,
289 )
290 return page_path
293def write_pending_marker(
294 drafts_dir: Path,
295 slug: str,
296 marker_line: str,
297 frontmatter: str = "",
298) -> Path | None:
299 """Write a PENDING marker page under ``drafts/<slug>.md``. None when withheld.
301 ``marker_line`` is the leading HTML comment that both identifies
302 the marker kind and carries the context (source, label). The
303 optional ``frontmatter`` preserves minimal metadata for the
304 drafts surface to round-trip (e.g. ``bad_title``-style fields).
306 An existing draft that is not itself a marker is generated content
307 awaiting review and is kept: the marker is not written over it, and
308 the caller gets None so it does not count a marker that never landed.
309 """
310 draft_path = drafts_dir / f"{slug}.md"
311 existing = _read_draft(draft_path)
312 if existing is not None and not is_pending_marker_text(existing):
313 log.warning(
314 "Keeping the draft at %s: it holds content pending review, not a marker",
315 draft_path,
316 )
317 return None
318 body = marker_line + "\n"
319 if frontmatter:
320 body += "\n" + frontmatter
321 atomic_write_text(draft_path, body)
322 return draft_path
325def delete_pending_marker_if_present(drafts_dir: Path, slug: str) -> bool:
326 """Delete an existing PENDING marker for *slug*; return whether one was removed.
328 Match is slug-equality (not fuzzy): an LLM that rephrases a
329 label on retry (``brake system`` → ``braking system``) leaves
330 the old marker behind for the user to drain via ``wiki drafts
331 reject``. Documented limitation; follow-up if the pattern
332 matters.
333 """
334 draft_path = drafts_dir / f"{slug}.md"
335 body = _read_draft(draft_path)
336 if body is None or not is_pending_marker_text(body):
337 return False
338 draft_path.unlink()
339 return True
342def delete_drift_draft_if_present(drafts_dir: Path, slug: str, source_names: list[str]) -> bool:
343 """Delete *slug*'s superseded drift draft; return whether one was removed.
345 A regen that lands under the drift threshold supersedes the proposal an
346 earlier regen of the same sources parked in ``drafts/``: accepting the
347 older draft afterwards would overwrite the newer published body. The
348 drafts namespace is flat, so a draft carrying another source's key (a
349 concept and an entity can share a slug) is kept.
350 """
351 # circular: persistence -> batch via short_source_hash (batch imports
352 # persist_and_finalize / divert_concept_collision from persistence).
353 from lilbee.wiki.batch import short_source_hash
355 draft_path = drafts_dir / f"{slug}.md"
356 text = _read_draft(draft_path)
357 if text is None or not text.lstrip().startswith(_DRIFT_MARKER_PREFIX):
358 return False
359 source_key = short_source_hash(", ".join(sorted(source_names)))
360 if _draft_belongs_to_other_source(draft_path, source_key, source_names):
361 log.info("Keeping drift draft %s: it holds another source's proposal", draft_path)
362 return False
363 draft_path.unlink()
364 log.info("Removed superseded drift draft %s", draft_path)
365 return True
368def divert_concept_collision(
369 *,
370 slug: str,
371 source: str,
372 first_source: str,
373 content: str,
374 drafts_dir: Path,
375 origin_subdir: str,
376 source_ids: Mapping[str, str],
377) -> Path:
378 """Write the losing concept to ``drafts/<slug>-collision-<hash>.md``.
380 The winning source's page is unchanged on disk. Hash is the
381 first 8 hex of sha256(source_filename); stable per source so a
382 retry on the same two sources lands at the same draft path,
383 letting the user iterate without marker sprawl. ``origin_subdir``
384 is the published subdir the losing page would have landed in; it
385 rides the marker like the drift note's ``origin:`` field so that
386 accepting a collision with no published counterpart restores it to
387 its own page type instead of ``summaries/``.
388 """
389 # circular: persistence -> batch via short_source_hash (batch imports
390 # persist_and_finalize / divert_concept_collision from persistence).
391 from lilbee.wiki.batch import short_source_hash
393 short = short_source_hash(source)
394 collision_slug = f"{slug}-collision-{short}"
395 marker = (
396 f"{PENDING_COLLISION_MARKER_PREFIX} with source {first_source}, "
397 f"content from {source} held for review; origin: {origin_subdir} -->\n\n"
398 )
399 path = drafts_dir / f"{collision_slug}.md"
400 atomic_write_text(path, marker + with_source_ids(content, source_ids))
401 log.warning(
402 "Concept slug collision: %s already written by %s; diverted %s's version to %s",
403 slug,
404 first_source,
405 source,
406 path,
407 )
408 return path