Coverage for src/lilbee/wiki/persistence.py: 100%
122 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-08-14 11:46 +0000
1"""Disk-write side effects for wiki page generation.
3Owns the orchestrator that lands a generated page on disk plus the
4draft-routing helpers (drift redirects, PENDING markers for parse
5failures, collision markers for duplicate concept slugs). Higher-level
6code in :mod:`lilbee.wiki.page` calls into here for the publish step;
7the actual ``write_page`` lives there to keep file-handling close to
8content assembly.
9"""
11from __future__ import annotations
13import logging
14from pathlib import Path
16from lilbee.core.config import Config
17from lilbee.data.store import CitationRecord, Store
18from lilbee.wiki.index import append_wiki_log, update_wiki_index
19from lilbee.wiki.shared import (
20 PENDING_COLLISION_MARKER_PREFIX,
21 PageTarget,
22 WikiLogAction,
23 WikiSubdir,
24 atomic_write_text,
25 is_pending_marker_text,
26 parse_frontmatter,
27)
28from lilbee.wiki.stats import BuildStats
30log = logging.getLogger(__name__)
32# Leading comment on a drift-diverted draft. The drafts surface matches the
33# same wording with a regex.
34_DRIFT_MARKER_PREFIX = "<!-- DRIFT:"
37def origin_marker(subdir: str) -> str:
38 """Leading comment recording the page type a draft would have published as.
40 Drift and collision drafts carry ``origin:`` inside their own markers. A
41 draft held only by the faithfulness gate has no marker of its own, so
42 without this one, accepting it with no published counterpart files it
43 under ``summaries/`` instead of its own page type.
44 """
45 return f"<!-- origin: {subdir} -->"
48# Once ``<wiki_dir>/`` is stripped, a well-formed source leaves at least
49# ``<subdir>/<slug>.md``. Anything shorter has no subdir.
50_WIKI_SOURCE_MIN_PARTS = 2
52# Drift-marker field carrying the hash of the diverting page's sources, so a
53# later divert can tell its own draft from another source's.
54_DRIFT_SOURCE_FIELD = "source: "
56# Frontmatter field listing the sources a generated page was written from.
57_DRAFT_SOURCES_FIELD = "sources"
60def _read_draft(draft_path: Path) -> str | None:
61 """Return the draft's text, or None when it is absent or unreadable."""
62 if not draft_path.is_file():
63 return None
64 try:
65 return draft_path.read_text(encoding="utf-8")
66 except OSError:
67 return None
70def draft_source_names(draft_path: Path) -> list[str] | None:
71 """Sorted source names in an existing draft's frontmatter, or None when it has none.
73 A PENDING marker carries no ``sources`` field and so reads as unowned: it is
74 a placeholder, not review content. Reading past the marker run is the
75 parser's job.
76 """
77 text = _read_draft(draft_path)
78 if text is None:
79 return None
80 sources = parse_frontmatter(text).get(_DRAFT_SOURCES_FIELD)
81 # Frontmatter is untyped YAML: a hand-edited draft can hold anything.
82 return sorted(sources) if isinstance(sources, list) else None
85def _draft_belongs_to_other_source(
86 draft_path: Path, source_key: str, source_names: list[str]
87) -> bool:
88 """Return whether an existing draft holds reviewable content from another source.
90 A PENDING marker or an empty file is a placeholder, not review content, so
91 neither blocks the write. A drift draft carries its source key in the
92 marker; a quality-gate draft carries its sources in frontmatter, so a
93 same-source draft of either kind is superseded in place rather than
94 treated as a collision.
95 """
96 text = _read_draft(draft_path)
97 if text is None or not text.strip() or is_pending_marker_text(text):
98 return False
99 first_line = text.splitlines()[0]
100 if f"{_DRIFT_SOURCE_FIELD}{source_key}" in first_line:
101 return False
102 return draft_source_names(draft_path) != sorted(source_names)
105def divert_to_drafts(
106 new_content: str,
107 drafts_dir: Path,
108 slug: str,
109 change_ratio: float,
110 diff_text: str,
111 origin_subdir: str,
112 source_names: list[str],
113) -> Path:
114 """Write new content to wiki/drafts/ with a drift note instead of overwriting.
116 ``origin_subdir`` is the published subdir the page would have landed in
117 (``concepts``, ``entities``, ...); it rides the drift marker so that
118 accepting an unpaired draft restores it to its own page type instead of
119 defaulting to ``summaries/``. The marker also carries a hash of
120 ``source_names``: when ``drafts/<slug>.md`` already holds another source's
121 diverted content, this one lands at a ``-collision-<hash>`` draft rather
122 than overwriting a page awaiting review.
123 """
124 # circular: persistence -> batch via short_source_hash (batch imports
125 # persist_and_finalize / divert_concept_collision from persistence).
126 from lilbee.wiki.batch import short_source_hash
128 sources_label = ", ".join(sorted(source_names))
129 source_key = short_source_hash(sources_label)
130 note = (
131 f"{_DRIFT_MARKER_PREFIX} {change_ratio:.0%} content changed; origin: {origin_subdir}; "
132 f"{_DRIFT_SOURCE_FIELD}{source_key} - flagged for human review -->\n\n"
133 )
134 log.warning(
135 "Drift detected for %s (%.0f%% changed), diverted to drafts. Diff:\n%s",
136 slug,
137 change_ratio * 100,
138 diff_text,
139 )
140 draft_path = drafts_dir / f"{slug}.md"
141 if _draft_belongs_to_other_source(draft_path, source_key, source_names):
142 return divert_concept_collision(
143 slug=slug,
144 source=sources_label,
145 first_source=f"{WikiSubdir.DRAFTS}/{slug}.md",
146 content=note + new_content,
147 drafts_dir=drafts_dir,
148 origin_subdir=origin_subdir,
149 )
150 atomic_write_text(draft_path, note + new_content)
151 return draft_path
154def subdir_from_wiki_source(wiki_source: str, wiki_dir: str) -> str | None:
155 """Return the subdir component (``summaries``, ``concepts``, ...) of *wiki_source*.
157 ``wiki_source`` is the ``<wiki_dir>/<subdir>/<slug>.md`` path stored in
158 citations and chunks. ``wiki_dir`` is stripped as a prefix rather than
159 split positionally, so a nested wiki_dir (``notes/wiki``) still resolves
160 to its subdir. Returns None when the source is not under *wiki_dir* or
161 carries no subdir.
162 """
163 relative = wiki_source.removeprefix(wiki_dir + "/")
164 if relative == wiki_source:
165 return None
166 parts = relative.split("/")
167 return parts[0] if len(parts) >= _WIKI_SOURCE_MIN_PARTS else None
170def persist_and_finalize(
171 content: str,
172 target: PageTarget,
173 verified: list[CitationRecord],
174 source_names: list[str],
175 store: Store,
176 config: Config,
177 *,
178 stats: BuildStats | None = None,
179) -> Path:
180 """Write page to disk, persist citations, index body chunks, update index and log.
182 Only a published page carries store state: a page routed to ``drafts/``
183 (drift diversion or a failed quality gate) is written and logged, then
184 returns. Its citations, chunks, and the raw sources it would supersede stay
185 untouched until the draft is accepted. Every outcome is recorded on *stats*.
186 """
187 # circular: page -> persistence via persist_and_finalize
188 from lilbee.wiki.page import index_wiki_page, write_page
190 stats = BuildStats.ensure(stats)
191 page_path = write_page(
192 target.wiki_root,
193 target.subdir,
194 target.slug,
195 content,
196 config.wiki_drift_threshold,
197 source_names,
198 target.page_type,
199 )
200 published_path = target.wiki_root / target.subdir / f"{target.slug}.md"
201 if page_path != published_path:
202 # A drafts-target collision writes a PENDING marker, not review content;
203 # count it the same way the batch collision branch does.
204 if is_pending_marker_text(_read_draft(page_path) or ""):
205 stats.record_pending_marker()
206 else:
207 stats.record_drafted()
208 append_wiki_log(
209 WikiLogAction.GENERATED,
210 f"{target.page_type} page for {target.label} diverted to draft "
211 f"{page_path.name} (published page unchanged)",
212 config,
213 )
214 return page_path
216 if target.subdir == WikiSubdir.DRAFTS:
217 stats.record_drafted()
218 append_wiki_log(
219 WikiLogAction.GENERATED,
220 f"{target.page_type} page for {target.label} held in "
221 f"{WikiSubdir.DRAFTS}/{page_path.name} pending review",
222 config,
223 )
224 return page_path
226 stats.record_published(target.wiki_source, len(verified))
227 for rec in verified:
228 rec["wiki_source"] = target.wiki_source
229 store.replace_citations_for_wiki(target.wiki_source, verified)
231 index_wiki_page(content, target.wiki_source, store, config)
233 if config.wiki_prune_raw and target.supersedes_sources:
234 for name in source_names:
235 try:
236 store.delete_by_source(name)
237 except Exception:
238 # Best-effort pruning of raw sources the new page supersedes; one
239 # failed delete must not abort the loop or fail the generated page.
240 log.warning("Failed to prune raw source %s", name, exc_info=True)
242 update_wiki_index(config)
243 append_wiki_log(
244 WikiLogAction.GENERATED,
245 f"{target.page_type} page for {target.label} -> {target.subdir}/{target.slug}.md",
246 config,
247 )
248 return page_path
251def write_pending_marker(
252 drafts_dir: Path,
253 slug: str,
254 marker_line: str,
255 frontmatter: str = "",
256) -> Path | None:
257 """Write a PENDING marker page under ``drafts/<slug>.md``. None when withheld.
259 ``marker_line`` is the leading HTML comment that both identifies
260 the marker kind and carries the context (source, label). The
261 optional ``frontmatter`` preserves minimal metadata for the
262 drafts surface to round-trip (e.g. ``bad_title``-style fields).
264 An existing draft that is not itself a marker is generated content
265 awaiting review and is kept: the marker is not written over it, and
266 the caller gets None so it does not count a marker that never landed.
267 """
268 draft_path = drafts_dir / f"{slug}.md"
269 existing = _read_draft(draft_path)
270 if existing is not None and not is_pending_marker_text(existing):
271 log.warning(
272 "Keeping the draft at %s: it holds content pending review, not a marker",
273 draft_path,
274 )
275 return None
276 body = marker_line + "\n"
277 if frontmatter:
278 body += "\n" + frontmatter
279 atomic_write_text(draft_path, body)
280 return draft_path
283def delete_pending_marker_if_present(drafts_dir: Path, slug: str) -> bool:
284 """Delete an existing PENDING marker for *slug*; return whether one was removed.
286 Match is slug-equality (not fuzzy): an LLM that rephrases a
287 label on retry (``brake system`` → ``braking system``) leaves
288 the old marker behind for the user to drain via ``wiki drafts
289 reject``. Documented limitation; follow-up if the pattern
290 matters.
291 """
292 draft_path = drafts_dir / f"{slug}.md"
293 body = _read_draft(draft_path)
294 if body is None or not is_pending_marker_text(body):
295 return False
296 draft_path.unlink()
297 return True
300def delete_drift_draft_if_present(drafts_dir: Path, slug: str, source_names: list[str]) -> bool:
301 """Delete *slug*'s superseded drift draft; return whether one was removed.
303 A regen that lands under the drift threshold supersedes the proposal an
304 earlier regen of the same sources parked in ``drafts/``: accepting the
305 older draft afterwards would overwrite the newer published body. The
306 drafts namespace is flat, so a draft carrying another source's key (a
307 concept and an entity can share a slug) is kept.
308 """
309 # circular: persistence -> batch via short_source_hash (batch imports
310 # persist_and_finalize / divert_concept_collision from persistence).
311 from lilbee.wiki.batch import short_source_hash
313 draft_path = drafts_dir / f"{slug}.md"
314 text = _read_draft(draft_path)
315 if text is None or not text.lstrip().startswith(_DRIFT_MARKER_PREFIX):
316 return False
317 source_key = short_source_hash(", ".join(sorted(source_names)))
318 if _draft_belongs_to_other_source(draft_path, source_key, source_names):
319 log.info("Keeping drift draft %s: it holds another source's proposal", draft_path)
320 return False
321 draft_path.unlink()
322 log.info("Removed superseded drift draft %s", draft_path)
323 return True
326def divert_concept_collision(
327 *,
328 slug: str,
329 source: str,
330 first_source: str,
331 content: str,
332 drafts_dir: Path,
333 origin_subdir: str,
334) -> Path:
335 """Write the losing concept to ``drafts/<slug>-collision-<hash>.md``.
337 The winning source's page is unchanged on disk. Hash is the
338 first 8 hex of sha256(source_filename); stable per source so a
339 retry on the same two sources lands at the same draft path,
340 letting the user iterate without marker sprawl. ``origin_subdir``
341 is the published subdir the losing page would have landed in; it
342 rides the marker like the drift note's ``origin:`` field so that
343 accepting a collision with no published counterpart restores it to
344 its own page type instead of ``summaries/``.
345 """
346 # circular: persistence -> batch via short_source_hash (batch imports
347 # persist_and_finalize / divert_concept_collision from persistence).
348 from lilbee.wiki.batch import short_source_hash
350 short = short_source_hash(source)
351 collision_slug = f"{slug}-collision-{short}"
352 marker = (
353 f"{PENDING_COLLISION_MARKER_PREFIX} with source {first_source}, "
354 f"content from {source} held for review; origin: {origin_subdir} -->\n\n"
355 )
356 path = drafts_dir / f"{collision_slug}.md"
357 atomic_write_text(path, marker + content)
358 log.warning(
359 "Concept slug collision: %s already written by %s; diverted %s's version to %s",
360 slug,
361 first_source,
362 source,
363 path,
364 )
365 return path