Coverage for src/lilbee/wiki/page.py: 100%
154 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-11-06 08:17 +0000
1"""Single-page generation pipeline for wiki summary and synthesis pages.
3Given a label, prompt, and grounding chunks, drives the LLM call,
4parses + verifies citations, scores faithfulness, builds the
5frontmatter / body / citation block, and lands the page on disk via
6:mod:`lilbee.wiki.persistence`. Also owns ``index_wiki_page``: the
7post-write step that chunks, embeds, and stores the wiki body itself
8so wiki content participates in retrieval.
9"""
11from __future__ import annotations
13import json
14import logging
15from collections.abc import Callable, Mapping
16from datetime import UTC, datetime
17from pathlib import Path
18from typing import Any
20from lilbee.app.services import get_services
21from lilbee.core.config import DEFAULT_NUM_CTX, Config
22from lilbee.data.extract.chunk import chunk_text
23from lilbee.data.store import (
24 ChunkType,
25 CitationRecord,
26 SearchChunk,
27 Store,
28)
29from lilbee.providers.base import ChosenSeed, LLMProvider
30from lilbee.retrieval.reasoning import strip_reasoning
31from lilbee.wiki.citations import (
32 ParsedCitation,
33 extract_body,
34 parse_wiki_citations,
35 render_citation_block,
36 render_provenance,
37 scrub_unverified_markers,
38 strip_citation_block,
39 verify_citations,
40 wiki_sourced_count,
41)
42from lilbee.wiki.persistence import (
43 delete_drift_draft_if_present,
44 divert_concept_collision,
45 divert_to_drafts,
46 draft_source_names,
47 origin_marker,
48 persist_and_finalize,
49 subdir_from_wiki_source,
50 with_source_ids,
51)
52from lilbee.wiki.quality import check_faithfulness, content_change_ratio, diff_summary
53from lilbee.wiki.shared import (
54 WIKI_CONTENT_SUBDIRS,
55 PageTarget,
56 WikiSubdir,
57 atomic_write_text,
58)
59from lilbee.wiki.stats import BuildStats
61log = logging.getLogger(__name__)
63WikiProgressCallback = Callable[[str, dict[str, object]], None]
64"""Callback for wiki generation progress: (stage, data) -> None."""
66# Floor on the chunk budget as a fraction of the context window, applied when
67# the output cap plus the template reserve already exceed ``num_ctx``.
68_MIN_CONTEXT_BUDGET_FRACTION = 0.25
70# Approximate characters per token for budget estimation. 4 chars/token
71# is a widely used heuristic for English text.
72_CHARS_PER_TOKEN = 4
74# Separator ``chunks_to_text`` puts between formatted chunk blocks.
75_CHUNK_SEPARATOR = "\n\n"
77# Seed used for wiki generation when the user set none, so an unchanged
78# corpus regenerates identically at the low wiki sampling temperature.
79WIKI_DEFAULT_SEED = ChosenSeed(1)
81# Directive recognized by chat templates that support a reasoning mode
82# (Qwen3, DeepSeek-R1, etc.). Wiki generation is a summarization task
83# where chain-of-thought adds wall-clock cost without improving output,
84# so we suppress it whenever the provider reports the capability.
85_NO_THINK_DIRECTIVE = "/no_think"
87# Capability string a provider's get_capabilities reports for reasoning
88# models (Qwen3, DeepSeek-R1).
89_CAPABILITY_THINKING = "thinking"
92def build_wiki_messages(prompt: str, provider: LLMProvider, config: Config) -> list[dict[str, str]]:
93 """Build the chat messages list for a wiki-gen call.
95 When the provider reports the ``thinking`` capability for the active
96 chat model, prepends ``/no_think`` so the chat template disables the
97 reasoning mode. Otherwise the prompt passes through unchanged.
98 """
99 capabilities = provider.get_capabilities(config.chat_model)
100 if _CAPABILITY_THINKING in capabilities:
101 prompt = f"{_NO_THINK_DIRECTIVE}\n\n{prompt}"
102 return [{"role": "user", "content": prompt}]
105def wiki_generation_options(config: Config) -> dict[str, Any]:
106 """Sampling options for a wiki LLM call: wiki temperature, output cap, fixed seed."""
107 return config.generation_options(
108 temperature=config.wiki_temperature,
109 max_tokens=config.wiki_summary_max_tokens,
110 seed=WIKI_DEFAULT_SEED if config.seed is None else config.seed,
111 )
114def prompt_overhead_tokens(config: Config, prompt_chars: int | None = None) -> int:
115 """Token cost of the non-chunk prompt text plus the ``/no_think`` prefix.
117 ``prompt_chars`` is the rendered length of everything the prompt carries
118 besides the chunks, so per-call substitutions (concept instruction, entity
119 list, source list) are charged against the budget. Without it the longest
120 raw template stands in, measured rather than assumed so overriding a prompt
121 from settings moves the chunk budget with it.
122 """
123 longest = max(
124 len(config.wiki_synthesis_prompt),
125 len(config.wiki_entity_batch_prompt),
126 )
127 chars = longest if prompt_chars is None else prompt_chars
128 return -(-(chars + len(_NO_THINK_DIRECTIVE)) // _CHARS_PER_TOKEN)
131def truncate_chunks_to_budget(
132 chunks: list[SearchChunk],
133 config: Config,
134 prompt_chars: int | None = None,
135) -> list[SearchChunk]:
136 """Drop trailing chunks so the prompt plus its generation fits the context window.
138 The budget is ``num_ctx`` less the output cap (``wiki_summary_max_tokens``)
139 less the prompt overhead. The quarter-window floor applies only when that
140 leaves nothing: a positive budget is used as-is, so the floor can never
141 raise the budget past what the window actually has room for. Chunks are
142 measured as :func:`chunks_to_text` formats them, numbering and separators
143 included. Uses a chars/4 heuristic for token estimation.
144 """
145 context_window = config.num_ctx or DEFAULT_NUM_CTX
146 overhead = prompt_overhead_tokens(config, prompt_chars)
147 available = context_window - config.wiki_summary_max_tokens - overhead
148 budget_tokens = (
149 available if available > 0 else int(context_window * _MIN_CONTEXT_BUDGET_FRACTION)
150 )
151 budget_chars = budget_tokens * _CHARS_PER_TOKEN
153 total_chars = 0
154 kept: list[SearchChunk] = []
155 for index, chunk in enumerate(chunks):
156 chunk_chars = len(_format_chunk(chunk, index)) + len(_CHUNK_SEPARATOR)
157 if total_chars + chunk_chars > budget_chars and kept:
158 break
159 kept.append(chunk)
160 total_chars += chunk_chars
162 if len(kept) < len(chunks):
163 log.warning(
164 "Truncated chunks from %d to %d to fit context window (%d tokens)",
165 len(chunks),
166 len(kept),
167 context_window,
168 )
169 return kept
172def _format_chunk(chunk: SearchChunk, index: int) -> str:
173 """Render one chunk as the numbered block the prompt embeds."""
174 location = ""
175 if chunk.page_start:
176 location = f" (page {chunk.page_start})"
177 elif chunk.line_start:
178 location = f" (lines {chunk.line_start}-{chunk.line_end})"
179 return f"[Chunk {index + 1}]{location}:\n{chunk.chunk}"
182def chunks_to_text(chunks: list[SearchChunk]) -> str:
183 """Format chunks as numbered text blocks for the LLM prompt."""
184 return _CHUNK_SEPARATOR.join(_format_chunk(chunk, i) for i, chunk in enumerate(chunks))
187def build_frontmatter(
188 config: Config,
189 source_names: list[str],
190 score: float,
191 chunks: list[SearchChunk] | None = None,
192) -> str:
193 """Build YAML frontmatter for a wiki page.
195 When ``chunks`` is provided the frontmatter carries a ``provenance``
196 block naming the source/chunk-index pairs that fed the generator and
197 the extraction method from config, so a bad page is auditable
198 without re-running the pipeline.
199 """
200 # A JSON array is valid YAML flow syntax and escapes quotes/backslashes/
201 # unicode, so a filename like ``a"b\c.txt`` cannot corrupt the frontmatter.
202 sources_yaml = json.dumps(sorted(source_names))
203 provenance_block = render_provenance(config, chunks) if chunks is not None else ""
204 return (
205 f"---\n"
206 f"generated_by: {config.chat_model}\n"
207 f"generated_at: {datetime.now(UTC).isoformat()}\n"
208 f"sources: {sources_yaml}\n"
209 f"faithfulness_score: {score:.2f}\n"
210 f"{provenance_block}"
211 f"---\n\n"
212 )
215def write_page(
216 wiki_root: Path,
217 subdir: str,
218 slug: str,
219 full_content: str,
220 drift_threshold: float,
221 source_names: list[str],
222 page_type: str,
223 *,
224 source_ids: Mapping[str, str],
225) -> Path:
226 """Write page to disk with drift detection. Returns path written to.
228 ``slug`` may contain forward slashes (e.g. ``cv-manual/page-0042``);
229 any intermediate directories are created before writing. Publishing
230 retires this page's own drift draft: that proposal predates this body
231 and accepting it would undo the regen. Another source's draft under the
232 same slug is kept. A ``drafts/`` target skips drift entirely and routes
233 through :func:`_write_draft_page`; ``page_type`` is the subdir the page
234 would have published to, which a diverted page carries in its marker. A
235 draft records ``source_ids``, the id of each of *source_names*.
236 """
237 page_path = wiki_root / subdir / f"{slug}.md"
238 drafts_dir = wiki_root / WikiSubdir.DRAFTS
239 if subdir == WikiSubdir.DRAFTS:
240 return _write_draft_page(
241 page_path, drafts_dir, slug, full_content, source_names, page_type, source_ids
242 )
244 if page_path.exists():
245 old_content = page_path.read_text(encoding="utf-8")
246 # Frontmatter carries a fresh timestamp and score on every regen, so the
247 # ratio is taken over body prose only.
248 ratio = content_change_ratio(extract_body(old_content), extract_body(full_content))
249 if ratio > drift_threshold:
250 diff_text = diff_summary(old_content, full_content)
251 return divert_to_drafts(
252 full_content,
253 drafts_dir,
254 slug,
255 ratio,
256 diff_text,
257 subdir,
258 source_names,
259 source_ids=source_ids,
260 )
262 atomic_write_text(page_path, full_content)
263 delete_drift_draft_if_present(drafts_dir, slug, source_names)
264 return page_path
267def _write_draft_page(
268 draft_path: Path,
269 drafts_dir: Path,
270 slug: str,
271 full_content: str,
272 source_names: list[str],
273 page_type: str,
274 source_ids: Mapping[str, str],
275) -> Path:
276 """Land a below-threshold page in ``drafts/``, keeping another source's draft.
278 A newer proposal from the same sources supersedes the draft in place; one
279 from different sources lands at a collision draft rather than overwriting a
280 page awaiting review. No drift ratio is computed: a drafts target is a
281 proposal, not a published body it could be drifting from.
282 """
283 owner = draft_source_names(draft_path)
284 if owner is not None and owner != sorted(source_names):
285 return divert_concept_collision(
286 slug=slug,
287 source=", ".join(sorted(source_names)),
288 first_source=f"{WikiSubdir.DRAFTS}/{slug}.md",
289 content=full_content,
290 drafts_dir=drafts_dir,
291 origin_subdir=page_type,
292 source_ids=source_ids,
293 )
294 recorded = with_source_ids(full_content, source_ids)
295 atomic_write_text(draft_path, f"{origin_marker(page_type)}\n\n{recorded}")
296 return draft_path
299def assemble_content(
300 frontmatter: str,
301 wiki_text: str,
302 citation_block: str,
303) -> str:
304 """Combine frontmatter, body, and citations into the full page content."""
305 full = frontmatter + wiki_text
306 if citation_block:
307 full += "\n\n" + citation_block
308 return full
311def indexable_chunks(content: str) -> list[str]:
312 """The chunks a page's body would index as; empty means nothing to index.
314 One definition of "indexable" for the indexer and for the accept-time
315 refusal, because a non-empty body can still chunk to nothing ("#", "---")
316 and the two must not disagree about that.
317 """
318 body = extract_body(content).strip()
319 return chunk_text(body, mime_type="text/markdown", use_semantic=True) if body else []
322def index_wiki_page(
323 content: str,
324 wiki_source: str,
325 store: Store,
326 config: Config,
327 chunks: list[str] | None = None,
328) -> int:
329 """Chunk a wiki page body, embed it, and write rows with ``chunk_type="wiki"``.
331 ``wiki_source`` must follow the ``<config.wiki_dir>/<subdir>/<slug>.md``
332 shape (see :attr:`PageTarget.wiki_source`). Three branches:
334 - subdir in :data:`WIKI_CONTENT_SUBDIRS`: chunk, embed, then swap
335 the page's rows in one locked replace, so an embedder failure
336 leaves the previous rows searchable. Returns the row count.
337 - subdir is ``drafts/`` or ``archive/``: skip without touching the
338 store. Returns 0.
339 - malformed ``wiki_source`` (no subdir component): log.warning and
340 return 0. Does not raise because the caller set is narrow (only
341 internal wiki paths reach here) and surfacing the bad input in
342 the log is sufficient triage.
344 Record shape matches the markdown-ingest convention in
345 ``lilbee.data.ingest``: ``content_type="text"``, all four page/line
346 positions ``0`` (wiki pages are not paginated).
348 ``chunks`` lets a caller that already chunked this content pass the result
349 in rather than paying for a second semantic pass under the build lock.
350 """
351 subdir = subdir_from_wiki_source(wiki_source, config.wiki_dir)
352 if subdir is None:
353 log.warning("index_wiki_page: malformed wiki_source %r (no subdir)", wiki_source)
354 return 0
355 if subdir not in WIKI_CONTENT_SUBDIRS:
356 return 0
358 if chunks is None:
359 chunks = indexable_chunks(content)
360 if not chunks:
361 return store.replace_chunks(wiki_source, ChunkType.WIKI, [])
363 vectors = get_services().embedder.embed_batch(chunks)
364 records = [
365 {
366 "source": wiki_source,
367 "content_type": "text",
368 "chunk_type": ChunkType.WIKI,
369 "page_start": 0,
370 "page_end": 0,
371 "line_start": 0,
372 "line_end": 0,
373 "chunk": text,
374 "chunk_index": idx,
375 "vector": vector,
376 }
377 for idx, (text, vector) in enumerate(zip(chunks, vectors, strict=True))
378 ]
379 return store.replace_chunks(wiki_source, ChunkType.WIKI, records)
382def generate_page(
383 label: str,
384 prompt: str,
385 chunks: list[SearchChunk],
386 citation_resolver: Callable[[list[ParsedCitation]], list[CitationRecord]],
387 page_type: str,
388 slug: str,
389 source_names: list[str],
390 provider: LLMProvider,
391 store: Store,
392 config: Config,
393 on_progress: WikiProgressCallback | None = None,
394 stats: BuildStats | None = None,
395 supersedes_sources: bool = True,
396) -> Path | None:
397 """Core generation pipeline shared by summary and synthesis pages.
399 ``supersedes_sources`` is False when the sources merely mention the
400 subject rather than being replaced by the page, so ``wiki_prune_raw``
401 does not delete them.
402 """
403 stats = BuildStats.ensure(stats)
405 def _emit(stage: str, **data: object) -> None:
406 if on_progress is not None:
407 on_progress(stage, data)
409 _emit("preparing", chunks=len(chunks), source=label)
411 messages = build_wiki_messages(prompt, provider, config)
412 _emit("generating", source=label)
413 options = wiki_generation_options(config)
414 try:
415 response = provider.chat(messages, stream=False, options=options)
416 wiki_text = strip_reasoning(response.text).strip()
417 except Exception as exc:
418 log.warning("LLM failed to generate wiki page for %s: %s", label, exc)
419 _emit("failed", error=str(exc))
420 return None
422 if not wiki_text:
423 log.warning("LLM returned empty response for wiki page %s", label)
424 _emit("failed", error="Model returned empty response")
425 return None
427 parsed_citations = parse_wiki_citations(wiki_text)
428 resolved = citation_resolver(parsed_citations)
429 verified = verify_citations(resolved, chunks, label, config)
430 stats.record_citations(
431 len(verified),
432 len(parsed_citations) - wiki_sourced_count(resolved, config) - len(verified),
433 )
434 if not verified:
435 log.warning("No valid citations for %s, skipping", label)
436 _emit("failed", error="No valid citations found")
437 return None
439 _emit("faithfulness_check")
440 score = check_faithfulness(chunks, wiki_text, label, config)
441 threshold = config.wiki_embedding_faithfulness_threshold
442 subdir = page_type if score >= threshold else WikiSubdir.DRAFTS
443 if subdir == WikiSubdir.DRAFTS:
444 log.info("Wiki page %s scored %.2f (< %.2f), sending to drafts", label, score, threshold)
446 wiki_text = scrub_unverified_markers(strip_citation_block(wiki_text), verified)
447 frontmatter = build_frontmatter(config, source_names, score, chunks=chunks)
448 citation_block = render_citation_block(verified)
449 full_content = assemble_content(frontmatter, wiki_text, citation_block)
451 wiki_root = config.data_root / config.wiki_dir
452 target = PageTarget(
453 wiki_root=wiki_root,
454 subdir=subdir,
455 slug=slug,
456 wiki_source=f"{config.wiki_dir}/{subdir}/{slug}.md",
457 page_type=page_type,
458 label=label,
459 supersedes_sources=supersedes_sources,
460 )
461 page_path = persist_and_finalize(
462 full_content, target, verified, source_names, store, config, stats=stats
463 )
465 log.info(
466 "Generated wiki page for %s -> %s (score=%.2f, citations=%d)",
467 label,
468 target.subdir,
469 score,
470 len(verified),
471 )
472 return page_path