Coverage for src/lilbee/wiki/page.py: 100%
155 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-08-14 11:46 +0000
1"""Single-page generation pipeline for wiki summary and synthesis pages.
3Given a label, prompt, and grounding chunks, drives the LLM call,
4parses + verifies citations, scores faithfulness, builds the
5frontmatter / body / citation block, and lands the page on disk via
6:mod:`lilbee.wiki.persistence`. Also owns ``index_wiki_page``: the
7post-write step that chunks, embeds, and stores the wiki body itself
8so wiki content participates in retrieval.
9"""
11from __future__ import annotations
13import json
14import logging
15from collections.abc import Callable
16from datetime import UTC, datetime
17from pathlib import Path
18from typing import Any
20from lilbee.app.services import get_services
21from lilbee.core.config import CHUNKS_TABLE, DEFAULT_NUM_CTX, Config
22from lilbee.data.extract.chunk import chunk_text
23from lilbee.data.store import (
24 ChunkType,
25 CitationRecord,
26 SearchChunk,
27 Store,
28 escape_sql_string,
29)
30from lilbee.providers.base import LLMProvider
31from lilbee.retrieval.reasoning import strip_reasoning
32from lilbee.wiki.citations import (
33 ParsedCitation,
34 extract_body,
35 parse_wiki_citations,
36 render_citation_block,
37 render_provenance,
38 scrub_unverified_markers,
39 strip_citation_block,
40 verify_citations,
41 wiki_sourced_count,
42)
43from lilbee.wiki.persistence import (
44 delete_drift_draft_if_present,
45 divert_concept_collision,
46 divert_to_drafts,
47 draft_source_names,
48 origin_marker,
49 persist_and_finalize,
50 subdir_from_wiki_source,
51)
52from lilbee.wiki.quality import check_faithfulness, content_change_ratio, diff_summary
53from lilbee.wiki.shared import (
54 WIKI_CONTENT_SUBDIRS,
55 PageTarget,
56 WikiSubdir,
57 atomic_write_text,
58)
59from lilbee.wiki.stats import BuildStats
61log = logging.getLogger(__name__)
63WikiProgressCallback = Callable[[str, dict[str, object]], None]
64"""Callback for wiki generation progress: (stage, data) -> None."""
66# Floor on the chunk budget as a fraction of the context window, applied when
67# the output cap plus the template reserve already exceed ``num_ctx``.
68_MIN_CONTEXT_BUDGET_FRACTION = 0.25
70# Approximate characters per token for budget estimation. 4 chars/token
71# is a widely used heuristic for English text.
72_CHARS_PER_TOKEN = 4
74# Separator ``chunks_to_text`` puts between formatted chunk blocks.
75_CHUNK_SEPARATOR = "\n\n"
77# Seed used for wiki generation when the user set none, so an unchanged
78# corpus regenerates identically at the low wiki sampling temperature.
79WIKI_DEFAULT_SEED = 1
81# Directive recognized by chat templates that support a reasoning mode
82# (Qwen3, DeepSeek-R1, etc.). Wiki generation is a summarization task
83# where chain-of-thought adds wall-clock cost without improving output,
84# so we suppress it whenever the provider reports the capability.
85_NO_THINK_DIRECTIVE = "/no_think"
87# Capability string a provider's get_capabilities reports for reasoning
88# models (Qwen3, DeepSeek-R1).
89_CAPABILITY_THINKING = "thinking"
92def build_wiki_messages(prompt: str, provider: LLMProvider, config: Config) -> list[dict[str, str]]:
93 """Build the chat messages list for a wiki-gen call.
95 When the provider reports the ``thinking`` capability for the active
96 chat model, prepends ``/no_think`` so the chat template disables the
97 reasoning mode. Otherwise the prompt passes through unchanged.
98 """
99 capabilities = provider.get_capabilities(config.chat_model)
100 if _CAPABILITY_THINKING in capabilities:
101 prompt = f"{_NO_THINK_DIRECTIVE}\n\n{prompt}"
102 return [{"role": "user", "content": prompt}]
105def wiki_generation_options(config: Config) -> dict[str, Any]:
106 """Sampling options for a wiki LLM call: wiki temperature, output cap, fixed seed."""
107 return config.generation_options(
108 temperature=config.wiki_temperature,
109 max_tokens=config.wiki_summary_max_tokens,
110 seed=WIKI_DEFAULT_SEED if config.seed is None else config.seed,
111 )
114def prompt_overhead_tokens(config: Config, prompt_chars: int | None = None) -> int:
115 """Token cost of the non-chunk prompt text plus the ``/no_think`` prefix.
117 ``prompt_chars`` is the rendered length of everything the prompt carries
118 besides the chunks, so per-call substitutions (concept instruction, entity
119 list, source list) are charged against the budget. Without it the longest
120 raw template stands in, measured rather than assumed so overriding a prompt
121 from settings moves the chunk budget with it.
122 """
123 longest = max(
124 len(config.wiki_synthesis_prompt),
125 len(config.wiki_entity_batch_prompt),
126 )
127 chars = longest if prompt_chars is None else prompt_chars
128 return -(-(chars + len(_NO_THINK_DIRECTIVE)) // _CHARS_PER_TOKEN)
131def truncate_chunks_to_budget(
132 chunks: list[SearchChunk],
133 config: Config,
134 prompt_chars: int | None = None,
135) -> list[SearchChunk]:
136 """Drop trailing chunks so the prompt plus its generation fits the context window.
138 The budget is ``num_ctx`` less the output cap (``wiki_summary_max_tokens``)
139 less the prompt overhead. The quarter-window floor applies only when that
140 leaves nothing: a positive budget is used as-is, so the floor can never
141 raise the budget past what the window actually has room for. Chunks are
142 measured as :func:`chunks_to_text` formats them, numbering and separators
143 included. Uses a chars/4 heuristic for token estimation.
144 """
145 context_window = config.num_ctx or DEFAULT_NUM_CTX
146 overhead = prompt_overhead_tokens(config, prompt_chars)
147 available = context_window - config.wiki_summary_max_tokens - overhead
148 budget_tokens = (
149 available if available > 0 else int(context_window * _MIN_CONTEXT_BUDGET_FRACTION)
150 )
151 budget_chars = budget_tokens * _CHARS_PER_TOKEN
153 total_chars = 0
154 kept: list[SearchChunk] = []
155 for index, chunk in enumerate(chunks):
156 chunk_chars = len(_format_chunk(chunk, index)) + len(_CHUNK_SEPARATOR)
157 if total_chars + chunk_chars > budget_chars and kept:
158 break
159 kept.append(chunk)
160 total_chars += chunk_chars
162 if len(kept) < len(chunks):
163 log.warning(
164 "Truncated chunks from %d to %d to fit context window (%d tokens)",
165 len(chunks),
166 len(kept),
167 context_window,
168 )
169 return kept
172def _format_chunk(chunk: SearchChunk, index: int) -> str:
173 """Render one chunk as the numbered block the prompt embeds."""
174 location = ""
175 if chunk.page_start:
176 location = f" (page {chunk.page_start})"
177 elif chunk.line_start:
178 location = f" (lines {chunk.line_start}-{chunk.line_end})"
179 return f"[Chunk {index + 1}]{location}:\n{chunk.chunk}"
182def chunks_to_text(chunks: list[SearchChunk]) -> str:
183 """Format chunks as numbered text blocks for the LLM prompt."""
184 return _CHUNK_SEPARATOR.join(_format_chunk(chunk, i) for i, chunk in enumerate(chunks))
187def build_frontmatter(
188 config: Config,
189 source_names: list[str],
190 score: float,
191 chunks: list[SearchChunk] | None = None,
192) -> str:
193 """Build YAML frontmatter for a wiki page.
195 When ``chunks`` is provided the frontmatter carries a ``provenance``
196 block naming the source/chunk-index pairs that fed the generator and
197 the extraction method from config, so a bad page is auditable
198 without re-running the pipeline.
199 """
200 # A JSON array is valid YAML flow syntax and escapes quotes/backslashes/
201 # unicode, so a filename like ``a"b\c.txt`` cannot corrupt the frontmatter.
202 sources_yaml = json.dumps(sorted(source_names))
203 provenance_block = render_provenance(config, chunks) if chunks is not None else ""
204 return (
205 f"---\n"
206 f"generated_by: {config.chat_model}\n"
207 f"generated_at: {datetime.now(UTC).isoformat()}\n"
208 f"sources: {sources_yaml}\n"
209 f"faithfulness_score: {score:.2f}\n"
210 f"{provenance_block}"
211 f"---\n\n"
212 )
215def write_page(
216 wiki_root: Path,
217 subdir: str,
218 slug: str,
219 full_content: str,
220 drift_threshold: float,
221 source_names: list[str],
222 page_type: str,
223) -> Path:
224 """Write page to disk with drift detection. Returns path written to.
226 ``slug`` may contain forward slashes (e.g. ``cv-manual/page-0042``);
227 any intermediate directories are created before writing. Publishing
228 retires this page's own drift draft: that proposal predates this body
229 and accepting it would undo the regen. Another source's draft under the
230 same slug is kept. A ``drafts/`` target skips drift entirely and routes
231 through :func:`_write_draft_page`; ``page_type`` is the subdir the page
232 would have published to, which a diverted page carries in its marker.
233 """
234 page_path = wiki_root / subdir / f"{slug}.md"
235 drafts_dir = wiki_root / WikiSubdir.DRAFTS
236 if subdir == WikiSubdir.DRAFTS:
237 return _write_draft_page(page_path, drafts_dir, slug, full_content, source_names, page_type)
239 if page_path.exists():
240 old_content = page_path.read_text(encoding="utf-8")
241 # Frontmatter carries a fresh timestamp and score on every regen, so the
242 # ratio is taken over body prose only.
243 ratio = content_change_ratio(extract_body(old_content), extract_body(full_content))
244 if ratio > drift_threshold:
245 diff_text = diff_summary(old_content, full_content)
246 return divert_to_drafts(
247 full_content, drafts_dir, slug, ratio, diff_text, subdir, source_names
248 )
250 atomic_write_text(page_path, full_content)
251 delete_drift_draft_if_present(drafts_dir, slug, source_names)
252 return page_path
255def _write_draft_page(
256 draft_path: Path,
257 drafts_dir: Path,
258 slug: str,
259 full_content: str,
260 source_names: list[str],
261 page_type: str,
262) -> Path:
263 """Land a below-threshold page in ``drafts/``, keeping another source's draft.
265 A newer proposal from the same sources supersedes the draft in place; one
266 from different sources lands at a collision draft rather than overwriting a
267 page awaiting review. No drift ratio is computed: a drafts target is a
268 proposal, not a published body it could be drifting from.
269 """
270 owner = draft_source_names(draft_path)
271 if owner is not None and owner != sorted(source_names):
272 return divert_concept_collision(
273 slug=slug,
274 source=", ".join(sorted(source_names)),
275 first_source=f"{WikiSubdir.DRAFTS}/{slug}.md",
276 content=full_content,
277 drafts_dir=drafts_dir,
278 origin_subdir=page_type,
279 )
280 atomic_write_text(draft_path, f"{origin_marker(page_type)}\n\n{full_content}")
281 return draft_path
284def assemble_content(
285 frontmatter: str,
286 wiki_text: str,
287 citation_block: str,
288) -> str:
289 """Combine frontmatter, body, and citations into the full page content."""
290 full = frontmatter + wiki_text
291 if citation_block:
292 full += "\n\n" + citation_block
293 return full
296def indexable_chunks(content: str) -> list[str]:
297 """The chunks a page's body would index as; empty means nothing to index.
299 One definition of "indexable" for the indexer and for the accept-time
300 refusal, because a non-empty body can still chunk to nothing ("#", "---")
301 and the two must not disagree about that.
302 """
303 body = extract_body(content).strip()
304 return chunk_text(body, mime_type="text/markdown", use_semantic=True) if body else []
307def index_wiki_page(
308 content: str,
309 wiki_source: str,
310 store: Store,
311 config: Config,
312 chunks: list[str] | None = None,
313) -> int:
314 """Chunk a wiki page body, embed it, and write rows with ``chunk_type="wiki"``.
316 ``wiki_source`` must follow the ``<config.wiki_dir>/<subdir>/<slug>.md``
317 shape (see :attr:`PageTarget.wiki_source`). Three branches:
319 - subdir in :data:`WIKI_CONTENT_SUBDIRS`: chunk, embed, then swap
320 the page's rows in one locked replace, so an embedder failure
321 leaves the previous rows searchable. Returns the row count.
322 - subdir is ``drafts/`` or ``archive/``: skip without touching the
323 store. Returns 0.
324 - malformed ``wiki_source`` (no subdir component): log.warning and
325 return 0. Does not raise because the caller set is narrow (only
326 internal wiki paths reach here) and surfacing the bad input in
327 the log is sufficient triage.
329 Record shape matches the markdown-ingest convention in
330 ``lilbee.data.ingest``: ``content_type="text"``, all four page/line
331 positions ``0`` (wiki pages are not paginated).
333 ``chunks`` lets a caller that already chunked this content pass the result
334 in rather than paying for a second semantic pass under the build lock.
335 """
336 subdir = subdir_from_wiki_source(wiki_source, config.wiki_dir)
337 if subdir is None:
338 log.warning("index_wiki_page: malformed wiki_source %r (no subdir)", wiki_source)
339 return 0
340 if subdir not in WIKI_CONTENT_SUBDIRS:
341 return 0
343 predicate = f"source = '{escape_sql_string(wiki_source)}' AND chunk_type = '{ChunkType.WIKI}'"
344 if chunks is None:
345 chunks = indexable_chunks(content)
346 if not chunks:
347 store.clear_table(CHUNKS_TABLE, predicate)
348 return 0
350 vectors = get_services().embedder.embed_batch(chunks)
351 records = [
352 {
353 "source": wiki_source,
354 "content_type": "text",
355 "chunk_type": ChunkType.WIKI,
356 "page_start": 0,
357 "page_end": 0,
358 "line_start": 0,
359 "line_end": 0,
360 "chunk": text,
361 "chunk_index": idx,
362 "vector": vector,
363 }
364 for idx, (text, vector) in enumerate(zip(chunks, vectors, strict=True))
365 ]
366 return store.replace_chunks(records, predicate)
369def generate_page(
370 label: str,
371 prompt: str,
372 chunks: list[SearchChunk],
373 citation_resolver: Callable[[list[ParsedCitation]], list[CitationRecord]],
374 page_type: str,
375 slug: str,
376 source_names: list[str],
377 provider: LLMProvider,
378 store: Store,
379 config: Config,
380 on_progress: WikiProgressCallback | None = None,
381 stats: BuildStats | None = None,
382 supersedes_sources: bool = True,
383) -> Path | None:
384 """Core generation pipeline shared by summary and synthesis pages.
386 ``supersedes_sources`` is False when the sources merely mention the
387 subject rather than being replaced by the page, so ``wiki_prune_raw``
388 does not delete them.
389 """
390 stats = BuildStats.ensure(stats)
392 def _emit(stage: str, **data: object) -> None:
393 if on_progress is not None:
394 on_progress(stage, data)
396 _emit("preparing", chunks=len(chunks), source=label)
398 messages = build_wiki_messages(prompt, provider, config)
399 _emit("generating", source=label)
400 options = wiki_generation_options(config)
401 try:
402 response = provider.chat(messages, stream=False, options=options)
403 wiki_text = strip_reasoning(response.text).strip()
404 except Exception as exc:
405 log.warning("LLM failed to generate wiki page for %s: %s", label, exc)
406 _emit("failed", error=str(exc))
407 return None
409 if not wiki_text:
410 log.warning("LLM returned empty response for wiki page %s", label)
411 _emit("failed", error="Model returned empty response")
412 return None
414 parsed_citations = parse_wiki_citations(wiki_text)
415 resolved = citation_resolver(parsed_citations)
416 verified = verify_citations(resolved, chunks, label, config)
417 stats.record_citations(
418 len(verified),
419 len(parsed_citations) - wiki_sourced_count(resolved, config) - len(verified),
420 )
421 if not verified:
422 log.warning("No valid citations for %s, skipping", label)
423 _emit("failed", error="No valid citations found")
424 return None
426 _emit("faithfulness_check")
427 score = check_faithfulness(chunks, wiki_text, label, config)
428 threshold = config.wiki_embedding_faithfulness_threshold
429 subdir = page_type if score >= threshold else WikiSubdir.DRAFTS
430 if subdir == WikiSubdir.DRAFTS:
431 log.info("Wiki page %s scored %.2f (< %.2f), sending to drafts", label, score, threshold)
433 wiki_text = scrub_unverified_markers(strip_citation_block(wiki_text), verified)
434 frontmatter = build_frontmatter(config, source_names, score, chunks=chunks)
435 citation_block = render_citation_block(verified)
436 full_content = assemble_content(frontmatter, wiki_text, citation_block)
438 wiki_root = config.data_root / config.wiki_dir
439 target = PageTarget(
440 wiki_root=wiki_root,
441 subdir=subdir,
442 slug=slug,
443 wiki_source=f"{config.wiki_dir}/{subdir}/{slug}.md",
444 page_type=page_type,
445 label=label,
446 supersedes_sources=supersedes_sources,
447 )
448 page_path = persist_and_finalize(
449 full_content, target, verified, source_names, store, config, stats=stats
450 )
452 log.info(
453 "Generated wiki page for %s -> %s (score=%.2f, citations=%d)",
454 label,
455 target.subdir,
456 score,
457 len(verified),
458 )
459 return page_path