Coverage for src/lilbee/wiki/lazy.py: 100%
81 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-11-06 08:17 +0000
1"""Generate one wiki page on demand, from the index rather than from a build.
3A build walks every source document and spends a call on each. This path walks
4one subject: it takes that subject's chunks across every source naming it and
5writes a single page. The cost is one call for the page someone actually asked
6for, and the evidence is wider than a build's, which assigns each entity to the
7one source mentioning it most and never sees the rest.
8"""
10from __future__ import annotations
12import functools
13import logging
14from contextlib import suppress
15from typing import TYPE_CHECKING
17from lilbee.app.services import get_services
18from lilbee.core.config import cfg
19from lilbee.core.text import clean_label_for_display
20from lilbee.runtime.lock import SyncMarkWaitCancelledError, source_keys_in_use
21from lilbee.runtime.progress import (
22 DetailedProgressCallback,
23 EventType,
24 WikiPageEvent,
25 WikiPhase,
26 WikiPhaseEvent,
27 noop_callback,
28)
30from .batch import hash_existing_sources
31from .citations import resolve_multi_source_citations
32from .generation import rewrite_links_across_wiki
33from .page import (
34 chunks_to_text,
35 generate_page,
36 truncate_chunks_to_budget,
37)
38from .shared import WIKI_BUILD_LOCK
39from .stubs import WikiStub, load_stub_index
41if TYPE_CHECKING:
42 import threading
43 from pathlib import Path
45 from lilbee.core.config import Config
46 from lilbee.data.store import CitationRecord, SearchChunk, Store
48 from .citations import ParsedCitation
49 from .stats import BuildStats
51log = logging.getLogger(__name__)
54class UnknownStubError(LookupError):
55 """Raised when a slug names no entry in the wiki index."""
58def _chunks_for_stub(stub: WikiStub, store: Store) -> tuple[dict[str, list[SearchChunk]], int]:
59 """The stub's chunks grouped by source, plus how many refs went unresolved.
61 Refs are looked up per source and filtered by index, so a source that was
62 re-ingested with fewer chunks contributes what it still has instead of
63 failing the page.
64 """
65 paths = store.source_paths(stub.sources)
66 wanted: dict[str, set[int]] = {}
67 for source_id, index in stub.chunk_refs:
68 if source_id in paths:
69 wanted.setdefault(paths[source_id], set()).add(index)
70 if not wanted:
71 # Subtracting a re-indexed source can empty the refs while leaving real
72 # evidence: the cap may have given every ref to the source that went.
73 # The recorded sources are the truth, so fall back to reading them.
74 wanted = {path: set() for path in paths.values()}
76 # Most-mentioning source first, so when the context budget truncates it
77 # drops the documents with least to say rather than the alphabetically
78 # unlucky ones.
79 counts = {paths[source_id]: n for source_id, n in stub.source_mentions if source_id in paths}
80 order = sorted(wanted, key=lambda name: (-counts.get(name, 0), name))
82 by_source: dict[str, list[SearchChunk]] = {}
83 resolved = 0
84 for source in order:
85 available = {c.chunk_index: c for c in store.get_chunks_by_source(source)}
86 indexes = wanted[source] or set(available)
87 kept = [available[i] for i in sorted(indexes) if i in available]
88 resolved += len(kept)
89 if kept:
90 by_source[source] = kept
91 return by_source, max(0, len(stub.chunk_refs) - resolved)
94def _resolve(slug: str, stubs: dict[str, WikiStub]) -> WikiStub | None:
95 """Find a stub by bare slug or by the subdir-qualified form.
97 Every surface shows pages as ``entities/ford``, so that is what a user
98 types, while the index is keyed by the bare slug. Both resolve.
99 """
100 direct = stubs.get(slug)
101 if direct is not None:
102 return direct
103 return next((stub for stub in stubs.values() if stub.wiki_slug == slug), None)
106def resolve_stub(slug: str, config: Config | None = None) -> WikiStub | None:
107 """The index entry for *slug*, by bare or subdir-qualified form."""
108 return _resolve(slug, load_stub_index(config or cfg))
111def generate_stub_page(
112 slug: str,
113 store: Store,
114 config: Config | None = None,
115 *,
116 stats: BuildStats | None = None,
117 on_progress: DetailedProgressCallback = noop_callback,
118 cancel: threading.Event | None = None,
119) -> Path | None:
120 """Write the page for one indexed subject. Returns its path, or None.
122 Runs the same citation verification, faithfulness gate, and drafts
123 quarantine a build does; nothing here bypasses them. Holds the wiki mutex,
124 so a page generated from the browse tree cannot interleave with a build,
125 and the sync mark, so no reset runs while the page is written.
126 Emits the same wiki_phase/wiki_page events a build does; a *cancel* set
127 before the model call skips it, and one set during the wait for the sync
128 mark ends the wait.
129 """
130 if config is None:
131 config = cfg
132 provider = get_services().provider
133 with (
134 suppress(SyncMarkWaitCancelledError),
135 source_keys_in_use(config.data_root, cancel),
136 WIKI_BUILD_LOCK,
137 ):
138 stubs = load_stub_index(config)
139 stub = _resolve(slug, stubs)
140 if stub is None:
141 raise UnknownStubError(f"no indexed page named {slug!r}")
143 chunks_by_source, unresolved = _chunks_for_stub(stub, store)
144 if not chunks_by_source:
145 log.warning("No chunks remain for %s; the index is stale", slug)
146 return None
147 if unresolved:
148 log.info("%d of %s's indexed chunks are gone", unresolved, slug)
150 on_progress(EventType.WIKI_PHASE, WikiPhaseEvent(phase=WikiPhase.GENERATE, total=1))
151 if cancel is not None and cancel.is_set():
152 log.info("Generation of %s cancelled before the model call", slug)
153 return None
155 source_names = sorted(chunks_by_source)
156 all_chunks = [c for chunks in chunks_by_source.values() for c in chunks]
157 source_list = "\n".join(f"- {name}" for name in source_names)
158 display = clean_label_for_display(stub.label)
159 render = functools.partial(
160 config.wiki_entity_page_prompt.format, topic=display, source_list=source_list
161 )
162 all_chunks = truncate_chunks_to_budget(all_chunks, config, len(render(chunks_text="")))
163 prompt = render(chunks_text=chunks_to_text(all_chunks))
164 source_hashes = hash_existing_sources(source_names)
166 def resolver(parsed: list[ParsedCitation]) -> list[CitationRecord]:
167 return resolve_multi_source_citations(
168 parsed, source_names, source_hashes, chunks_by_source
169 )
171 path = generate_page(
172 label=stub.label,
173 prompt=prompt,
174 chunks=all_chunks,
175 citation_resolver=resolver,
176 page_type=stub.subdir,
177 slug=stub.slug,
178 source_names=source_names,
179 provider=provider,
180 store=store,
181 config=config,
182 stats=stats,
183 # These documents mention the subject; the page does not replace
184 # them. Pruning here would delete every document that named it.
185 supersedes_sources=False,
186 )
187 if path is not None:
188 # The link pass a build runs; without it the page has no [[links]]
189 # and sits alone in the graph. No entities: the surface map is then
190 # built from the pages on disk, this one included, so links go both
191 # ways.
192 rewrite_links_across_wiki([], config)
193 on_progress(
194 EventType.WIKI_PAGE,
195 WikiPageEvent(label=stub.label, pages=1, current=1, total=1),
196 )
197 return path
198 log.info("Generation of %s cancelled during the wait for the sync mark", slug)
199 return None