Coverage for src/lilbee/server/wiki.py: 100%
174 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-11-06 08:17 +0000
1"""Wiki layer route handlers: page listing, reading, citations, lint, generation, pruning.
3Every route needs the token: pages are generated from the user's own corpus,
4so even the titles in a listing are their content.
5"""
7from __future__ import annotations
9import asyncio
10from functools import partial
11from pathlib import Path
12from typing import Any
14from litestar import MediaType, Response, delete, get, patch, post
15from litestar.exceptions import ClientException, NotFoundException
16from litestar.openapi.datastructures import ResponseSpec
17from litestar.params import FromPath, FromQuery
18from litestar.response import Stream
19from litestar.status_codes import HTTP_200_OK, HTTP_409_CONFLICT
21from lilbee.app import services as svc_mod
22from lilbee.core.config import cfg
23from lilbee.core.security import PathTraversalError
24from lilbee.data.store import Store
25from lilbee.server import handlers
26from lilbee.server.handlers.sse import SSE_MEDIA_TYPE
27from lilbee.server.models import (
28 DraftInfoResponse,
29 WikiBuildDryRunResult,
30 WikiCitationRecord,
31 WikiCitationsResult,
32 WikiDraftAcceptResponse,
33 WikiDraftDiffResponse,
34 WikiDraftRejectResponse,
35 WikiEntityCandidateResponse,
36 WikiIndexResult,
37 WikiLintIssueItem,
38 WikiLintResult,
39 WikiPageDetail,
40 WikiPruneRecordResponse,
41 WikiPruneResult,
42 WikiStatusResult,
43 WikiWipeResult,
44)
45from lilbee.wiki import lint as lint_mod
46from lilbee.wiki import prune as prune_mod
47from lilbee.wiki import wipe as wipe_mod
48from lilbee.wiki.browse import (
49 find_page,
50 list_pages,
51 read_page,
52)
53from lilbee.wiki.drafts import (
54 DraftAcceptError,
55 accept_draft,
56 diff_draft,
57 list_drafts,
58 reject_draft,
59)
60from lilbee.wiki.shared import (
61 INVALID_DRAFT_SLUG_ERROR,
62 WIKI_DISABLED_ERROR,
63 WikiSubdir,
64 total_wiki_pages,
65)
66from lilbee.wiki.stubs import load_stub_index, shown_sources, ungenerated_stubs
69def _wiki_root() -> Path:
70 """Resolve the wiki directory under data_root."""
71 return cfg.data_root / cfg.wiki_dir
74def _require_wiki() -> None:
75 """Raise 404 if the wiki feature is disabled."""
76 if not cfg.wiki:
77 raise NotFoundException(detail=WIKI_DISABLED_ERROR)
80def _find_page(slug: str) -> Path | None:
81 """Resolve a slug to a wiki page path via the browse module."""
82 return find_page(_wiki_root(), slug)
85@get("/api/wiki")
86async def wiki_list_route() -> list[dict[str, Any]]:
87 """List all wiki pages across subdirectories.
89 Reads the tree without touching it. The listing is built by walking pages,
90 and every path that changes the tree (build, update, synthesize, prune,
91 draft-accept) refreshes index.md itself, so a read never rewrites it.
92 """
93 _require_wiki()
94 # list_pages walks the whole tree; offload so the listing doesn't block
95 # the event loop.
96 pages = await asyncio.to_thread(list_pages, _wiki_root())
97 return [p.to_dict() for p in pages]
100@get("/api/wiki/drafts")
101async def wiki_drafts_route() -> list[DraftInfoResponse]:
102 """List pending wiki drafts with drift, faithfulness, and pending-marker info."""
103 _require_wiki()
104 # list_drafts reads every draft and stats its published counterpart;
105 # offload like the page listing.
106 drafts = await asyncio.to_thread(list_drafts, _wiki_root())
107 return [DraftInfoResponse(**d.to_dict()) for d in drafts]
110@get("/api/wiki/drafts/diff/{slug:path}")
111async def wiki_draft_diff_route(slug: FromPath[str]) -> WikiDraftDiffResponse:
112 """Return the unified diff of a draft against its published counterpart.
114 The ``diff`` action prefix precedes the slug because Litestar's
115 ``{slug:path}`` parameter is greedy and does not support a fixed
116 trailing segment. Keeping the action as a literal prefix lets
117 nested slugs (``cars/caprice``) flow through unchanged.
118 """
119 _require_wiki()
120 slug = slug.lstrip("/")
121 try:
122 # diff_draft reads both files and diffs them; offload like the listing.
123 store = svc_mod.get_services().store
124 diff = await asyncio.to_thread(diff_draft, slug, _wiki_root(), store)
125 except FileNotFoundError as exc:
126 raise NotFoundException(detail=f"draft not found: {slug}") from exc
127 except PathTraversalError as exc:
128 raise ClientException(detail=INVALID_DRAFT_SLUG_ERROR) from exc
129 return WikiDraftDiffResponse(slug=slug, diff=diff)
132@post("/api/wiki/drafts/accept/{slug:path}")
133async def wiki_draft_accept_route(slug: FromPath[str]) -> WikiDraftAcceptResponse:
134 """Accept a draft: overwrite the published page and re-index its chunks.
136 See :func:`wiki_draft_diff_route` for the action-prefix rationale.
137 """
138 _require_wiki()
139 slug = slug.lstrip("/")
140 store = svc_mod.get_services().store
141 try:
142 # accept_draft re-chunks and embeds, so it runs off the event loop; it
143 # takes the wiki build mutex itself.
144 result = await asyncio.to_thread(accept_draft, slug, _wiki_root(), store)
145 except FileNotFoundError as exc:
146 raise NotFoundException(detail=f"draft not found: {slug}") from exc
147 except DraftAcceptError as exc:
148 raise ClientException(detail=str(exc), status_code=HTTP_409_CONFLICT) from exc
149 except PathTraversalError as exc:
150 raise ClientException(detail=INVALID_DRAFT_SLUG_ERROR) from exc
151 return WikiDraftAcceptResponse(**result.to_dict())
154@delete("/api/wiki/drafts/{slug:path}", status_code=200)
155async def wiki_draft_reject_route(slug: FromPath[str]) -> WikiDraftRejectResponse:
156 """Reject a draft: delete the draft file without touching the published page."""
157 _require_wiki()
158 slug = slug.lstrip("/")
159 try:
160 # reject takes the wiki build mutex, so it runs off the event loop.
161 await asyncio.to_thread(reject_draft, slug, _wiki_root())
162 except FileNotFoundError as exc:
163 raise NotFoundException(detail=f"draft not found: {slug}") from exc
164 except PathTraversalError as exc:
165 raise ClientException(detail=INVALID_DRAFT_SLUG_ERROR) from exc
166 return WikiDraftRejectResponse(slug=slug)
169@get("/api/wiki/citations")
170async def wiki_citations_reverse_route(
171 source: FromQuery[str] = "",
172) -> list[WikiCitationRecord]:
173 """Reverse citation lookup: which wiki pages cite a given source."""
174 _require_wiki()
175 if not source:
176 raise ClientException(detail="pass ?source=<document path> to look up citing wiki pages")
177 # get_citations_for_source queries LanceDB; offload like wiki_lint_route.
178 records = await asyncio.to_thread(svc_mod.get_services().store.get_citations_for_source, source)
179 return [WikiCitationRecord(**r) for r in records]
182@get("/api/wiki/{slug:path}")
183async def wiki_read_route(slug: FromPath[str]) -> WikiPageDetail | WikiCitationsResult:
184 """Read a specific wiki page as markdown, or its citations."""
185 _require_wiki()
186 slug = slug.lstrip("/")
187 if slug.endswith("/citations"):
188 real_slug = slug.removesuffix("/citations")
189 return await _citations_for_slug(real_slug)
190 result = read_page(_wiki_root(), slug)
191 if result is None:
192 raise NotFoundException(detail=f"wiki page not found: {slug}")
193 return WikiPageDetail(
194 slug=result.slug,
195 title=result.title,
196 content=result.content,
197 frontmatter=result.frontmatter,
198 )
201async def _citations_for_slug(slug: str) -> WikiCitationsResult:
202 """Return citation chain for a wiki page."""
203 path = _find_page(slug)
204 if path is None:
205 raise NotFoundException(detail=f"wiki page not found: {slug}")
206 wiki_source = f"{cfg.wiki_dir}/{slug}.md"
207 # get_citations_for_wiki queries LanceDB; offload like wiki_lint_route.
208 records = await asyncio.to_thread(
209 svc_mod.get_services().store.get_citations_for_wiki, wiki_source
210 )
211 return WikiCitationsResult(slug=slug, citations=[WikiCitationRecord(**r) for r in records])
214@post("/api/wiki/lint")
215async def wiki_lint_route(
216 wiki_source: FromQuery[str] = "",
217) -> WikiLintResult:
218 """Lint the wiki; an empty ``wiki_source`` lints every page.
220 Same single-page argument as ``lilbee wiki lint <page>`` and the
221 ``wiki_lint`` MCP tool, and the same issue counts.
222 """
223 _require_wiki()
224 store = svc_mod.get_services().store
225 # Either arm reads every cited source and embeds; offload so a lint of a
226 # large wiki does not block the loop.
227 report = await asyncio.to_thread(_lint_report, wiki_source, store)
228 return WikiLintResult(
229 issues=[WikiLintIssueItem(**i.to_dict()) for i in report.issues],
230 total=len(report.issues),
231 errors=report.error_count,
232 warnings=report.warning_count,
233 )
236def _lint_report(wiki_source: str, store: Store) -> lint_mod.LintReport:
237 """Lint one page or the whole wiki, as the CLI and MCP surfaces do."""
238 if wiki_source:
239 return lint_mod.LintReport(issues=lint_mod.lint_wiki_page(wiki_source, store))
240 return lint_mod.lint_all(store)
243@post("/api/wiki/prune")
244async def wiki_prune_route() -> WikiPruneResult:
245 """Trigger pruning of stale/orphaned wiki pages."""
246 _require_wiki()
247 # prune walks the whole tree and store, so it runs off the event loop; it
248 # takes the wiki build mutex itself.
249 report = await asyncio.to_thread(prune_mod.prune_wiki, svc_mod.get_services().store)
250 return WikiPruneResult(
251 records=[WikiPruneRecordResponse(**r.to_dict()) for r in report.records],
252 archived=report.archived_count,
253 flagged=report.flagged_count,
254 reconciled=report.reconciled_count,
255 )
258@post("/api/wiki/index")
259async def wiki_index_route() -> WikiIndexResult:
260 """Rebuild the browse index of pages the corpus could have.
262 Spends no LLM call. Extraction walks every chunk, so it runs off the event
263 loop and takes the wiki build mutex itself.
264 """
265 _require_wiki()
266 from lilbee.wiki.stubs import refresh_stub_index
268 stubs = await asyncio.to_thread(refresh_stub_index, svc_mod.get_services().store)
269 return WikiIndexResult(entries=len(stubs))
272@get("/api/wiki/stubs")
273async def wiki_stubs_route() -> list[WikiEntityCandidateResponse]:
274 """List the pages the corpus names that nothing has written yet.
276 The other half of the browse tree: ``GET /api/wiki`` walks written pages,
277 this lists the ones a client can ask for. Entries whose page now exists are
278 left out, so a client never offers to regenerate a live page. Costs no LLM
279 call, and ``slug`` is what ``POST /api/wiki/generate/{slug}`` takes.
280 """
281 _require_wiki()
282 # Reading the index stats every candidate's page and draft; offload like
283 # the page listing.
284 return await asyncio.to_thread(_ungenerated_candidates)
287def _ungenerated_candidates() -> list[WikiEntityCandidateResponse]:
288 """The indexed subjects with no page yet, in slug order, each naming its documents."""
289 store = svc_mod.get_services().store
290 return [
291 WikiEntityCandidateResponse(
292 slug=stub.slug,
293 label=stub.label,
294 kind=stub.kind,
295 type_hint=stub.type_hint,
296 mentions=stub.mentions,
297 sources=shown_sources(stub, store),
298 )
299 for stub in ungenerated_stubs(load_stub_index(), _wiki_root())
300 ]
303@post("/api/wiki/generate/{slug:path}", media_type=SSE_MEDIA_TYPE)
304async def wiki_generate_route(slug: FromPath[str]) -> Stream:
305 """Generate one indexed page, streaming progress. Costs a single LLM call.
307 One model call takes long enough to look hung, so the response is an SSE
308 stream like /api/wiki/build: wiki_phase and wiki_page events, then a done
309 event carrying the slug the read route accepts. 404 when the slug names
310 nothing in the index; an entry whose sources are gone surfaces as an
311 error event, since the run discovers it after the stream has started.
312 """
313 _require_wiki()
314 slug = slug.lstrip("/")
315 from lilbee.wiki.lazy import resolve_stub
317 # Resolving reads the index file; offload like the stub listing.
318 if await asyncio.to_thread(resolve_stub, slug) is None:
319 raise NotFoundException(detail=f"no indexed page named {slug!r}")
320 return Stream(handlers.wiki_generate_stream(slug), media_type=SSE_MEDIA_TYPE)
323@delete("/api/wiki", status_code=200)
324async def wiki_wipe_route() -> WikiWipeResult:
325 """Delete every generated wiki page and its indexed rows.
327 Answers while the wiki is disabled, unlike the other write routes: turning
328 the setting off is exactly when a client needs to clear what was already
329 generated. The wipe touches the whole tree and the store, so it runs off
330 the event loop and takes the wiki build mutex itself.
331 """
332 report = await asyncio.to_thread(wipe_mod.wipe_wiki, svc_mod.get_services().store)
333 return WikiWipeResult(
334 pages_removed=report.pages_removed,
335 sources_cleared=report.sources_cleared,
336 rows_deleted=report.rows_deleted,
337 )
340@post(
341 "/api/wiki/build",
342 media_type=SSE_MEDIA_TYPE,
343 responses={
344 HTTP_200_OK: ResponseSpec(
345 data_container=WikiBuildDryRunResult,
346 media_type=MediaType.JSON,
347 generate_examples=False,
348 description="Entity candidates a build would cover, when dry_run=true.",
349 )
350 },
351)
352async def wiki_build_route(
353 dry_run: FromQuery[bool] = False,
354) -> Stream | Response[WikiBuildDryRunResult]:
355 """Build the concept and entity wiki across all ingested sources.
357 A build issues per-source LLM calls and embeddings and can run for a long
358 time, so the response is 201 with an SSE stream: wiki_phase and wiki_page
359 events while it runs, then a done event carrying the summary. The work runs
360 in a worker thread and holds the wiki build mutex, so a second request
361 streams its own progress only once the first run finishes.
363 ``dry_run=true`` creates nothing, so it answers 200 with plain JSON: the
364 NER entity candidates a build would cover, with no LLM call made. The two
365 arms carry different content types, so each is declared under its own
366 status code.
367 """
368 _require_wiki()
369 if dry_run:
370 return await _build_dry_run()
371 return Stream(handlers.wiki_build_stream(), media_type=SSE_MEDIA_TYPE)
374async def _build_dry_run() -> Response[WikiBuildDryRunResult]:
375 """Extract entity candidates off the event loop and shape them for the wire."""
376 from lilbee.wiki.generation import DRY_RUN_CONCEPT_NOTE, preview_build_entities
378 rows = await asyncio.to_thread(preview_build_entities, cfg)
379 result = WikiBuildDryRunResult(
380 entities=[WikiEntityCandidateResponse(**row) for row in rows],
381 count=len(rows),
382 note=DRY_RUN_CONCEPT_NOTE,
383 )
384 return Response(result, media_type=MediaType.JSON, status_code=HTTP_200_OK)
387@patch("/api/wiki/update", media_type=SSE_MEDIA_TYPE)
388async def wiki_update_route() -> Stream:
389 """Refresh the concept and entity wiki after an ingest.
391 A full rebuild, streamed like /api/wiki/build.
392 """
393 _require_wiki()
394 return Stream(handlers.wiki_build_stream(), media_type=SSE_MEDIA_TYPE)
397@post("/api/wiki/synthesize", media_type=SSE_MEDIA_TYPE)
398async def wiki_synthesize_route() -> Stream:
399 """Generate synthesis pages for concept clusters spanning 3+ sources.
401 Streams per-cluster progress and shares the wiki build mutex, so synthesis
402 can't race a build over the same on-disk wiki tree.
403 """
404 _require_wiki()
405 return Stream(handlers.wiki_synthesize_stream(), media_type=SSE_MEDIA_TYPE)
408@get("/api/wiki/status")
409async def wiki_status_route() -> WikiStatusResult:
410 """Wiki layer status: page counts and recent lint counts.
412 Answers while the wiki is disabled so a client can render the disabled
413 state without a second round trip to /api/config.
414 """
415 root = _wiki_root()
416 if not cfg.wiki or not root.exists():
417 # A disabled wiki can still have a tree left over from an earlier
418 # build; report the disabled state rather than linting it.
419 return WikiStatusResult(wiki_enabled=cfg.wiki)
421 summaries_dir = root / WikiSubdir.SUMMARIES
422 drafts_dir = root / WikiSubdir.DRAFTS
423 summaries = list(summaries_dir.rglob("*.md")) if summaries_dir.exists() else []
424 drafts = list(drafts_dir.rglob("*.md")) if drafts_dir.exists() else []
426 # record_log=False: a status poll is a read, and appending a LINT entry to
427 # log.md on every poll is what that flag exists to avoid. The MCP and CLI
428 # status surfaces already pass it.
429 report = await asyncio.to_thread(
430 partial(lint_mod.lint_all, svc_mod.get_services().store, record_log=False)
431 )
432 return WikiStatusResult(
433 wiki_enabled=cfg.wiki,
434 summaries=len(summaries),
435 drafts=len(drafts),
436 pages=total_wiki_pages(root),
437 lint_errors=report.error_count,
438 lint_warnings=report.warning_count,
439 )