Coverage for src/lilbee/server/wiki.py: 100%
173 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-08-14 11:46 +0000
1"""Wiki layer route handlers: page listing, reading, citations, lint, generation, pruning.
3Every route needs the token: pages are generated from the user's own corpus,
4so even the titles in a listing are their content.
5"""
7from __future__ import annotations
9import asyncio
10from functools import partial
11from pathlib import Path
12from typing import Any
14from litestar import MediaType, Response, delete, get, patch, post
15from litestar.exceptions import ClientException, NotFoundException
16from litestar.openapi.datastructures import ResponseSpec
17from litestar.params import FromPath, FromQuery
18from litestar.response import Stream
19from litestar.status_codes import HTTP_200_OK, HTTP_409_CONFLICT
21from lilbee.app import services as svc_mod
22from lilbee.core.config import cfg
23from lilbee.core.security import PathTraversalError
24from lilbee.data.store import Store
25from lilbee.server import handlers
26from lilbee.server.handlers.sse import SSE_MEDIA_TYPE
27from lilbee.server.models import (
28 DraftInfoResponse,
29 WikiBuildDryRunResult,
30 WikiCitationRecord,
31 WikiCitationsResult,
32 WikiDraftAcceptResponse,
33 WikiDraftDiffResponse,
34 WikiDraftRejectResponse,
35 WikiEntityCandidateResponse,
36 WikiIndexResult,
37 WikiLintIssueItem,
38 WikiLintResult,
39 WikiPageDetail,
40 WikiPruneRecordResponse,
41 WikiPruneResult,
42 WikiStatusResult,
43 WikiWipeResult,
44)
45from lilbee.wiki import lint as lint_mod
46from lilbee.wiki import prune as prune_mod
47from lilbee.wiki import wipe as wipe_mod
48from lilbee.wiki.browse import (
49 find_page,
50 list_pages,
51 read_page,
52)
53from lilbee.wiki.drafts import (
54 DraftAcceptError,
55 accept_draft,
56 diff_draft,
57 list_drafts,
58 reject_draft,
59)
60from lilbee.wiki.shared import (
61 INVALID_DRAFT_SLUG_ERROR,
62 WIKI_DISABLED_ERROR,
63 WikiSubdir,
64 total_wiki_pages,
65)
66from lilbee.wiki.stubs import WikiStub, load_stub_index, ungenerated_stubs
69def _wiki_root() -> Path:
70 """Resolve the wiki directory under data_root."""
71 return cfg.data_root / cfg.wiki_dir
74def _require_wiki() -> None:
75 """Raise 404 if the wiki feature is disabled."""
76 if not cfg.wiki:
77 raise NotFoundException(detail=WIKI_DISABLED_ERROR)
80def _find_page(slug: str) -> Path | None:
81 """Resolve a slug to a wiki page path via the browse module."""
82 return find_page(_wiki_root(), slug)
85@get("/api/wiki")
86async def wiki_list_route() -> list[dict[str, Any]]:
87 """List all wiki pages across subdirectories.
89 Reads the tree without touching it. The listing is built by walking pages,
90 and every path that changes the tree (build, update, synthesize, prune,
91 draft-accept) refreshes index.md itself, so a read never rewrites it.
92 """
93 _require_wiki()
94 # list_pages walks the whole tree; offload so the listing doesn't block
95 # the event loop.
96 pages = await asyncio.to_thread(list_pages, _wiki_root())
97 return [p.to_dict() for p in pages]
100@get("/api/wiki/drafts")
101async def wiki_drafts_route() -> list[DraftInfoResponse]:
102 """List pending wiki drafts with drift, faithfulness, and pending-marker info."""
103 _require_wiki()
104 # list_drafts reads every draft and stats its published counterpart;
105 # offload like the page listing.
106 drafts = await asyncio.to_thread(list_drafts, _wiki_root())
107 return [DraftInfoResponse(**d.to_dict()) for d in drafts]
110@get("/api/wiki/drafts/diff/{slug:path}")
111async def wiki_draft_diff_route(slug: FromPath[str]) -> WikiDraftDiffResponse:
112 """Return the unified diff of a draft against its published counterpart.
114 The ``diff`` action prefix precedes the slug because Litestar's
115 ``{slug:path}`` parameter is greedy and does not support a fixed
116 trailing segment. Keeping the action as a literal prefix lets
117 nested slugs (``cars/caprice``) flow through unchanged.
118 """
119 _require_wiki()
120 slug = slug.lstrip("/")
121 try:
122 # diff_draft reads both files and diffs them; offload like the listing.
123 diff = await asyncio.to_thread(diff_draft, slug, _wiki_root())
124 except FileNotFoundError as exc:
125 raise NotFoundException(detail=f"draft not found: {slug}") from exc
126 except PathTraversalError as exc:
127 raise ClientException(detail=INVALID_DRAFT_SLUG_ERROR) from exc
128 return WikiDraftDiffResponse(slug=slug, diff=diff)
131@post("/api/wiki/drafts/accept/{slug:path}")
132async def wiki_draft_accept_route(slug: FromPath[str]) -> WikiDraftAcceptResponse:
133 """Accept a draft: overwrite the published page and re-index its chunks.
135 See :func:`wiki_draft_diff_route` for the action-prefix rationale.
136 """
137 _require_wiki()
138 slug = slug.lstrip("/")
139 store = svc_mod.get_services().store
140 try:
141 # accept_draft re-chunks and embeds, so it runs off the event loop; it
142 # takes the wiki build mutex itself.
143 result = await asyncio.to_thread(accept_draft, slug, _wiki_root(), store)
144 except FileNotFoundError as exc:
145 raise NotFoundException(detail=f"draft not found: {slug}") from exc
146 except DraftAcceptError as exc:
147 raise ClientException(detail=str(exc), status_code=HTTP_409_CONFLICT) from exc
148 except PathTraversalError as exc:
149 raise ClientException(detail=INVALID_DRAFT_SLUG_ERROR) from exc
150 return WikiDraftAcceptResponse(**result.to_dict())
153@delete("/api/wiki/drafts/{slug:path}", status_code=200)
154async def wiki_draft_reject_route(slug: FromPath[str]) -> WikiDraftRejectResponse:
155 """Reject a draft: delete the draft file without touching the published page."""
156 _require_wiki()
157 slug = slug.lstrip("/")
158 try:
159 # reject takes the wiki build mutex, so it runs off the event loop.
160 await asyncio.to_thread(reject_draft, slug, _wiki_root())
161 except FileNotFoundError as exc:
162 raise NotFoundException(detail=f"draft not found: {slug}") from exc
163 except PathTraversalError as exc:
164 raise ClientException(detail=INVALID_DRAFT_SLUG_ERROR) from exc
165 return WikiDraftRejectResponse(slug=slug)
168@get("/api/wiki/citations")
169async def wiki_citations_reverse_route(
170 source: FromQuery[str] = "",
171) -> list[WikiCitationRecord]:
172 """Reverse citation lookup: which wiki pages cite a given source."""
173 _require_wiki()
174 if not source:
175 raise ClientException(detail="pass ?source=<document path> to look up citing wiki pages")
176 # get_citations_for_source queries LanceDB; offload like wiki_lint_route.
177 records = await asyncio.to_thread(svc_mod.get_services().store.get_citations_for_source, source)
178 return [WikiCitationRecord(**r) for r in records]
181@get("/api/wiki/{slug:path}")
182async def wiki_read_route(slug: FromPath[str]) -> WikiPageDetail | WikiCitationsResult:
183 """Read a specific wiki page as markdown, or its citations."""
184 _require_wiki()
185 slug = slug.lstrip("/")
186 if slug.endswith("/citations"):
187 real_slug = slug.removesuffix("/citations")
188 return await _citations_for_slug(real_slug)
189 result = read_page(_wiki_root(), slug)
190 if result is None:
191 raise NotFoundException(detail=f"wiki page not found: {slug}")
192 return WikiPageDetail(
193 slug=result.slug,
194 title=result.title,
195 content=result.content,
196 frontmatter=result.frontmatter,
197 )
200async def _citations_for_slug(slug: str) -> WikiCitationsResult:
201 """Return citation chain for a wiki page."""
202 path = _find_page(slug)
203 if path is None:
204 raise NotFoundException(detail=f"wiki page not found: {slug}")
205 wiki_source = f"{cfg.wiki_dir}/{slug}.md"
206 # get_citations_for_wiki queries LanceDB; offload like wiki_lint_route.
207 records = await asyncio.to_thread(
208 svc_mod.get_services().store.get_citations_for_wiki, wiki_source
209 )
210 return WikiCitationsResult(slug=slug, citations=[WikiCitationRecord(**r) for r in records])
213@post("/api/wiki/lint")
214async def wiki_lint_route(
215 wiki_source: FromQuery[str] = "",
216) -> WikiLintResult:
217 """Lint the wiki; an empty ``wiki_source`` lints every page.
219 Same single-page argument as ``lilbee wiki lint <page>`` and the
220 ``wiki_lint`` MCP tool, and the same issue counts.
221 """
222 _require_wiki()
223 store = svc_mod.get_services().store
224 # Either arm reads every cited source and embeds; offload so a lint of a
225 # large wiki does not block the loop.
226 report = await asyncio.to_thread(_lint_report, wiki_source, store)
227 return WikiLintResult(
228 issues=[WikiLintIssueItem(**i.to_dict()) for i in report.issues],
229 total=len(report.issues),
230 errors=report.error_count,
231 warnings=report.warning_count,
232 )
235def _lint_report(wiki_source: str, store: Store) -> lint_mod.LintReport:
236 """Lint one page or the whole wiki, as the CLI and MCP surfaces do."""
237 if wiki_source:
238 return lint_mod.LintReport(issues=lint_mod.lint_wiki_page(wiki_source, store))
239 return lint_mod.lint_all(store)
242@post("/api/wiki/prune")
243async def wiki_prune_route() -> WikiPruneResult:
244 """Trigger pruning of stale/orphaned wiki pages."""
245 _require_wiki()
246 # prune walks the whole tree and store, so it runs off the event loop; it
247 # takes the wiki build mutex itself.
248 report = await asyncio.to_thread(prune_mod.prune_wiki, svc_mod.get_services().store)
249 return WikiPruneResult(
250 records=[WikiPruneRecordResponse(**r.to_dict()) for r in report.records],
251 archived=report.archived_count,
252 flagged=report.flagged_count,
253 reconciled=report.reconciled_count,
254 )
257@post("/api/wiki/index")
258async def wiki_index_route() -> WikiIndexResult:
259 """Rebuild the browse index of pages the corpus could have.
261 Spends no LLM call. Extraction walks every chunk, so it runs off the event
262 loop and takes the wiki build mutex itself.
263 """
264 _require_wiki()
265 from lilbee.wiki.stubs import refresh_stub_index
267 stubs = await asyncio.to_thread(refresh_stub_index, svc_mod.get_services().store)
268 return WikiIndexResult(entries=len(stubs))
271@get("/api/wiki/stubs")
272async def wiki_stubs_route() -> list[WikiEntityCandidateResponse]:
273 """List the pages the corpus names that nothing has written yet.
275 The other half of the browse tree: ``GET /api/wiki`` walks written pages,
276 this lists the ones a client can ask for. Entries whose page now exists are
277 left out, so a client never offers to regenerate a live page. Costs no LLM
278 call, and ``slug`` is what ``POST /api/wiki/generate/{slug}`` takes.
279 """
280 _require_wiki()
281 # Reading the index stats every candidate's page and draft; offload like
282 # the page listing.
283 stubs = await asyncio.to_thread(_ungenerated_stubs)
284 return [
285 WikiEntityCandidateResponse(
286 slug=stub.slug,
287 label=stub.label,
288 kind=stub.kind,
289 type_hint=stub.type_hint,
290 mentions=stub.mentions,
291 sources=list(stub.sources),
292 )
293 for stub in stubs
294 ]
297def _ungenerated_stubs() -> list[WikiStub]:
298 """The indexed subjects with no page yet, in slug order."""
299 return ungenerated_stubs(load_stub_index(), _wiki_root())
302@post("/api/wiki/generate/{slug:path}", media_type=SSE_MEDIA_TYPE)
303async def wiki_generate_route(slug: FromPath[str]) -> Stream:
304 """Generate one indexed page, streaming progress. Costs a single LLM call.
306 One model call takes long enough to look hung, so the response is an SSE
307 stream like /api/wiki/build: wiki_phase and wiki_page events, then a done
308 event carrying the slug the read route accepts. 404 when the slug names
309 nothing in the index; an entry whose sources are gone surfaces as an
310 error event, since the run discovers it after the stream has started.
311 """
312 _require_wiki()
313 slug = slug.lstrip("/")
314 from lilbee.wiki.lazy import resolve_stub
316 # Resolving reads the index file; offload like the stub listing.
317 if await asyncio.to_thread(resolve_stub, slug) is None:
318 raise NotFoundException(detail=f"no indexed page named {slug!r}")
319 return Stream(handlers.wiki_generate_stream(slug), media_type=SSE_MEDIA_TYPE)
322@delete("/api/wiki", status_code=200)
323async def wiki_wipe_route() -> WikiWipeResult:
324 """Delete every generated wiki page and its indexed rows.
326 Answers while the wiki is disabled, unlike the other write routes: turning
327 the setting off is exactly when a client needs to clear what was already
328 generated. The wipe touches the whole tree and the store, so it runs off
329 the event loop and takes the wiki build mutex itself.
330 """
331 report = await asyncio.to_thread(wipe_mod.wipe_wiki, svc_mod.get_services().store)
332 return WikiWipeResult(
333 pages_removed=report.pages_removed,
334 sources_cleared=report.sources_cleared,
335 rows_deleted=report.rows_deleted,
336 )
339@post(
340 "/api/wiki/build",
341 media_type=SSE_MEDIA_TYPE,
342 responses={
343 HTTP_200_OK: ResponseSpec(
344 data_container=WikiBuildDryRunResult,
345 media_type=MediaType.JSON,
346 generate_examples=False,
347 description="Entity candidates a build would cover, when dry_run=true.",
348 )
349 },
350)
351async def wiki_build_route(
352 dry_run: FromQuery[bool] = False,
353) -> Stream | Response[WikiBuildDryRunResult]:
354 """Build the concept and entity wiki across all ingested sources.
356 A build issues per-source LLM calls and embeddings and can run for a long
357 time, so the response is 201 with an SSE stream: wiki_phase and wiki_page
358 events while it runs, then a done event carrying the summary. The work runs
359 in a worker thread and holds the wiki build mutex, so a second request
360 streams its own progress only once the first run finishes.
362 ``dry_run=true`` creates nothing, so it answers 200 with plain JSON: the
363 NER entity candidates a build would cover, with no LLM call made. The two
364 arms carry different content types, so each is declared under its own
365 status code.
366 """
367 _require_wiki()
368 if dry_run:
369 return await _build_dry_run()
370 return Stream(handlers.wiki_build_stream(), media_type=SSE_MEDIA_TYPE)
373async def _build_dry_run() -> Response[WikiBuildDryRunResult]:
374 """Extract entity candidates off the event loop and shape them for the wire."""
375 from lilbee.wiki.generation import DRY_RUN_CONCEPT_NOTE, preview_build_entities
377 rows = await asyncio.to_thread(preview_build_entities, cfg)
378 result = WikiBuildDryRunResult(
379 entities=[WikiEntityCandidateResponse(**row) for row in rows],
380 count=len(rows),
381 note=DRY_RUN_CONCEPT_NOTE,
382 )
383 return Response(result, media_type=MediaType.JSON, status_code=HTTP_200_OK)
386@patch("/api/wiki/update", media_type=SSE_MEDIA_TYPE)
387async def wiki_update_route() -> Stream:
388 """Refresh the concept and entity wiki after an ingest.
390 A full rebuild, streamed like /api/wiki/build.
391 """
392 _require_wiki()
393 return Stream(handlers.wiki_build_stream(), media_type=SSE_MEDIA_TYPE)
396@post("/api/wiki/synthesize", media_type=SSE_MEDIA_TYPE)
397async def wiki_synthesize_route() -> Stream:
398 """Generate synthesis pages for concept clusters spanning 3+ sources.
400 Streams per-cluster progress and shares the wiki build mutex, so synthesis
401 can't race a build over the same on-disk wiki tree.
402 """
403 _require_wiki()
404 return Stream(handlers.wiki_synthesize_stream(), media_type=SSE_MEDIA_TYPE)
407@get("/api/wiki/status")
408async def wiki_status_route() -> WikiStatusResult:
409 """Wiki layer status: page counts and recent lint counts.
411 Answers while the wiki is disabled so a client can render the disabled
412 state without a second round trip to /api/config.
413 """
414 root = _wiki_root()
415 if not cfg.wiki or not root.exists():
416 # A disabled wiki can still have a tree left over from an earlier
417 # build; report the disabled state rather than linting it.
418 return WikiStatusResult(wiki_enabled=cfg.wiki)
420 summaries_dir = root / WikiSubdir.SUMMARIES
421 drafts_dir = root / WikiSubdir.DRAFTS
422 summaries = list(summaries_dir.rglob("*.md")) if summaries_dir.exists() else []
423 drafts = list(drafts_dir.rglob("*.md")) if drafts_dir.exists() else []
425 # record_log=False: a status poll is a read, and appending a LINT entry to
426 # log.md on every poll is what that flag exists to avoid. The MCP and CLI
427 # status surfaces already pass it.
428 report = await asyncio.to_thread(
429 partial(lint_mod.lint_all, svc_mod.get_services().store, record_log=False)
430 )
431 return WikiStatusResult(
432 wiki_enabled=cfg.wiki,
433 summaries=len(summaries),
434 drafts=len(drafts),
435 pages=total_wiki_pages(root),
436 lint_errors=report.error_count,
437 lint_warnings=report.warning_count,
438 )