Coverage for src/lilbee/wiki/prune.py: 100%
148 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-09-17 10:02 +0000
1"""Prune stale and orphaned wiki pages.
3Pruning rules:
41. All cited sources deleted -> archive the page
52. Synthesis cluster shrinks below MIN_CLUSTER_SOURCES live sources -> archive the page
63. Stale citations (stale_hash or excerpt_missing) exceed
7 ``wiki_stale_citation_threshold`` -> flag the page in the prune report
9Archived pages are moved to wiki/archive/ and removed from the vector store.
10"""
12from __future__ import annotations
14import logging
15import shutil
16import threading
17from dataclasses import dataclass, field
18from enum import Enum
19from pathlib import Path
21from lilbee.core.config import Config, cfg
22from lilbee.data.store import Store
23from lilbee.data.store.types import CitationRecord
24from lilbee.wiki.index import append_wiki_log, update_wiki_index
25from lilbee.wiki.lint import IssueType, lint_wiki_page
26from lilbee.wiki.persistence import subdir_from_wiki_source
27from lilbee.wiki.shared import (
28 MIN_CLUSTER_SOURCES,
29 WIKI_BUILD_LOCK,
30 WIKI_CONTENT_SUBDIRS,
31 WikiLogAction,
32 WikiSubdir,
33)
35log = logging.getLogger(__name__)
37_STALE_TYPES = {IssueType.STALE_HASH, IssueType.EXCERPT_MISSING}
40class PruneAction(Enum):
41 """What happened to a wiki page during pruning."""
43 ARCHIVED = "archived"
44 FLAGGED = "flagged"
45 RECONCILED = "reconciled"
48@dataclass(frozen=True)
49class PruneRecord:
50 """A single pruning action taken on a wiki page."""
52 wiki_source: str
53 action: PruneAction
54 reason: str
56 def to_dict(self) -> dict[str, str]:
57 """Serialize to a plain dict suitable for JSON output."""
58 return {
59 "wiki_source": self.wiki_source,
60 "action": self.action.value,
61 "reason": self.reason,
62 }
65@dataclass
66class PruneReport:
67 """Aggregated results from pruning wiki pages."""
69 records: list[PruneRecord] = field(default_factory=list)
71 @property
72 def archived_count(self) -> int:
73 return sum(1 for r in self.records if r.action == PruneAction.ARCHIVED)
75 @property
76 def flagged_count(self) -> int:
77 return sum(1 for r in self.records if r.action == PruneAction.FLAGGED)
79 @property
80 def reconciled_count(self) -> int:
81 return sum(1 for r in self.records if r.action == PruneAction.RECONCILED)
84def _delete_wiki_rows(wiki_source: str, store: Store) -> bool:
85 """Delete a wiki page's chunk and citation rows. Returns whether it succeeded."""
86 try:
87 store.delete_by_source(wiki_source)
88 if not store.delete_citations_for_wiki(wiki_source):
89 log.warning(
90 "Citation delete failed for %s; the next prune pass retries it", wiki_source
91 )
92 return False
93 except Exception:
94 log.warning(
95 "Failed to delete store rows for %s; the next prune pass retries them",
96 wiki_source,
97 exc_info=True,
98 )
99 return False
100 return True
103def _archive_page(
104 wiki_source: str,
105 wiki_root: Path,
106 store: Store,
107 config: Config,
108) -> None:
109 """Move a wiki page to wiki/archive/, then delete its store rows.
111 The file moves first so a crash or a failed delete leaves rows without a
112 page, which :func:`_reconcile_orphan_rows` deletes on the next pass.
113 Deleting rows first would leave a page on disk with no citations, and every
114 archival check reads an uncited page as "nothing to do", so nothing would
115 ever retry it.
116 """
117 relative = wiki_source.removeprefix(config.wiki_dir + "/")
118 source_path = wiki_root / relative
120 # Mirror the source subdir under archive/ (archive/concepts/foo.md), not a flat
121 # archive/foo.md: same-slug pages from different subdirs would overwrite there.
122 archive_path = wiki_root / WikiSubdir.ARCHIVE / relative
123 archive_path.parent.mkdir(parents=True, exist_ok=True)
125 if source_path.exists():
126 shutil.move(source_path, archive_path)
127 log.info("Archived wiki page %s -> %s", source_path, archive_path)
128 else:
129 log.warning("Wiki page file not found for archival: %s", source_path)
130 _delete_wiki_rows(wiki_source, store)
133def _live_sources(citations: list[CitationRecord], store: Store, config: Config) -> set[str]:
134 """Return the distinct cited source files still backing a page.
136 A document source is live only when it is both registered in the store and
137 present on disk. ``lilbee remove`` unregisters a source while its file may
138 remain on disk, and an on-disk delete removes the file; either drops it. A
139 source whose raw chunks were pruned at publish (``wiki_prune_raw``) stays
140 registered and so stays live. Wiki-page sources (a synthesis page citing
141 other pages) never appear in the document registry, so they are judged by
142 file presence alone.
143 """
144 from lilbee.data.ingest.discovery import resolve_source_path
146 registered = store.source_ingested_at_map()
147 wiki_prefix = config.wiki_dir + "/"
148 live: set[str] = set()
149 for source_filename in {c["source_filename"] for c in citations}:
150 if not resolve_source_path(source_filename).exists():
151 continue
152 if source_filename.startswith(wiki_prefix) or source_filename in registered:
153 live.add(source_filename)
154 return live
157def _check_all_sources_deleted(
158 wiki_source: str,
159 store: Store,
160 config: Config | None = None,
161) -> bool:
162 """Return True if every cited source has left the library (file deleted or
163 the source removed from the index)."""
164 if config is None:
165 config = cfg
166 citations = store.get_citations_for_wiki(wiki_source)
167 if not citations:
168 return False
169 return not _live_sources(citations, store, config)
172def _check_cluster_below_threshold(
173 wiki_source: str,
174 store: Store,
175 config: Config,
176 min_sources: int = MIN_CLUSTER_SOURCES,
177) -> bool:
178 """Return True if a synthesis page's live source count dropped below min_sources."""
179 # Match the subdir component, not the substring: a summary whose slug is
180 # ``synthesis/report`` or a wiki_dir ending in ``synthesis`` would satisfy a
181 # substring test and get its rows deleted as a shrunken cluster.
182 if subdir_from_wiki_source(wiki_source, config.wiki_dir) != WikiSubdir.SYNTHESIS:
183 return False
184 citations = store.get_citations_for_wiki(wiki_source)
185 if not citations:
186 return False
187 return len(_live_sources(citations, store, config)) < min_sources
190def _check_stale_majority(
191 wiki_source: str,
192 store: Store,
193 config: Config,
194) -> bool:
195 """Return True if the stale citation fraction exceeds ``wiki_stale_citation_threshold``."""
196 issues = lint_wiki_page(wiki_source, store, config)
197 if not issues:
198 return False
199 citations = store.get_citations_for_wiki(wiki_source)
200 if not citations:
201 return False
202 stale_count = sum(1 for i in issues if i.issue_type in _STALE_TYPES)
203 return stale_count / len(citations) > config.wiki_stale_citation_threshold
206def _archive_and_record(
207 wiki_source: str,
208 wiki_root: Path,
209 store: Store,
210 config: Config,
211 reason: str,
212) -> PruneRecord:
213 """Archive a wiki page and return its PruneRecord."""
214 _archive_page(wiki_source, wiki_root, store, config)
215 return PruneRecord(wiki_source=wiki_source, action=PruneAction.ARCHIVED, reason=reason)
218def _evaluate_page(
219 wiki_source: str, wiki_root: Path, store: Store, config: Config
220) -> PruneRecord | None:
221 """Check a single wiki page against pruning rules. Returns a record or None."""
222 if _check_all_sources_deleted(wiki_source, store, config):
223 return _archive_and_record(
224 wiki_source, wiki_root, store, config, "all cited sources deleted"
225 )
226 if _check_cluster_below_threshold(wiki_source, store, config):
227 return _archive_and_record(
228 wiki_source,
229 wiki_root,
230 store,
231 config,
232 f"synthesis cluster below {MIN_CLUSTER_SOURCES} live sources",
233 )
234 if _check_stale_majority(wiki_source, store, config):
235 return PruneRecord(
236 wiki_source=wiki_source,
237 action=PruneAction.FLAGGED,
238 reason="majority of citations stale",
239 )
240 return None
243def _reconcile_orphan_rows(store: Store, wiki_root: Path, config: Config) -> list[PruneRecord]:
244 """Delete wiki rows whose page is no longer a file under a content subdir.
246 The page scan only ever revisits pages still on disk, so rows left behind by
247 an interrupted archive, a manual delete, or a migration would otherwise keep
248 serving retired content in search forever.
249 """
250 prefix = config.wiki_dir + "/"
251 records: list[PruneRecord] = []
252 for wiki_source in sorted(store.wiki_chunk_sources() | store.wiki_citation_sources()):
253 subdir = subdir_from_wiki_source(wiki_source, config.wiki_dir)
254 page_path = wiki_root / wiki_source.removeprefix(prefix)
255 if subdir in WIKI_CONTENT_SUBDIRS and page_path.is_file():
256 continue
257 if not _delete_wiki_rows(wiki_source, store):
258 continue
259 log.info("Reconciled orphaned wiki rows for %s (no page on disk)", wiki_source)
260 records.append(
261 PruneRecord(
262 wiki_source=wiki_source,
263 action=PruneAction.RECONCILED,
264 reason="indexed rows without a page on disk",
265 )
266 )
267 return records
270def _finalize_prune(report: PruneReport, wiki_root: Path, config: Config) -> None:
271 """Update wiki index and log after pruning.
273 A pass that reconciled rows for a wiki directory the user deleted writes
274 nothing back: index.md and log.md would recreate the tree it removed.
275 """
276 if not report.records:
277 return
278 log.info(
279 "Wiki prune: %d archived, %d flagged, %d reconciled",
280 report.archived_count,
281 report.flagged_count,
282 report.reconciled_count,
283 )
284 if not wiki_root.exists():
285 return
286 update_wiki_index(config)
287 for rec in report.records:
288 append_wiki_log(
289 WikiLogAction.PRUNE,
290 f"{rec.action.value} {rec.wiki_source}: {rec.reason}",
291 config,
292 )
295def prune_wiki(
296 store: Store,
297 config: Config | None = None,
298 *,
299 cancel: threading.Event | None = None,
300) -> PruneReport:
301 """Scan all wiki pages and prune stale/orphaned ones.
303 The page scan covers pages still on disk; reconciliation then covers rows
304 whose page is not, including the case of a wiki directory removed wholesale.
305 Archiving and the index rewrite make this a writer, so it holds the wiki
306 build mutex for the whole pass.
308 Each page costs a store lookup, so a large wiki takes long enough to want
309 stopping. Setting *cancel* ends the scan at the next page and still
310 reconciles and finalizes what it collected, leaving the wiki consistent.
311 """
312 if config is None:
313 config = cfg
314 wiki_root = config.data_root / config.wiki_dir
315 report = PruneReport()
316 with WIKI_BUILD_LOCK:
317 for subdir in WIKI_CONTENT_SUBDIRS:
318 subdir_path = wiki_root / subdir
319 if not subdir_path.is_dir():
320 continue
321 for md_path in sorted(subdir_path.rglob("*.md")):
322 if cancel is not None and cancel.is_set():
323 log.info("Wiki prune cancelled after %d pages", len(report.records))
324 break
325 relative = md_path.relative_to(wiki_root)
326 wiki_source = f"{config.wiki_dir}/{relative.as_posix()}"
327 record = _evaluate_page(wiki_source, wiki_root, store, config)
328 if record:
329 report.records.append(record)
330 report.records.extend(_reconcile_orphan_rows(store, wiki_root, config))
331 _finalize_prune(report, wiki_root, config)
332 return report