Coverage for src/lilbee/wiki/shared.py: 100%

1"""Shared wiki utilities: frontmatter parsing, constants, page targets.""" 

2 

3from __future__ import annotations 

4 

5import logging 

6import os 

7import re 

8import tempfile 

9import threading 

10from collections.abc import Sequence 

11from dataclasses import dataclass 

12from enum import StrEnum 

13from pathlib import Path 

14from typing import Any 

15 

16import psutil 

17import yaml 

18 

19log = logging.getLogger(__name__) 

20 

21MIN_CLUSTER_SOURCES = 3 # minimum unique sources for a synthesis page 

22 

23# Held by every mutating wiki entry point (build, synthesize, prune, draft 

24# accept, lint's log append, post-ingest update) while it writes. Pages, 

25# index.md and log.md are shared files that CLI, TUI, MCP and HTTP all reach 

26# inside one process, so the serialization lives with the writers rather than 

27# on any one surface. Re-entrant: a writer holding it calls helpers that take 

28# it again (a prune that lints, a lint that records a log entry). 

29WIKI_BUILD_LOCK = threading.RLock() 

30 

31# A temp file of ``atomic_write_text``: this prefix, the writer's process id, a 

32# hyphen, what ``mkstemp`` adds, and the suffix. 

33_TEMP_PREFIX = ".lilbee-" 

34_TEMP_SUFFIX = ".tmp" 

35_TEMP_NAME_RE = re.compile( 

36 rf"{re.escape(_TEMP_PREFIX)}(?P<pid>[0-9]+)-[a-z0-9_]+{re.escape(_TEMP_SUFFIX)}" 

37) 

38 

39 

40class WikiSubdir(StrEnum): 

41 """Filesystem subdirectory under ``$data_root/$wiki_dir/``.""" 

42 

43 SUMMARIES = "summaries" 

44 SYNTHESIS = "synthesis" 

45 CONCEPTS = "concepts" 

46 ENTITIES = "entities" 

47 DRAFTS = "drafts" 

48 ARCHIVE = "archive" 

49 

50 

51class WikiPageType(StrEnum): 

52 """Kind of wiki page. Values are used as frontmatter/API labels.""" 

53 

54 SUMMARY = "summary" 

55 SYNTHESIS = "synthesis" 

56 CONCEPT = "concept" 

57 ENTITY = "entity" 

58 DRAFT = "draft" 

59 ARCHIVE = "archive" 

60 

61 

62WIKI_CONTENT_SUBDIRS: tuple[WikiSubdir, ...] = ( 

63 WikiSubdir.SUMMARIES, 

64 WikiSubdir.SYNTHESIS, 

65 WikiSubdir.CONCEPTS, 

66 WikiSubdir.ENTITIES, 

67) 

68 

69 

70def count_pages_in(wiki_root: Path, subdirs: Sequence[WikiSubdir]) -> int: 

71 """Count ``.md`` pages under *subdirs* of *wiki_root*, at any depth.""" 

72 total = 0 

73 for subdir in subdirs: 

74 directory = wiki_root / subdir 

75 if directory.is_dir(): 

76 total += sum(1 for _ in directory.rglob("*.md")) 

77 return total 

78 

79 

80def total_wiki_pages(wiki_root: Path) -> int: 

81 """Count published ``.md`` pages across every wiki content subdir. 

82 

83 ``wiki build`` writes concepts/entities/synthesis pages, while summaries come 

84 from draft-accept, so counting only summaries (+ drafts) reports zero pages 

85 after a normal build even though searchable pages exist. 

86 """ 

87 return count_pages_in(wiki_root, WIKI_CONTENT_SUBDIRS) 

88 

89 

90WIKI_DISABLED_ERROR = "wiki not enabled" 

91 

92# Generic, path-free error for a draft slug that fails traversal validation. 

93# Shared across every transport (REST/CLI/MCP/TUI) so the absolute candidate 

94# path from validate_path_within is never echoed to a caller. 

95INVALID_DRAFT_SLUG_ERROR = "invalid draft slug" 

96 

97# PENDING-marker keyword phrases written into ``drafts/<slug>.md`` by the 

98# batched generator and matched by the drafts-review surface. Centralized 

99# here so the gen-side writer and the drafts-side reader agree on the 

100# exact wording. Changing a keyword here requires updating any cached 

101# markers on disk (one-shot find -delete or a regen). 

102PENDING_MARKER_KEYWORD_PARSE = "PENDING: batch parse failed" 

103PENDING_MARKER_KEYWORD_COLLISION = "PENDING: concept slug collision" 

104 

105# Marker lines tolerate whitespace variation, so cached markers written by an 

106# older build still classify the same way for every reader. 

107# Opening text of each marker, used by the writers. 

108PENDING_PARSE_MARKER_PREFIX = f"<!-- {PENDING_MARKER_KEYWORD_PARSE}" 

109PENDING_COLLISION_MARKER_PREFIX = f"<!-- {PENDING_MARKER_KEYWORD_COLLISION}" 

110 

111# Matched by every reader. Keyword spacing is loose because cached markers 

112# vary, and the match is anchored because a marker is always the first thing on 

113# its line: a body quoting one mid-line is content, not a placeholder. The tail 

114# is non-greedy rather than [^>]*, because a comment ends at "-->" and the 

115# writers interpolate raw source filenames, which may contain ">". 

116_PARSE_KEYWORD_PATTERN = PENDING_MARKER_KEYWORD_PARSE.replace(" ", r"\s+") 

117_COLLISION_KEYWORD_PATTERN = PENDING_MARKER_KEYWORD_COLLISION.replace(" ", r"\s+") 

118PENDING_PARSE_MARKER_RE = re.compile(rf"\s*<!--\s*{_PARSE_KEYWORD_PATTERN}.*?-->", re.IGNORECASE) 

119PENDING_COLLISION_MARKER_RE = re.compile( 

120 rf"\s*<!--\s*{_COLLISION_KEYWORD_PATTERN}.*?-->", re.IGNORECASE 

121) 

122 

123 

124def is_pending_marker_text(text: str) -> bool: 

125 """Whether *text* opens with a PENDING marker rather than review content. 

126 

127 One definition for every reader. The markers are always written as the 

128 first line, and the keyword spacing is matched loosely because cached 

129 markers vary, so a stricter prefix test would disagree with the drafts 

130 surface about the same file. 

131 """ 

132 first_line = text.splitlines()[0] if text else "" 

133 return any( 

134 pattern.match(first_line) 

135 for pattern in (PENDING_PARSE_MARKER_RE, PENDING_COLLISION_MARKER_RE) 

136 ) 

137 

138 

139class PendingKind(StrEnum): 

140 """Reason a wiki draft is in ``drafts/`` instead of a published page. 

141 

142 Derived from a draft's leading marker line and surfaced through 

143 ``DraftInfo.pending_kind`` to CLI / HTTP / MCP callers. StrEnum members 

144 serialise as their string value, so the JSON payload stays a plain 

145 string. ``DRIFT`` is display-only, never written to disk, but exposed so 

146 consumers don't hard-code ``"drift"``. 

147 """ 

148 

149 PARSE = "parse" 

150 COLLISION = "collision" 

151 DRIFT = "drift" 

152 

153 

154class WikiLogAction(StrEnum): 

155 """Verbs written into ``wiki/log.md`` audit-trail entries. 

156 

157 Distinct from WIKI_STATUS_* (which are result statuses returned to 

158 CLI/MCP/HTTP callers); these label internal audit-trail rows. 

159 """ 

160 

161 GENERATED = "generated" 

162 BUILD = "build" 

163 SYNTHESIZE = "synthesize" 

164 INGEST = "ingest" 

165 LINT = "lint" 

166 PRUNE = "prune" 

167 

168 

169SUBDIR_TO_TYPE: dict[str, WikiPageType] = { 

170 WikiSubdir.SUMMARIES.value: WikiPageType.SUMMARY, 

171 WikiSubdir.SYNTHESIS.value: WikiPageType.SYNTHESIS, 

172 WikiSubdir.CONCEPTS.value: WikiPageType.CONCEPT, 

173 WikiSubdir.ENTITIES.value: WikiPageType.ENTITY, 

174 WikiSubdir.DRAFTS.value: WikiPageType.DRAFT, 

175 WikiSubdir.ARCHIVE.value: WikiPageType.ARCHIVE, 

176} 

177 

178# One source of truth for sidebar-style headings keyed by page type. 

179# Consumed by ``wiki/index.py`` and the TUI sidebar via 

180# ``cli/tui/messages.WIKI_TYPE_HEADINGS``. 

181WIKI_TYPE_HEADINGS: dict[WikiPageType, str] = { 

182 WikiPageType.CONCEPT: "Concepts", 

183 WikiPageType.ENTITY: "Entities", 

184 WikiPageType.SUMMARY: "Source Summaries", 

185 WikiPageType.SYNTHESIS: "Synthesis", 

186} 

187 

188 

189@dataclass(frozen=True) 

190class PageTarget: 

191 """Grouping of page location fields for wiki generation.""" 

192 

193 wiki_root: Path 

194 subdir: str 

195 slug: str 

196 wiki_source: str 

197 page_type: str 

198 label: str 

199 # Whether this page replaces the documents it was written from. True for a 

200 # build, whose source is the one document the page summarizes. False when 

201 # the sources merely mention the subject: pruning them would delete every 

202 # document that named it. 

203 supersedes_sources: bool = True 

204 

205 

206def atomic_write_text(path: Path, text: str) -> None: 

207 """Write *text* to *path* via a temp file and ``os.replace``, creating parents. 

208 

209 A crash mid-write leaves the previous page intact rather than a truncated 

210 one. ``mkstemp`` creates the temp file owner-only and ``os.replace`` keeps 

211 that mode. The temp file carries the writer's process id in its name. 

212 """ 

213 path.parent.mkdir(parents=True, exist_ok=True) 

214 prefix = f"{_TEMP_PREFIX}{os.getpid()}-" 

215 fd, tmp_name = tempfile.mkstemp(dir=path.parent, prefix=prefix, suffix=_TEMP_SUFFIX) 

216 try: 

217 with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as handle: 

218 handle.write(text) 

219 os.replace(tmp_name, path) 

220 except BaseException: 

221 Path(tmp_name).unlink(missing_ok=True) 

222 raise 

223 

224 

225def rewrite_keeping_time(path: Path, text: str) -> None: 

226 """Replace the text of the file *path* atomically; it keeps its access and modified times.""" 

227 written = path.stat() 

228 atomic_write_text(path, text) 

229 os.utime(path, ns=(written.st_atime_ns, written.st_mtime_ns)) 

230 

231 

232def _left_by_a_dead_writer(name: str) -> bool: 

233 """Whether *name* is a temp file of ``atomic_write_text`` whose process has ended.""" 

234 match = _TEMP_NAME_RE.fullmatch(name) 

235 return match is not None and not psutil.pid_exists(int(match["pid"])) 

236 

237 

238def remove_dead_temp_files(wiki_root: Path) -> None: 

239 """Delete each temp file under *wiki_root* that a wiki writer left when its process died.""" 

240 with WIKI_BUILD_LOCK: 

241 for directory, _subdirs, names in os.walk(wiki_root): 

242 for name in filter(_left_by_a_dead_writer, names): 

243 left = Path(directory, name) 

244 try: 

245 left.unlink(missing_ok=True) 

246 except OSError as exc: 

247 log.warning("Could not delete the leftover temp file %s: %s", left, exc) 

248 

249 

250def frontmatter_span(lines: list[str]) -> tuple[int, int] | None: 

251 """The line indexes of the frontmatter's opening and closing ``---``, or None without one. 

252 

253 A draft carries its marker comments above the frontmatter (drift, 

254 collision, origin), so the leading marker run is skipped before the 

255 opening delimiter is looked for. The writers separate stacked markers with 

256 a blank line, so blank lines are consumed too once a marker has been seen, 

257 and never before one: a page with no marker still requires ``---`` on line 

258 zero. Scans line by line so ``---`` inside YAML content is not mistaken for 

259 the closing delimiter. 

260 """ 

261 start = 0 

262 seen_marker = False 

263 while start < len(lines): 

264 stripped = lines[start].lstrip() 

265 is_marker = stripped.startswith("<!--") 

266 is_blank_after_marker = seen_marker and not stripped 

267 if not (is_marker or is_blank_after_marker): 

268 break 

269 seen_marker = seen_marker or is_marker 

270 start += 1 

271 if start >= len(lines) or lines[start].strip() != "---": 

272 return None 

273 for i in range(start + 1, len(lines)): 

274 if lines[i].strip() == "---": 

275 return start, i 

276 return None 

277 

278 

279def parse_frontmatter(text: str) -> dict[str, Any]: 

280 """Extract YAML frontmatter fields from a wiki page string; empty without frontmatter.""" 

281 lines = text.splitlines() 

282 span = frontmatter_span(lines) 

283 if span is None: 

284 return {} 

285 start, end_idx = span 

286 block = "\n".join(lines[start + 1 : end_idx]) 

287 try: 

288 return yaml.safe_load(block) or {} 

289 except yaml.YAMLError: 

290 return {}