Coverage for src/lilbee/wiki/shared.py: 100%

1"""Shared wiki utilities: frontmatter parsing, constants, page targets.""" 

2 

3from __future__ import annotations 

4 

5import os 

6import re 

7import tempfile 

8import threading 

9from collections.abc import Sequence 

10from dataclasses import dataclass 

11from enum import StrEnum 

12from pathlib import Path 

13from typing import Any 

14 

15import yaml 

16 

17MIN_CLUSTER_SOURCES = 3 # minimum unique sources for a synthesis page 

18 

19# Held by every mutating wiki entry point (build, synthesize, prune, draft 

20# accept, lint's log append, post-ingest update) while it writes. Pages, 

21# index.md and log.md are shared files that CLI, TUI, MCP and HTTP all reach 

22# inside one process, so the serialization lives with the writers rather than 

23# on any one surface. Re-entrant: a writer holding it calls helpers that take 

24# it again (a prune that lints, a lint that records a log entry). 

25WIKI_BUILD_LOCK = threading.RLock() 

26 

27 

28class WikiSubdir(StrEnum): 

29 """Filesystem subdirectory under ``$data_root/$wiki_dir/``.""" 

30 

31 SUMMARIES = "summaries" 

32 SYNTHESIS = "synthesis" 

33 CONCEPTS = "concepts" 

34 ENTITIES = "entities" 

35 DRAFTS = "drafts" 

36 ARCHIVE = "archive" 

37 

38 

39class WikiPageType(StrEnum): 

40 """Kind of wiki page. Values are used as frontmatter/API labels.""" 

41 

42 SUMMARY = "summary" 

43 SYNTHESIS = "synthesis" 

44 CONCEPT = "concept" 

45 ENTITY = "entity" 

46 DRAFT = "draft" 

47 ARCHIVE = "archive" 

48 

49 

50WIKI_CONTENT_SUBDIRS: tuple[WikiSubdir, ...] = ( 

51 WikiSubdir.SUMMARIES, 

52 WikiSubdir.SYNTHESIS, 

53 WikiSubdir.CONCEPTS, 

54 WikiSubdir.ENTITIES, 

55) 

56 

57 

58def count_pages_in(wiki_root: Path, subdirs: Sequence[WikiSubdir]) -> int: 

59 """Count ``.md`` pages under *subdirs* of *wiki_root*, at any depth.""" 

60 total = 0 

61 for subdir in subdirs: 

62 directory = wiki_root / subdir 

63 if directory.is_dir(): 

64 total += sum(1 for _ in directory.rglob("*.md")) 

65 return total 

66 

67 

68def total_wiki_pages(wiki_root: Path) -> int: 

69 """Count published ``.md`` pages across every wiki content subdir. 

70 

71 ``wiki build`` writes concepts/entities/synthesis pages, while summaries come 

72 from draft-accept, so counting only summaries (+ drafts) reports zero pages 

73 after a normal build even though searchable pages exist. 

74 """ 

75 return count_pages_in(wiki_root, WIKI_CONTENT_SUBDIRS) 

76 

77 

78WIKI_DISABLED_ERROR = "wiki not enabled" 

79 

80# Generic, path-free error for a draft slug that fails traversal validation. 

81# Shared across every transport (REST/CLI/MCP/TUI) so the absolute candidate 

82# path from validate_path_within is never echoed to a caller. 

83INVALID_DRAFT_SLUG_ERROR = "invalid draft slug" 

84 

85# PENDING-marker keyword phrases written into ``drafts/<slug>.md`` by the 

86# batched generator and matched by the drafts-review surface. Centralized 

87# here so the gen-side writer and the drafts-side reader agree on the 

88# exact wording. Changing a keyword here requires updating any cached 

89# markers on disk (one-shot find -delete or a regen). 

90PENDING_MARKER_KEYWORD_PARSE = "PENDING: batch parse failed" 

91PENDING_MARKER_KEYWORD_COLLISION = "PENDING: concept slug collision" 

92 

93# Marker lines tolerate whitespace variation, so cached markers written by an 

94# older build still classify the same way for every reader. 

95# Opening text of each marker, used by the writers. 

96PENDING_PARSE_MARKER_PREFIX = f"<!-- {PENDING_MARKER_KEYWORD_PARSE}" 

97PENDING_COLLISION_MARKER_PREFIX = f"<!-- {PENDING_MARKER_KEYWORD_COLLISION}" 

98 

99# Matched by every reader. Keyword spacing is loose because cached markers 

100# vary, and the match is anchored because a marker is always the first thing on 

101# its line: a body quoting one mid-line is content, not a placeholder. The tail 

102# is non-greedy rather than [^>]*, because a comment ends at "-->" and the 

103# writers interpolate raw source filenames, which may contain ">". 

104_PARSE_KEYWORD_PATTERN = PENDING_MARKER_KEYWORD_PARSE.replace(" ", r"\s+") 

105_COLLISION_KEYWORD_PATTERN = PENDING_MARKER_KEYWORD_COLLISION.replace(" ", r"\s+") 

106PENDING_PARSE_MARKER_RE = re.compile(rf"\s*<!--\s*{_PARSE_KEYWORD_PATTERN}.*?-->", re.IGNORECASE) 

107PENDING_COLLISION_MARKER_RE = re.compile( 

108 rf"\s*<!--\s*{_COLLISION_KEYWORD_PATTERN}.*?-->", re.IGNORECASE 

109) 

110 

111 

112def is_pending_marker_text(text: str) -> bool: 

113 """Whether *text* opens with a PENDING marker rather than review content. 

114 

115 One definition for every reader. The markers are always written as the 

116 first line, and the keyword spacing is matched loosely because cached 

117 markers vary, so a stricter prefix test would disagree with the drafts 

118 surface about the same file. 

119 """ 

120 first_line = text.splitlines()[0] if text else "" 

121 return any( 

122 pattern.match(first_line) 

123 for pattern in (PENDING_PARSE_MARKER_RE, PENDING_COLLISION_MARKER_RE) 

124 ) 

125 

126 

127class PendingKind(StrEnum): 

128 """Reason a wiki draft is in ``drafts/`` instead of a published page. 

129 

130 Derived from a draft's leading marker line and surfaced through 

131 ``DraftInfo.pending_kind`` to CLI / HTTP / MCP callers. StrEnum members 

132 serialise as their string value, so the JSON payload stays a plain 

133 string. ``DRIFT`` is display-only, never written to disk, but exposed so 

134 consumers don't hard-code ``"drift"``. 

135 """ 

136 

137 PARSE = "parse" 

138 COLLISION = "collision" 

139 DRIFT = "drift" 

140 

141 

142class WikiLogAction(StrEnum): 

143 """Verbs written into ``wiki/log.md`` audit-trail entries. 

144 

145 Distinct from WIKI_STATUS_* (which are result statuses returned to 

146 CLI/MCP/HTTP callers); these label internal audit-trail rows. 

147 """ 

148 

149 GENERATED = "generated" 

150 BUILD = "build" 

151 SYNTHESIZE = "synthesize" 

152 INGEST = "ingest" 

153 LINT = "lint" 

154 PRUNE = "prune" 

155 

156 

157SUBDIR_TO_TYPE: dict[str, WikiPageType] = { 

158 WikiSubdir.SUMMARIES.value: WikiPageType.SUMMARY, 

159 WikiSubdir.SYNTHESIS.value: WikiPageType.SYNTHESIS, 

160 WikiSubdir.CONCEPTS.value: WikiPageType.CONCEPT, 

161 WikiSubdir.ENTITIES.value: WikiPageType.ENTITY, 

162 WikiSubdir.DRAFTS.value: WikiPageType.DRAFT, 

163 WikiSubdir.ARCHIVE.value: WikiPageType.ARCHIVE, 

164} 

165 

166# One source of truth for sidebar-style headings keyed by page type. 

167# Consumed by ``wiki/index.py`` and the TUI sidebar via 

168# ``cli/tui/messages.WIKI_TYPE_HEADINGS``. 

169WIKI_TYPE_HEADINGS: dict[WikiPageType, str] = { 

170 WikiPageType.CONCEPT: "Concepts", 

171 WikiPageType.ENTITY: "Entities", 

172 WikiPageType.SUMMARY: "Source Summaries", 

173 WikiPageType.SYNTHESIS: "Synthesis", 

174} 

175 

176 

177@dataclass(frozen=True) 

178class PageTarget: 

179 """Grouping of page location fields for wiki generation.""" 

180 

181 wiki_root: Path 

182 subdir: str 

183 slug: str 

184 wiki_source: str 

185 page_type: str 

186 label: str 

187 # Whether this page replaces the documents it was written from. True for a 

188 # build, whose source is the one document the page summarizes. False when 

189 # the sources merely mention the subject: pruning them would delete every 

190 # document that named it. 

191 supersedes_sources: bool = True 

192 

193 

194def atomic_write_text(path: Path, text: str) -> None: 

195 """Write *text* to *path* via a temp file and ``os.replace``, creating parents. 

196 

197 A crash mid-write leaves the previous page intact rather than a truncated 

198 one. ``mkstemp`` creates the temp file owner-only and ``os.replace`` keeps 

199 that mode. 

200 """ 

201 path.parent.mkdir(parents=True, exist_ok=True) 

202 fd, tmp_name = tempfile.mkstemp(dir=path.parent, suffix=".tmp") 

203 try: 

204 with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as handle: 

205 handle.write(text) 

206 os.replace(tmp_name, path) 

207 except BaseException: 

208 Path(tmp_name).unlink(missing_ok=True) 

209 raise 

210 

211 

212def parse_frontmatter(text: str) -> dict[str, Any]: 

213 """Extract YAML frontmatter fields from a wiki page string. 

214 

215 A draft carries its marker comments above the frontmatter (drift, 

216 collision, origin), so the leading marker run is skipped before the 

217 opening delimiter is looked for. Without that every marked draft parses 

218 as having no frontmatter at all. The writers separate stacked markers with 

219 a blank line, so blank lines are consumed too once a marker has been seen, 

220 and never before one: a page with no marker still requires ``---`` on line 

221 zero. Uses line-by-line scanning so ``---`` inside YAML content is not 

222 mistaken for the closing delimiter. 

223 """ 

224 lines = text.splitlines() 

225 start = 0 

226 seen_marker = False 

227 while start < len(lines): 

228 stripped = lines[start].lstrip() 

229 is_marker = stripped.startswith("<!--") 

230 is_blank_after_marker = seen_marker and not stripped 

231 if not (is_marker or is_blank_after_marker): 

232 break 

233 seen_marker = seen_marker or is_marker 

234 start += 1 

235 if start >= len(lines) or lines[start].strip() != "---": 

236 return {} 

237 end_idx: int | None = None 

238 for i in range(start + 1, len(lines)): 

239 if lines[i].strip() == "---": 

240 end_idx = i 

241 break 

242 if end_idx is None: 

243 return {} 

244 block = "\n".join(lines[start + 1 : end_idx]) 

245 try: 

246 return yaml.safe_load(block) or {} 

247 except yaml.YAMLError: 

248 return {}