Coverage for src/lilbee/crawler/bootstrap.py: 100%
161 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-08-14 11:46 +0000
1"""Playwright Chromium detection and install."""
3from __future__ import annotations
5import asyncio
6import contextlib
7import os
8import re
9import sys
10from pathlib import Path
12from filelock import FileLock
13from filelock import Timeout as FileLockTimeout
15from lilbee._frozen import is_frozen
16from lilbee.runtime.progress import (
17 DetailedProgressCallback,
18 EventType,
19 SetupDoneEvent,
20 SetupProgressEvent,
21 SetupStartEvent,
22)
25class CrawlerBrowserError(RuntimeError):
26 """Playwright is installed but its Chromium browser binary is not."""
29class CrawlerBackendError(RuntimeError):
30 """The ``crawler`` extra (crawl4ai) was never installed."""
33_CHROMIUM_COMPONENT = "chromium"
35# A Chromium unpack runs into the minutes on a slow link, so a caller queued
36# behind one waits well past any normal request timeout before giving up.
37_BOOTSTRAP_LOCK_TIMEOUT_S = 900.0
38# Rough size estimate for the Chromium download; Playwright bundles vary
39# slightly per platform but this gives the UI a decent denominator before
40# 'Total bytes' parses out of stdout.
41_CHROMIUM_ESTIMATE_MB = 180
42_CHROMIUM_SIZE_ESTIMATE_BYTES = _CHROMIUM_ESTIMATE_MB * 1024 * 1024
44# Unit -> bytes scale for Playwright stdout progress lines.
45_BYTE_UNIT_SCALE: dict[str, int] = {
46 "b": 1,
47 "kb": 1024,
48 "kib": 1024,
49 "mb": 1024 * 1024,
50 "mib": 1024 * 1024,
51}
53# Playwright 1.58 prints lines like
54# ``|■■■■■■■■ | 10% of 162.3 MiB`` during
55# the chromium download. The percent comes first, then "of <total> <unit>".
56_PROGRESS_LINE_RE = re.compile(
57 r"(\d+)\s*%\s*of\s*(\d+(?:\.\d+)?)\s*(MiB|Mb|MB|KiB|KB|B)",
58 re.IGNORECASE,
59)
62def _browsers_cache_path() -> Path:
63 """Return the root path where Playwright stores browser binaries."""
64 override = os.environ.get("PLAYWRIGHT_BROWSERS_PATH")
65 if override:
66 return Path(override).expanduser()
67 if sys.platform == "darwin":
68 return Path.home() / "Library" / "Caches" / "ms-playwright"
69 if sys.platform == "win32":
70 local = os.environ.get("LOCALAPPDATA", str(Path.home() / "AppData" / "Local"))
71 return Path(local) / "ms-playwright"
72 return Path.home() / ".cache" / "ms-playwright"
75def _read_chromium_revision(browsers_json: Path) -> str | None:
76 """Pull the ``chromium`` revision out of a Playwright ``browsers.json``."""
77 import json
79 try:
80 data = json.loads(browsers_json.read_text(encoding="utf-8"))
81 except (OSError, ValueError):
82 return None
83 for browser in data.get("browsers", []):
84 if browser.get("name") == _CHROMIUM_COMPONENT:
85 revision = browser.get("revision")
86 return str(revision) if revision is not None else None
87 return None
90def _expected_chromium_revision() -> str | None:
91 """Revision string Playwright was built against (e.g. ``"1217"``).
93 ``None`` means "unknown" -- treat as "do install" so a missing
94 browsers.json never short-circuits the bootstrap path.
95 """
96 try:
97 import playwright as _pw
98 except ImportError:
99 return None
100 for path in Path(_pw.__file__).parent.rglob("browsers.json"):
101 return _read_chromium_revision(path)
102 return None
105def chromium_installed() -> bool:
106 """Return True if the Chromium revision Playwright expects is on disk.
108 Matching by any ``chromium-*`` directory isn't enough: when the
109 system has chromium-1217 but the bundled Playwright driver expects
110 chromium-1208, launch fails with ``Executable doesn't exist`` even
111 though the bootstrap check thought everything was ready.
112 """
113 root = _browsers_cache_path()
114 if not root.exists():
115 return False
116 expected = _expected_chromium_revision()
117 if expected is None:
118 return any(p.is_dir() and p.name.startswith("chromium-") for p in root.iterdir())
119 return (root / f"{_CHROMIUM_COMPONENT}-{expected}").is_dir()
122def _bootstrap_lock_path() -> Path:
123 """Lock file serializing Chromium installs into the browsers cache."""
124 root = _browsers_cache_path()
125 root.mkdir(parents=True, exist_ok=True)
126 return root / ".lilbee-bootstrap.lock"
129def crawler_browsers_path() -> Path:
130 """Public accessor for the crawler browser cache root.
132 Used by the HTTP status endpoint to tell plugins where Chromium
133 lives. The underlying resolver stays private because callers should
134 not depend on the Playwright-specific directory layout.
135 """
136 return _browsers_cache_path()
139def _bytes_from_stdout(line: str) -> tuple[int, int] | None:
140 """Extract (downloaded_bytes, total_bytes) from a Playwright stdout line.
142 Matches the ``NN% of N.N MiB`` shape Playwright 1.58+ emits for the
143 Chromium download. Returns None when the line doesn't match. The
144 percent and total both parse out of the same line so callers never
145 have to handle a missing total.
146 """
147 match = _PROGRESS_LINE_RE.search(line)
148 if match is None:
149 return None
150 pct = int(match.group(1))
151 raw_total = float(match.group(2))
152 unit = match.group(3).lower()
153 scale = _BYTE_UNIT_SCALE.get(unit, 1)
154 total = int(raw_total * scale)
155 downloaded = int(total * pct / 100)
156 return downloaded, total
159def _emit_setup_start(on_progress: DetailedProgressCallback | None) -> None:
160 if on_progress is None:
161 return
162 on_progress(
163 EventType.SETUP_START,
164 SetupStartEvent(
165 component=_CHROMIUM_COMPONENT,
166 size_estimate_bytes=_CHROMIUM_SIZE_ESTIMATE_BYTES,
167 ),
168 )
171def _emit_setup_done(
172 on_progress: DetailedProgressCallback | None,
173 *,
174 success: bool,
175 error: str | None,
176) -> None:
177 if on_progress is None:
178 return
179 on_progress(
180 EventType.SETUP_DONE,
181 SetupDoneEvent(component=_CHROMIUM_COMPONENT, success=success, error=error),
182 )
185async def _drain_stdout_to_progress(
186 stream: asyncio.StreamReader,
187 on_progress: DetailedProgressCallback | None,
188) -> None:
189 while True:
190 line_bytes = await stream.readline()
191 if not line_bytes:
192 return
193 line = line_bytes.decode(errors="replace").rstrip()
194 parsed = _bytes_from_stdout(line)
195 if parsed is None or on_progress is None:
196 continue
197 downloaded, total = parsed
198 on_progress(
199 EventType.SETUP_PROGRESS,
200 SetupProgressEvent(
201 component=_CHROMIUM_COMPONENT,
202 downloaded_bytes=downloaded,
203 total_bytes=total,
204 detail=line,
205 ),
206 )
209async def _drain_stderr(stream: asyncio.StreamReader, tail: list[str]) -> None:
210 while True:
211 line_bytes = await stream.readline()
212 if not line_bytes:
213 return
214 tail.append(line_bytes.decode(errors="replace").rstrip())
217_PLAYWRIGHT_MISSING_HINT = (
218 "Chromium bootstrap requires the playwright Python package, which is "
219 "bundled with the release binary and the lilbee[crawler] extra. "
220 "Reinstall with 'pip install lilbee[crawler]' or download a fresh "
221 "release binary."
222)
225def _resolve_playwright_runner() -> tuple[list[str], dict[str, str]]:
226 """Return ``(argv_prefix, env)`` for invoking ``playwright install chromium``.
228 Spawns Playwright's bundled Node driver directly so the call works under a
229 pip install, ``uv tool install``, or a frozen (Nuitka onefile) binary. Falls
230 back to ``[sys.executable, '-m', 'playwright']`` for unfrozen builds when the
231 driver lookup fails; re-raises for frozen builds, where ``sys.executable``
232 is the lilbee exe and ``-m playwright`` would leak into typer.
233 """
234 try:
235 from playwright._impl._driver import compute_driver_executable, get_driver_env
236 except ImportError as exc:
237 raise CrawlerBrowserError(_PLAYWRIGHT_MISSING_HINT) from exc
238 try:
239 driver_exe, driver_cli = compute_driver_executable()
240 except Exception:
241 if not is_frozen():
242 return [sys.executable, "-m", "playwright"], dict(os.environ)
243 raise
244 return [str(driver_exe), str(driver_cli)], dict(get_driver_env())
247async def bootstrap_chromium(
248 on_progress: DetailedProgressCallback | None = None,
249) -> None:
250 """Run ``playwright install chromium`` as a subprocess, emitting events.
252 Short-circuits when ``chromium_installed()`` is already True. Emits
253 ``setup_start`` before spawning, ``setup_progress`` for each recognizable
254 progress line on stdout, and ``setup_done`` on exit (``success=False`` plus
255 the subprocess stderr tail on failure). Raises :class:`CrawlerBrowserError`
256 with the tail so task workers route to FAILED cleanly.
257 """
258 if chromium_installed():
259 _emit_setup_done(on_progress, success=True, error=None)
260 return
262 # Two callers (a second POST /setup/crawler, or a crawl bootstrapping on
263 # first use) would unpack a browser bundle into the same directory and
264 # corrupt it. On disk, not in memory, because the CLI and MCP are separate
265 # processes sharing this path. thread_local=False because the acquire runs
266 # in a worker thread and the release on the loop thread; with the default
267 # the release would not count and the lock would never be freed.
268 lock = FileLock(str(_bootstrap_lock_path()), thread_local=False)
269 try:
270 await asyncio.to_thread(lock.acquire, timeout=_BOOTSTRAP_LOCK_TIMEOUT_S)
271 except FileLockTimeout as exc:
272 message = (
273 "Timed out waiting for another Chromium install to finish. "
274 "If no other lilbee process is installing, retry."
275 )
276 _emit_setup_done(on_progress, success=False, error=message)
277 raise CrawlerBrowserError(message) from exc
278 try:
279 # The install we were queued behind may have been the one we needed.
280 if chromium_installed():
281 _emit_setup_done(on_progress, success=True, error=None)
282 return
283 await _install_chromium(on_progress)
284 finally:
285 lock.release()
288async def _install_chromium(on_progress: DetailedProgressCallback | None) -> None:
289 """Run the install subprocess. Caller holds the bootstrap lock."""
290 _emit_setup_start(on_progress)
292 try:
293 runner, runner_env = _resolve_playwright_runner()
294 except CrawlerBrowserError as exc:
295 _emit_setup_done(on_progress, success=False, error=str(exc))
296 raise
298 proc = await asyncio.create_subprocess_exec(
299 *runner,
300 "install",
301 "chromium",
302 stdout=asyncio.subprocess.PIPE,
303 stderr=asyncio.subprocess.PIPE,
304 env=runner_env,
305 )
306 # mypy narrowing: asyncio.create_subprocess_exec with PIPE guarantees
307 # non-None streams at runtime; the asserts only satisfy the type checker.
308 assert proc.stdout is not None # noqa: S101
309 assert proc.stderr is not None # noqa: S101
311 stderr_tail: list[str] = []
312 try:
313 await asyncio.gather(
314 _drain_stdout_to_progress(proc.stdout, on_progress),
315 _drain_stderr(proc.stderr, stderr_tail),
316 )
317 returncode = await proc.wait()
318 finally:
319 # On cancel (SSE client disconnect) or any error, don't leave the
320 # ~180MB chromium install running orphaned; terminate, then kill.
321 await _terminate_process(proc)
323 if returncode != 0:
324 tail = "\n".join(stderr_tail[-10:]) or f"exit code {returncode}"
325 _emit_setup_done(on_progress, success=False, error=tail)
326 raise CrawlerBrowserError(f"Chromium bootstrap failed (exit {returncode}): {tail}")
328 _emit_setup_done(on_progress, success=True, error=None)
331_TERMINATE_TIMEOUT_S = 5.0
334async def _terminate_process(proc: asyncio.subprocess.Process) -> None:
335 """Terminate then kill *proc* if it is still running, and reap it.
337 Best-effort and re-cancel-safe: the waits are shielded so a second
338 cancellation while unwinding cannot leave the child orphaned.
339 """
340 if proc.returncode is not None:
341 return
342 with contextlib.suppress(ProcessLookupError):
343 proc.terminate()
344 with contextlib.suppress(TimeoutError, asyncio.CancelledError):
345 await asyncio.wait_for(asyncio.shield(proc.wait()), timeout=_TERMINATE_TIMEOUT_S)
346 if proc.returncode is None:
347 with contextlib.suppress(ProcessLookupError):
348 proc.kill()
349 # SIGKILL is uncatchable so the reap should be near-instant; cap it anyway
350 # so a wedged reap can't hang the cleanup path indefinitely.
351 with contextlib.suppress(TimeoutError, asyncio.CancelledError):
352 await asyncio.wait_for(asyncio.shield(proc.wait()), timeout=_TERMINATE_TIMEOUT_S)