Coverage for src/lilbee/crawler/backends/crawlberg/events.py: 100%

1"""The crawl events crawlberg streams, and lilbee's reading of them.""" 

2 

3from __future__ import annotations 

4 

5import json 

6import re 

7from collections.abc import AsyncGenerator 

8from contextlib import aclosing 

9from dataclasses import dataclass 

10from enum import StrEnum 

11from typing import TYPE_CHECKING, Any, cast 

12 

13from lilbee.crawler import url_filter 

14from lilbee.crawler.bootstrap import ( 

15 CrawlEngineRefusedError, 

16 CrawlerBrowserError, 

17 install_deps_command, 

18) 

19from lilbee.crawler.messages import ( 

20 BROWSER_FLAGS_REFUSED_MESSAGE, 

21 CHROMIUM_LIBRARY_MISSING_MESSAGE, 

22 CRAWL_REFUSED_MESSAGE, 

23 EXCLUDE_PATTERN_REFUSED_MESSAGE, 

24 ROBOTS_REFUSED_MESSAGE, 

25 ROBOTS_UNREACHABLE_MESSAGE, 

26) 

27from lilbee.crawler.models import CancelToken 

28 

29# crawlberg is the optional ``crawler`` extra and loads a native library, so each 

30# function that needs it imports it on call. 

31if TYPE_CHECKING: 

32 import crawlberg 

33 

34_SEED_DEPTH = 0 

35_SSRF_ERROR_CODE = "ssrf_policy_violation" 

36# How crawlberg 1.12.3 starts its refusal of a setting, and the message that names that setting. 

37_REFUSED_SETTING_MESSAGES = { 

38 "invalid_config: browser.chrome_args": BROWSER_FLAGS_REFUSED_MESSAGE, 

39 "invalid_config: invalid exclude_path regex": EXCLUDE_PATTERN_REFUSED_MESSAGE, 

40} 

41# How crawlberg 1.12.3 starts the error of a page the site's robots.txt disallows, and of a 

42# page it refuses because the site's robots.txt could not be read. 

43_ROBOTS_DISALLOWED_START = "robots.txt disallows " 

44_ROBOTS_UNREACHABLE_START = "robots_unreachable: " 

45# The glibc loader's message for a shared library the browser needs and the host lacks. 

46_MISSING_LIBRARY = re.compile(r"error while loading shared libraries: (?P<library>[^\s:]+):") 

47 

48 

49class _EventKind(StrEnum): 

50 """The ``type`` of a crawlberg crawl event lilbee reads; the end-of-crawl event is not one.""" 

51 

52 PAGE = "page" 

53 ERROR = "error" 

54 

55 

56_EVENT_KINDS = {kind.value: kind for kind in _EventKind} 

57 

58 

59@dataclass(frozen=True) 

60class _Event: 

61 """The fields lilbee reads from one crawlberg crawl event.""" 

62 

63 kind: _EventKind 

64 url: str 

65 depth: int = _SEED_DEPTH 

66 markdown: str = "" 

67 error: str = "" 

68 links: tuple[str, ...] = () 

69 title: str = "" 

70 

71 @property 

72 def refused_by_ssrf(self) -> bool: 

73 """True for an error event crawlberg raised because its SSRF policy refused the URL.""" 

74 return self.kind is _EventKind.ERROR and self.error.startswith(_SSRF_ERROR_CODE) 

75 

76 

77def _refusal_message(reason: str) -> str: 

78 """crawlberg's *reason* for a refusal, with lilbee's setting named when the text tells it.""" 

79 # crawlberg's error has no type or attribute for the field; only the start of its text names it. 

80 template = next( 

81 ( 

82 message 

83 for start, message in _REFUSED_SETTING_MESSAGES.items() 

84 if reason.startswith(start) 

85 ), 

86 CRAWL_REFUSED_MESSAGE, 

87 ) 

88 return template.format(reason=reason) 

89 

90 

91def _create_engine(config: crawlberg.CrawlConfig) -> crawlberg.CrawlEngineHandle: 

92 """The engine crawlberg builds for *config*. 

93 

94 An exception of any type from ``crawlberg.create_engine`` is a 

95 :class:`CrawlEngineRefusedError` that carries crawlberg's reason. No other call is guarded. 

96 """ 

97 import crawlberg 

98 

99 try: 

100 return crawlberg.create_engine(config) 

101 except Exception as exc: 

102 raise CrawlEngineRefusedError(_refusal_message(str(exc))) from exc 

103 

104 

105def _error_text(error: str) -> str: 

106 """crawlberg's *error* for a page, in lilbee's words when robots.txt refused the page. 

107 

108 A browser that cannot start for a missing system library names the library and the fix. 

109 

110 A robots.txt that could not be read because the SSRF policy refuses its address is 

111 that SSRF refusal. 

112 """ 

113 # The dynamic loader's own words, which Chromium's stderr carries into crawlberg's text. 

114 if missing := _MISSING_LIBRARY.search(error): 

115 try: 

116 command = install_deps_command() 

117 except CrawlerBrowserError: 

118 return error 

119 return CHROMIUM_LIBRARY_MISSING_MESSAGE.format( 

120 library=missing["library"], command=command, reason=error 

121 ) 

122 # crawlberg's error event has no code for a robots.txt refusal; only its text tells it. 

123 if error.startswith(_ROBOTS_DISALLOWED_START): 

124 return ROBOTS_REFUSED_MESSAGE 

125 if not error.startswith(_ROBOTS_UNREACHABLE_START): 

126 return error 

127 reason = error.removeprefix(_ROBOTS_UNREACHABLE_START) 

128 if reason.startswith(_SSRF_ERROR_CODE): 

129 return reason 

130 return ROBOTS_UNREACHABLE_MESSAGE.format(reason=reason) 

131 

132 

133def _parse_event(raw: object) -> _Event | None: 

134 """Read a crawlberg event, whose payload is only reachable as JSON; None for other kinds. 

135 

136 Every address of the event is read without its username and password. 

137 """ 

138 payload: dict[str, Any] = json.loads(str(raw)) 

139 kind = _EVENT_KINDS.get(payload["type"]) 

140 if kind is None: 

141 return None 

142 if kind is _EventKind.ERROR: 

143 url = url_filter.without_userinfo(payload["url"]) 

144 return _Event(kind, url, error=_error_text(payload["error"])) 

145 result: dict[str, Any] = payload["result"] 

146 markdown: dict[str, Any] = result.get("markdown") or {} 

147 metadata: dict[str, Any] = result.get("metadata") or {} 

148 return _Event( 

149 kind, 

150 url_filter.without_userinfo(result["url"]), 

151 depth=result.get("depth", _SEED_DEPTH), 

152 markdown=markdown.get("content") or "", 

153 links=tuple(url_filter.without_userinfo(link["url"]) for link in result.get("links") or ()), 

154 title=metadata.get("title") or "", 

155 ) 

156 

157 

158async def _events( 

159 config: crawlberg.CrawlConfig, 

160 seed_url: str, 

161 cancel: CancelToken | None, 

162) -> AsyncGenerator[_Event, None]: 

163 """Stream one crawl's page and error events until it completes or *cancel* is set. 

164 

165 The engine is created and dropped inside this generator, so it never leaves the 

166 thread whose event loop runs the crawl. 

167 """ 

168 import crawlberg 

169 

170 engine = _create_engine(config) 

171 # crawl_stream is an async generator; its stub types it as an AsyncIterator. 

172 stream = cast(AsyncGenerator[object, None], crawlberg.crawl_stream(engine, seed_url)) 

173 async with aclosing(stream): 

174 async for raw in stream: 

175 if cancel is not None and cancel.is_set(): 

176 return 

177 event = _parse_event(raw) 

178 if event is not None: 

179 yield event