Coverage for src/lilbee/crawler/backends/crawlberg/config.py: 100%
100 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-11-06 08:17 +0000
1"""The crawlberg configuration lilbee builds for one crawl."""
3from __future__ import annotations
5import ipaddress
6import logging
7from collections.abc import Sequence
8from dataclasses import dataclass, field
9from typing import TYPE_CHECKING, TypeVar
11from lilbee.core.config.enums import CrawlRenderMode
12from lilbee.crawler import bootstrap, url_filter
13from lilbee.crawler.bootstrap import ChromiumMissingError
14from lilbee.crawler.messages import CHROMIUM_MISSING_MESSAGE
15from lilbee.crawler.models import (
16 AUTHORIZATION_HEADER,
17 ConcurrencySpec,
18 CrawlLogin,
19 FilterSpec,
20)
22# crawlberg is the optional ``crawler`` extra and loads a native library, so each
23# function that needs it imports it on call.
24if TYPE_CHECKING:
25 import crawlberg
27log = logging.getLogger(__name__)
29_Network = TypeVar("_Network", ipaddress.IPv4Network, ipaddress.IPv6Network)
30# The networks crawlberg's ``deny_private`` refuses (crawlberg 1.12.3, DEFAULT_DENY_NET_CIDRS).
31CRAWLBERG_DENIED_NETWORKS: tuple[ipaddress.IPv4Network | ipaddress.IPv6Network, ...] = tuple(
32 ipaddress.ip_network(cidr)
33 for cidr in (
34 "127.0.0.0/8",
35 "10.0.0.0/8",
36 "172.16.0.0/12",
37 "192.168.0.0/16",
38 "169.254.0.0/16",
39 "0.0.0.0/8",
40 "224.0.0.0/4",
41 "240.0.0.0/4",
42 "100.64.0.0/10",
43 "::1/128",
44 "::/128",
45 "fe80::/10",
46 "fc00::/7",
47 "ff00::/8",
48 )
49)
50_MAX_REDIRECTS = 10
51# Tracking query parameters stripped from a discovered link before it is queued, so the page
52# is fetched once at its plain address. A trailing ``*`` matches by prefix. ``ref`` is not one:
53# it names content on docs and forum sites, such as a branch, though crawlberg's default strips it.
54_TRACKING_PARAMS = (
55 "utm_*",
56 "fbclid",
57 "gclid",
58 "msclkid",
59 "yclid",
60 "mc_cid",
61 "mc_eid",
62 "_hsenc",
63 "_hsmi",
64 "hsCtaTracking",
65 "mkt_*",
66 "trk",
67 "trkInfo",
68 "dm_i",
69 "vero_id",
70 "vero_conv",
71 "oly_anon_id",
72 "oly_enc_id",
73 "igshid",
74 "pk_*",
75 "_ga",
76 "affiliate",
77 "aff_id",
78 "aff_ref",
79 "aff",
80 "partner",
81 "srsltid",
82 "replytocom",
83)
84# A browser fetch as a whole (launch, navigation, render, close) may take this many page timeouts.
85_BROWSER_OVERALL_TIMEOUTS = 2
86_BROWSER_SHUTDOWN_TIMEOUT_MS = 5000
87_RATE_LIMIT_STATUSES = (429, 503)
88# The most retries crawlberg 1.12.3 accepts; it refuses a config with more.
89_MAX_RETRIES = 20
90_MS_PER_SECOND = 1000
91_BROWSER_MODES = {CrawlRenderMode.HTTP: "never", CrawlRenderMode.BROWSER: "always"}
92# The headers a browser sends with a page request, which crawlberg's HTTP client does not send.
93# Browser mode sends none: crawlberg adds custom headers to Chrome's requests too.
94_BROWSER_ACCEPT_HEADERS = {
95 "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
96 "Accept-Language": "en-US,en;q=0.5",
97 "Upgrade-Insecure-Requests": "1",
98}
99_CUSTOM_HEADERS = {CrawlRenderMode.HTTP: _BROWSER_ACCEPT_HEADERS, CrawlRenderMode.BROWSER: {}}
102@dataclass(frozen=True)
103class _CrawlSpec:
104 """The per-call inputs of one crawl."""
106 depth: int | None
107 max_pages: int | None
108 timeout: float
109 concurrency: ConcurrencySpec = field(default_factory=ConcurrencySpec)
110 filters: FilterSpec = field(default_factory=FilterSpec)
113def _subtract(network: _Network, removed: _Network) -> list[_Network]:
114 """The parts of *network* outside *removed*."""
115 if not network.overlaps(removed):
116 return [network]
117 if network.subnet_of(removed):
118 return []
119 return list(network.address_exclude(removed))
122def _subtract_any(
123 network: ipaddress.IPv4Network | ipaddress.IPv6Network,
124 removed: ipaddress.IPv4Network | ipaddress.IPv6Network,
125) -> list[ipaddress.IPv4Network | ipaddress.IPv6Network]:
126 """:func:`_subtract` across address families; different families never overlap."""
127 # isinstance narrows the union so each call gets one address family.
128 if isinstance(network, ipaddress.IPv4Network) and isinstance(removed, ipaddress.IPv4Network):
129 return list(_subtract(network, removed))
130 if isinstance(network, ipaddress.IPv6Network) and isinstance(removed, ipaddress.IPv6Network):
131 return list(_subtract(network, removed))
132 return [network]
135def admitted_networks(
136 blocked: tuple[ipaddress.IPv4Network | ipaddress.IPv6Network, ...],
137) -> list[ipaddress.IPv4Network | ipaddress.IPv6Network]:
138 """The networks crawlberg refuses by default that *blocked* does not block."""
139 admitted: list[ipaddress.IPv4Network | ipaddress.IPv6Network] = []
140 for denied in CRAWLBERG_DENIED_NETWORKS:
141 pieces = [denied]
142 for network in blocked:
143 pieces = [part for piece in pieces for part in _subtract_any(piece, network)]
144 admitted.extend(pieces)
145 return admitted
148def _ssrf_policy() -> crawlberg.SsrfPolicy:
149 """crawlberg's SSRF policy, read from lilbee's blocklist at crawl time."""
150 import crawlberg
152 allowlist = [
153 crawlberg.HostMatcher.cidr(str(network))
154 for network in admitted_networks(url_filter.get_blocked_networks())
155 ]
156 return crawlberg.SsrfPolicy(
157 deny_private=True, allowlist=allowlist, max_redirects=_MAX_REDIRECTS
158 )
161def _content_config() -> crawlberg.ContentConfig:
162 """Markdown output with no frontmatter that keeps navigation, forms and all page text.
164 All page text includes the text of hidden elements.
165 """
166 import crawlberg
168 return crawlberg.ContentConfig(
169 remove_navigation=False,
170 remove_forms=False,
171 exclude_selectors=[],
172 preprocessing_preset="minimal",
173 extract_metadata=False,
174 keep_hidden=True,
175 )
178def _ms(seconds: float) -> int:
179 """*seconds* in whole milliseconds."""
180 return round(seconds * _MS_PER_SECOND)
183@dataclass(frozen=True)
184class _Pacing:
185 """crawlberg's per-domain delay and jitter ratio for waits in ``[mean, mean + range]``.
187 crawlberg scales the delay by a factor in ``[1 - ratio, 1 + ratio]``, so the delay is
188 the middle of that span and the ratio is half the range over it.
189 Each concurrent request waits that delay, so crawlberg's one gap per domain is the
190 delay over the request count.
191 """
193 rate_limit_ms: int
194 jitter_ratio: float
196 @classmethod
197 def of(cls, spec: ConcurrencySpec) -> _Pacing:
198 half_range = spec.max_delay_range / 2
199 middle = spec.mean_delay + half_range
200 ratio = half_range / middle if middle > 0 else 0.0
201 return cls(_ms(middle / spec.semaphore_count), ratio)
204def _retry_count(spec: ConcurrencySpec) -> int:
205 """The retries crawlberg makes for one page: the configured count, up to crawlberg's limit."""
206 if not spec.retry_on_rate_limit:
207 return 0
208 if spec.retry_max_attempts > _MAX_RETRIES:
209 log.warning(
210 "crawl_retry_max_attempts is %d; a crawl retries a page at most %d times",
211 spec.retry_max_attempts,
212 _MAX_RETRIES,
213 )
214 return min(spec.retry_max_attempts, _MAX_RETRIES)
217def _headless_shell() -> str:
218 """The path of Playwright's headless shell, the only Chrome a crawl launches."""
219 executable = bootstrap.headless_shell_executable()
220 if executable is None:
221 raise ChromiumMissingError(CHROMIUM_MISSING_MESSAGE)
222 return str(executable)
225def _browser_config(
226 render_mode: CrawlRenderMode, timeout_ms: int, chrome_args: Sequence[str], extra_wait_ms: int
227) -> crawlberg.BrowserConfig:
228 """The browser settings, with a deadline on each whole browser fetch.
230 Only browser mode names a Chrome binary, launch flags and the extra wait after a page loads.
231 The deadline adds the extra wait, which crawlberg counts inside it.
232 """
233 import crawlberg
235 browser = render_mode is CrawlRenderMode.BROWSER
236 extra_wait = extra_wait_ms if browser else None
237 return crawlberg.BrowserConfig(
238 mode=_BROWSER_MODES[render_mode],
239 timeout=timeout_ms,
240 overall_timeout=timeout_ms * _BROWSER_OVERALL_TIMEOUTS + (extra_wait or 0),
241 shutdown_timeout=_BROWSER_SHUTDOWN_TIMEOUT_MS,
242 chrome_path=_headless_shell() if browser else None,
243 chrome_args=list(chrome_args) if browser else [],
244 extra_wait=extra_wait,
245 )
248def _crawl_config(
249 render_mode: CrawlRenderMode,
250 spec: _CrawlSpec,
251 chrome_args: Sequence[str],
252 user_agent: str,
253 respect_robots_txt: bool,
254 extra_wait_ms: int,
255 login: CrawlLogin,
256) -> crawlberg.CrawlConfig:
257 """The crawlberg config for one crawl, with every default lilbee depends on set.
259 A blank *user_agent* leaves the user agent to crawlberg: Chrome's own in browser mode.
260 Browser mode accepts an invalid TLS certificate, as Chrome did before crawlberg 1.12 checked
261 it; HTTP mode refuses one.
262 crawlberg sends the *login* only to the seed host.
263 """
264 import crawlberg
266 concurrency = spec.concurrency
267 retries = concurrency.retry_on_rate_limit
268 pacing = _Pacing.of(concurrency)
269 timeout_ms = _ms(spec.timeout)
270 return crawlberg.CrawlConfig(
271 max_depth=spec.depth,
272 max_pages=spec.max_pages,
273 max_concurrent=concurrency.semaphore_count,
274 stay_on_domain=True,
275 allow_subdomains=spec.filters.include_subdomains,
276 exclude_paths=list(spec.filters.exclude_patterns),
277 path_patterns_match_url=True,
278 dedup_include_query=True,
279 strip_tracking_params=True,
280 tracking_params=list(_TRACKING_PARAMS),
281 request_timeout=timeout_ms,
282 user_agent=user_agent or None,
283 custom_headers={**_CUSTOM_HEADERS[render_mode], **login.cookie_header},
284 # crawlberg keeps custom headers on the seed host unless named here. The login
285 # cookie stays off this list, so it goes to the seed host only.
286 subdomain_header_allowlist=list(_CUSTOM_HEADERS[render_mode]),
287 auth=_auth(login),
288 rate_limit_ms=pacing.rate_limit_ms,
289 rate_limit_jitter_ratio=pacing.jitter_ratio,
290 max_redirects=_MAX_REDIRECTS,
291 retry_count=_retry_count(concurrency),
292 retry_codes=list(_RATE_LIMIT_STATUSES) if retries else [],
293 retry_initial_delay_ms=_ms(
294 (concurrency.retry_base_delay_min + concurrency.retry_base_delay_max) / 2
295 ),
296 retry_max_delay_ms=_ms(concurrency.retry_max_backoff),
297 respect_robots_txt=respect_robots_txt,
298 danger_accept_invalid_certs=render_mode is CrawlRenderMode.BROWSER,
299 cookies_enabled=True,
300 soft_http_errors=False,
301 download_documents=False,
302 content=_content_config(),
303 browser=_browser_config(render_mode, timeout_ms, chrome_args, extra_wait_ms),
304 ssrf=_ssrf_policy(),
305 )
308def _auth(login: CrawlLogin) -> crawlberg.AuthConfig | None:
309 """crawlberg's form of the basic credentials or authorization header of *login*."""
310 import crawlberg
312 if login.basic_credentials is not None:
313 return crawlberg.AuthConfig.basic(*login.basic_credentials)
314 if login.authorization:
315 return crawlberg.AuthConfig.header(AUTHORIZATION_HEADER, login.authorization)
316 return None