Coverage for src/lilbee/crawler/backends/crawlberg/links.py: 100%

1"""The links a crawl follows or leaves out, as crawlberg queues them.""" 

2 

3from __future__ import annotations 

4 

5import logging 

6import re 

7from collections import Counter 

8from collections.abc import Sequence 

9from urllib.parse import parse_qsl, quote_plus, urlsplit, urlunsplit 

10 

11from lilbee.crawler import url_filter 

12from lilbee.crawler.backends.crawlberg.config import _TRACKING_PARAMS, _CrawlSpec 

13from lilbee.crawler.backends.crawlberg.events import _Event 

14 

15log = logging.getLogger(__name__) 

16 

17_PREFIX_MARK = "*" 

18# What crawlberg's query encoder leaves as is besides letters, digits and ``-._``, and the one 

19# character Python's encoder leaves as is that crawlberg's does not. 

20_FORM_SAFE = "*" 

21_TILDE = "~" 

22_TILDE_ENCODED = "%7E" 

23_LEFT_OUT = "Left out %s: it matches %s, a pattern of the crawl_exclude_patterns setting" 

24 

25 

26def _is_tracking_param(name: str) -> bool: 

27 """True when crawlberg strips the query parameter *name* from a discovered link.""" 

28 return any( 

29 name.startswith(param.removesuffix(_PREFIX_MARK)) 

30 if param.endswith(_PREFIX_MARK) 

31 else name == param 

32 for param in _TRACKING_PARAMS 

33 ) 

34 

35 

36def _form_encoded(text: str) -> str: 

37 """*text* as crawlberg writes it in a query: only letters, digits and ``*-._`` stay as is.""" 

38 return quote_plus(text, safe=_FORM_SAFE).replace(_TILDE, _TILDE_ENCODED) 

39 

40 

41def _queued_url(link: str) -> str: 

42 """*link* as crawlberg queues it: no fragment, no tracking parameter, a form-encoded query.""" 

43 parts = urlsplit(link) 

44 query = "&".join( 

45 f"{_form_encoded(name)}={_form_encoded(value)}" 

46 for name, value in parse_qsl(parts.query, keep_blank_values=True) 

47 if not _is_tracking_param(name) 

48 ) 

49 return urlunsplit(parts._replace(query=query, fragment="")) 

50 

51 

52def _python_patterns(patterns: Sequence[str]) -> tuple[re.Pattern[str], ...]: 

53 """The *patterns* Python compiles; crawlberg also accepts some that Python refuses. 

54 

55 The log names an excluded address through Python. A pattern only crawlberg compiles 

56 still excludes its addresses and is left out of the log. 

57 """ 

58 compiled = [] 

59 for pattern in patterns: 

60 try: 

61 compiled.append(re.compile(pattern)) 

62 except re.error: 

63 log.debug( 

64 "The exclude pattern %s is not a Python pattern; the log cannot name it", pattern 

65 ) 

66 return tuple(compiled) 

67 

68 

69class _ExcludedLinks: 

70 """The addresses one crawl leaves out because an exclude pattern matches them. 

71 

72 crawlberg counts the addresses it excludes and names none. This makes crawlberg's 

73 decision again for each link of a page, so the log can name the address and the pattern. 

74 """ 

75 

76 def __init__(self, spec: _CrawlSpec, seed_url: str) -> None: 

77 self._seed_host = urlsplit(seed_url).hostname or "" 

78 self._depth = spec.depth 

79 self._include_subdomains = spec.filters.include_subdomains 

80 self._patterns = _python_patterns(spec.filters.exclude_patterns) 

81 self._seen_links: set[str] = set() 

82 self._checked: set[str] = set() 

83 self._left_out: Counter[str] = Counter() 

84 

85 def record_seed(self, seed_url: str) -> bool: 

86 """Warn about the seed and return True when a pattern matches it. 

87 

88 crawlberg fetches a seed that an exclude pattern matches, so the crawl of a matched 

89 seed does not start. The seed is matched as crawlberg's URL parser writes it; a seed 

90 that does not parse is one crawlberg refuses, so no pattern applies to it. 

91 """ 

92 address = url_filter.parsed_address(seed_url) 

93 if address is None: 

94 return False 

95 url = _queued_url(address) 

96 self._checked.add(url) 

97 matched = self._matching(url) 

98 if matched is None: 

99 return False 

100 log.warning(_LEFT_OUT, url, matched) 

101 return True 

102 

103 def record(self, event: _Event) -> None: 

104 """Count each link of *event* that the crawl leaves out, the first time a page holds it. 

105 

106 An exclude pattern leaves out links as a matter of course, so each one is a debug line 

107 and ``summarize`` reports the total. 

108 """ 

109 if self._depth is not None and event.depth >= self._depth: 

110 return 

111 new_links = [link for link in event.links if link not in self._seen_links] 

112 self._seen_links.update(new_links) 

113 for url in map(_queued_url, new_links): 

114 if url not in self._checked and self._in_scope(url): 

115 self._checked.add(url) 

116 matched = self._matching(url) 

117 if matched is not None: 

118 self._left_out[matched] += 1 

119 log.debug(_LEFT_OUT, url, matched) 

120 

121 def summarize(self) -> None: 

122 """Log one line with the number of addresses each pattern left out, when any.""" 

123 total = self._left_out.total() 

124 if not total: 

125 return 

126 by_pattern = ", ".join(f"{pattern} {count}" for pattern, count in self._left_out.items()) 

127 log.info( 

128 "Crawl left out %d %s by crawl_exclude_patterns: %s", 

129 total, 

130 "address" if total == 1 else "addresses", 

131 by_pattern, 

132 ) 

133 

134 def _in_scope(self, url: str) -> bool: 

135 host = urlsplit(url).hostname or "" 

136 return url_filter.host_in_scope( 

137 host, self._seed_host, include_subdomains=self._include_subdomains 

138 ) 

139 

140 def _matching(self, url: str) -> str | None: 

141 """The first exclude pattern that matches *url*, or None.""" 

142 return next((p.pattern for p in self._patterns if p.search(url)), None)