Coverage for src/lilbee/core/crawl_patterns.py: 100%

1"""Which crawl exclude patterns crawlberg refuses.""" 

2 

3from __future__ import annotations 

4 

5from collections.abc import Sequence 

6from typing import Any 

7 

8from lilbee.core.config.defaults import DEFAULT_CRAWL_EXCLUDE_PATTERNS, exclude_pattern_lines 

9 

10# The line of crawlberg's regex refusal that says what is wrong with the pattern. 

11_REGEX_ERROR_LINE = "error: " 

12 

13 

14def exclude_patterns_refusal(value: Any) -> str | None: 

15 """The message that names every pattern in *value* crawlberg refuses, or None. 

16 

17 A settings write and every crawl call this. Loading the config does not, because the check 

18 loads crawlberg. 

19 The shipped list is not checked: a test holds it to crawlberg's rules. 

20 """ 

21 patterns = exclude_pattern_lines(value) 

22 if not isinstance(patterns, list) or patterns == list(DEFAULT_CRAWL_EXCLUDE_PATTERNS): 

23 return None 

24 bad = [ 

25 f"[{index}] {patterns[index]!r}: {reason}" 

26 for index, reason in refused_exclude_patterns(patterns) 

27 ] 

28 return "invalid regex in crawl_exclude_patterns: " + "; ".join(bad) if bad else None 

29 

30 

31def refused_exclude_patterns(patterns: Sequence[str]) -> list[tuple[int, str]]: 

32 """Each of *patterns* that crawlberg refuses, as its index and crawlberg's reason. 

33 

34 crawlberg compiles an exclude pattern when it builds the engine, with its own regex rules 

35 and not Python's. The binding has no call that only compiles, so this builds an engine 

36 config for each pattern. Without the crawler extra nothing is refused here: no crawl can 

37 start, and the engine check runs when one does. 

38 """ 

39 try: 

40 import crawlberg 

41 except ImportError: 

42 return [] 

43 refused: list[tuple[int, str]] = [] 

44 for index, pattern in enumerate(patterns): 

45 try: 

46 crawlberg.create_engine(crawlberg.CrawlConfig(exclude_paths=[pattern])) 

47 except Exception as exc: 

48 refused.append((index, _compile_reason(str(exc)))) 

49 return refused 

50 

51 

52def _compile_reason(error: str) -> str: 

53 """The last ``error:`` line of crawlberg's regex refusal, or the whole text without one.""" 

54 for line in reversed(error.splitlines()): 

55 if line.startswith(_REGEX_ERROR_LINE): 

56 return line.removeprefix(_REGEX_ERROR_LINE) 

57 return error