Coverage for src/lilbee/crawler/models.py: 100%

1"""Backend-agnostic value types crossing the runner/fetcher seam.""" 

2 

3from __future__ import annotations 

4 

5import threading 

6from dataclasses import dataclass, field 

7from typing import TypeAlias 

8 

9# Explicit "no page limit" for a crawl. Distinct from None, which means 

10# "unspecified, use the protective default cfg.crawl_safety_max_pages". 

11CRAWL_PAGES_UNLIMITED = 0 

12 

13 

14@dataclass 

15class CrawlResult: 

16 """Outcome of crawling a single URL. 

17 

18 This is the high-level result surfaced to lilbee callers 

19 (CLI, MCP, HTTP, TUI). The adapter produces ``FetchedPage`` 

20 and the orchestration layer converts it to ``CrawlResult`` 

21 when returning up to the caller. 

22 """ 

23 

24 url: str 

25 markdown: str = "" 

26 success: bool = True 

27 error: str | None = None 

28 

29 def failure_reason(self) -> str | None: 

30 """Why this page has nothing to save, or None when it does. 

31 

32 The save layer skips exactly these pages, so the event layer reports 

33 failures from the same predicate: the two cannot drift apart. 

34 """ 

35 if not self.success: 

36 return self.error or "fetch failed" 

37 if not self.markdown.strip(): 

38 return "page produced empty markdown" 

39 return None 

40 

41 

42@dataclass 

43class FetchedPage: 

44 """Single page produced by a ``WebFetcher`` backend. 

45 

46 Distinct from :class:`CrawlResult` so the adapter surface 

47 stays narrow and neutral: just the bytes we needed out of 

48 the underlying SDK's response object. 

49 """ 

50 

51 url: str 

52 markdown: str = "" 

53 success: bool = True 

54 error: str | None = None 

55 links: list[str] = field(default_factory=list) 

56 

57 

58@dataclass 

59class ConcurrencySpec: 

60 """Backend-agnostic concurrency + rate-limit knobs. 

61 

62 The crawl4ai adapter translates these into ``RateLimiter`` and 

63 ``SemaphoreDispatcher`` calls; a future adapter with its own 

64 BFS loop maps them onto ``asyncio.Semaphore`` + retry logic. 

65 """ 

66 

67 semaphore_count: int = 1 

68 mean_delay: float = 0.0 

69 max_delay_range: float = 0.0 

70 retry_on_rate_limit: bool = False 

71 retry_base_delay_min: float = 0.0 

72 retry_base_delay_max: float = 0.0 

73 retry_max_backoff: float = 0.0 

74 retry_max_attempts: int = 0 

75 

76 

77@dataclass 

78class FilterSpec: 

79 """Backend-agnostic filter settings applied to discovered links. 

80 

81 Pure Python data; each adapter decides how to plug the settings 

82 into its own filter pipeline. 

83 """ 

84 

85 exclude_patterns: list[str] = field(default_factory=list) 

86 include_subdomains: bool = False 

87 

88 

89CancelToken: TypeAlias = threading.Event 

90"""Cancellation handle the orchestration layer passes to a fetcher. 

91 

92An already-``set()`` event means "stop as soon as you can". The 

93crawl4ai adapter polls it in both its streaming loop and its BFS 

94strategy's ``should_cancel`` hook; a future adapter can poll it 

95in whatever granularity it supports. 

96"""