Coverage for src/lilbee/crawler/fetcher.py: 100%

1"""Protocol contract for web-fetcher backends consumed by the crawl runner.""" 

2 

3from __future__ import annotations 

4 

5from collections.abc import AsyncGenerator 

6from typing import Any, Protocol, runtime_checkable 

7 

8from lilbee.crawler.models import ( 

9 CancelToken, 

10 ConcurrencySpec, 

11 FetchedPage, 

12 FilterSpec, 

13) 

14 

15 

16@runtime_checkable 

17class WebFetcher(Protocol): 

18 """Backend contract for fetching web pages as markdown. 

19 

20 Implementations must honour ``CancelToken`` promptly inside 

21 ``fetch_recursive`` so the streaming loop in ``runner.py`` can 

22 abort without waiting for an in-flight batch to drain. 

23 

24 Lifecycle ordering: 

25 

26 1. ``__aenter__`` is called before any fetch method. Adapters with 

27 per-operation setup may no-op here. 

28 2. ``fetch_single`` and ``fetch_recursive`` may be called multiple 

29 times during the same context; they must not assume fresh state. 

30 3. ``fetch_recursive`` returns an async generator; callers are 

31 expected to ``.aclose()`` it deterministically on early break. 

32 4. ``__aexit__`` tears the backend down and must succeed even if 

33 a prior fetch raised. 

34 """ 

35 

36 async def __aenter__(self) -> WebFetcher: ... 

37 

38 async def __aexit__( 

39 self, 

40 exc_type: type[BaseException] | None, 

41 exc: BaseException | None, 

42 tb: Any, 

43 ) -> None: ... 

44 

45 async def fetch_single( 

46 self, url: str, *, timeout: float, concurrency: ConcurrencySpec 

47 ) -> FetchedPage: 

48 """Fetch one URL with the pacing and retry settings of *concurrency*.""" 

49 ... 

50 

51 def fetch_recursive( 

52 self, 

53 seed_url: str, 

54 *, 

55 depth: int | None, 

56 max_pages: int | None, 

57 timeout: float, 

58 concurrency: ConcurrencySpec, 

59 filters: FilterSpec, 

60 cancel: CancelToken | None = None, 

61 ) -> AsyncGenerator[FetchedPage, None]: 

62 """Stream pages discovered by BFS from ``seed_url``. 

63 

64 ``depth`` / ``max_pages``: positive int caps, or ``None`` for 

65 unbounded. 

66 

67 Returns an async generator so the orchestration layer can 

68 react per page (progress events, save-to-disk, cancel) and 

69 deterministically ``.aclose()`` the stream when it breaks 

70 out early (e.g. on ``max_pages`` hard cap). 

71 """ 

72 ...