Coverage for src/lilbee/crawler/__init__.py: 100%

1"""Web crawling: fetch pages as markdown and save them to the documents directory.""" 

2 

3from __future__ import annotations 

4 

5import os 

6 

7from lilbee.crawler.availability import CRAWLER_MISSING_MESSAGE, crawler_available 

8from lilbee.crawler.bootstrap import ( 

9 CrawlerBackendError, 

10 CrawlerBrowserError, 

11 bootstrap_chromium, 

12 chromium_installed, 

13 crawler_browsers_path, 

14) 

15from lilbee.crawler.fetcher import WebFetcher 

16from lilbee.crawler.models import ( 

17 CancelToken, 

18 ConcurrencySpec, 

19 CrawlLogin, 

20 CrawlResult, 

21 FetchedPage, 

22 FilterSpec, 

23) 

24from lilbee.crawler.runner import ( 

25 crawl_and_save, 

26 crawl_recursive, 

27 crawl_single, 

28) 

29from lilbee.crawler.save import ( 

30 content_hash, 

31 url_to_filename, 

32) 

33from lilbee.crawler.url_filter import ( 

34 get_blocked_networks, 

35 is_url, 

36 require_valid_crawl_url, 

37 validate_crawl_url, 

38) 

39 

40__all__ = [ 

41 "CRAWLER_MISSING_MESSAGE", 

42 "CancelToken", 

43 "ConcurrencySpec", 

44 "CrawlLogin", 

45 "CrawlResult", 

46 "CrawlerBackendError", 

47 "CrawlerBrowserError", 

48 "FetchedPage", 

49 "FilterSpec", 

50 "WebFetcher", 

51 "bootstrap_chromium", 

52 "chromium_installed", 

53 "content_hash", 

54 "crawl_and_save", 

55 "crawl_recursive", 

56 "crawl_single", 

57 "crawler_available", 

58 "crawler_browsers_path", 

59 "get_blocked_networks", 

60 "is_url", 

61 "require_valid_crawl_url", 

62 "url_to_filename", 

63 "validate_crawl_url", 

64] 

65 

66# Pin Playwright's browser cache so install and launch agree on Chromium's 

67# location, regardless of wheel vs frozen-binary layout. 

68os.environ.setdefault("PLAYWRIGHT_BROWSERS_PATH", str(crawler_browsers_path()))