Coverage for src/lilbee/crawler/backends/crawlberg/__init__.py: 100%

1"""The crawlberg backend: builds the fetcher a crawl runs on.""" 

2 

3from __future__ import annotations 

4 

5from lilbee.core.config import cfg 

6from lilbee.core.config.enums import CrawlRenderMode 

7from lilbee.crawler.backends.crawlberg.fetcher import CrawlbergFetcher 

8from lilbee.crawler.fetcher import WebFetcher 

9from lilbee.crawler.models import CrawlLogin 

10 

11__all__ = ["fetcher_from_config"] 

12 

13 

14def fetcher_from_config(render_mode: CrawlRenderMode, login: CrawlLogin) -> WebFetcher: 

15 """The fetcher for one crawl, with its browser, user agent, robots.txt and login settings.""" 

16 return CrawlbergFetcher( 

17 render_mode=render_mode, 

18 chrome_args=cfg.crawl_browser_extra_args, 

19 user_agent=cfg.crawl_user_agent, 

20 respect_robots_txt=cfg.crawl_respect_robots_txt, 

21 extra_wait_ms=cfg.crawl_browser_extra_wait_ms, 

22 login=login, 

23 )