Coverage for src/lilbee/crawler/markdown.py: 100%

1"""HTML to markdown conversion, owned by lilbee rather than by the fetcher backend. 

2 

3Fetching a page and converting it are separate jobs. Keeping the conversion here 

4means a fetcher backend only has to return HTML, and the conversion runs where 

5lilbee can await it instead of inside the backend's own call stack. 

6""" 

7 

8from __future__ import annotations 

9 

10import re 

11 

12# Mirrors crawl4ai's own <base href> derivation (async_webcrawler) verbatim so a 

13# silenced re-conversion resolves relative links the way an un-silenced crawl would. 

14# A real HTML parser here would resolve differently and diverge from the backend; 

15# keep this in sync with crawl4ai instead. 

16_BASE_HREF = re.compile(r"<base\s[^>]*href\s*=\s*[\"']([^\"']+)[\"']", re.IGNORECASE) 

17 

18 

19def base_url_for(html: str, url: str, redirected_url: str | None = None) -> str: 

20 """The URL relative links resolve against: a ``<base href>`` if the page sets one.""" 

21 match = _BASE_HREF.search(html) 

22 if match: 

23 return match.group(1) 

24 return redirected_url or url 

25 

26 

27def html_to_markdown(html: str, base_url: str) -> str: 

28 """Convert *html* to markdown, resolving relative links against *base_url*. 

29 

30 Imports its backend on call: this is the one place lilbee depends on a 

31 third-party HTML-to-markdown implementation. 

32 """ 

33 from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator 

34 

35 result = DefaultMarkdownGenerator().generate_markdown(html, base_url=base_url) 

36 return str(result.raw_markdown or "")