Coverage for src/lilbee/wiki/ingest.py: 100%

1"""Wiki post-ingest hook: regenerate pages touched by a recent sync.""" 

2 

3from __future__ import annotations 

4 

5import asyncio 

6import logging 

7 

8from lilbee.app.services import get_services 

9from lilbee.core.config import Config, cfg 

10 

11log = logging.getLogger(__name__) 

12 

13 

14async def incremental_update(changed_sources: set[str], config: Config | None = None) -> None: 

15 """Regenerate only the wiki pages touched by *changed_sources*. 

16 

17 Builds a fresh ``ExtractedEntity`` set from the current corpus and 

18 keeps the records whose chunk trail includes one of the changed 

19 sources. An entity with no page on disk is not by itself a reason to 

20 regenerate: its page may be a draft or a marker held for review, and 

21 re-queueing those every sync burns LLM calls and overwrites pending 

22 review content. Above ``cfg.wiki_ingest_update_cap`` touched pages 

23 the auto-update bails out and logs a manual-update hint instead. 

24 """ 

25 if config is None: 

26 config = cfg 

27 if not config.wiki or not changed_sources: 

28 return 

29 from lilbee.data.store import SearchChunk 

30 from lilbee.wiki import append_wiki_log, build_wiki, update_wiki_index 

31 from lilbee.wiki.entity_extractor import get_entity_extractor 

32 from lilbee.wiki.shared import WIKI_BUILD_LOCK, WikiLogAction 

33 from lilbee.wiki.stats import BuildStats 

34 

35 svc = get_services() 

36 extractor = get_entity_extractor(config.wiki_entity_mode, svc.provider, config) 

37 

38 chunks: list[SearchChunk] = [] 

39 for record in svc.store.get_sources(): 

40 chunks.extend(svc.store.get_chunks_by_source(record["filename"])) 

41 entities = await asyncio.to_thread(extractor.extract, chunks) 

42 

43 touched = [ 

44 entity 

45 for entity in entities 

46 if any(ref.source in changed_sources for ref in entity.chunk_refs) 

47 ] 

48 

49 if not touched: 

50 return 

51 

52 if len(touched) > config.wiki_ingest_update_cap: 

53 # warning, not info: the default LILBEE_LOG_LEVEL is WARNING, so 

54 # log.info would silently drop the manual-update hint and the user 

55 # would see no signal at all during `lilbee sync` when the cap trips. 

56 log.warning( 

57 "Wiki auto-update skipped: %d pages touched (cap %d). " 

58 "Run 'lilbee wiki update' for a full rebuild.", 

59 len(touched), 

60 config.wiki_ingest_update_cap, 

61 ) 

62 

63 def _log_skip() -> None: 

64 with WIKI_BUILD_LOCK: 

65 append_wiki_log( 

66 WikiLogAction.INGEST, 

67 f"skipped: {len(touched)} pages exceeds cap {config.wiki_ingest_update_cap}", 

68 config, 

69 ) 

70 

71 # log.md is shared with the builders, so the append takes the mutex in a 

72 # worker thread for the same reason the regenerate path does. 

73 await asyncio.to_thread(_log_skip) 

74 return 

75 

76 # extract_concepts=False so an incremental sync does not churn 

77 # concept slugs. Concept curation is a deliberate, user-invoked 

78 # refresh (full `lilbee wiki build`). 

79 def _regenerate() -> None: 

80 stats = BuildStats() 

81 with WIKI_BUILD_LOCK: 

82 pages = build_wiki( 

83 touched, svc.provider, svc.store, config, extract_concepts=False, stats=stats 

84 ) 

85 update_wiki_index(config) 

86 append_wiki_log( 

87 WikiLogAction.INGEST, 

88 f"{len(pages)} pages regenerated for {', '.join(sorted(changed_sources))}; " 

89 f"{stats.summary_line()}", 

90 config, 

91 ) 

92 

93 # The mutex is taken in the worker thread: acquiring it here would block the 

94 # event loop for the length of another surface's build. 

95 await asyncio.to_thread(_regenerate)