Coverage for src/lilbee/wiki/ingest.py: 100%

1"""Wiki post-ingest hook: regenerate pages touched by a recent sync.""" 

2 

3from __future__ import annotations 

4 

5import asyncio 

6import logging 

7 

8from lilbee.app.services import get_services 

9from lilbee.core.config import Config, cfg 

10 

11log = logging.getLogger(__name__) 

12 

13 

14async def incremental_update(changed_sources: set[str], config: Config | None = None) -> None: 

15 """Regenerate only the wiki pages touched by *changed_sources*. 

16 

17 Builds a fresh ``ExtractedEntity`` set from the current corpus and 

18 keeps the records whose chunk trail includes one of the changed 

19 sources. An entity with no page on disk is not by itself a reason to 

20 regenerate: its page may be a draft or a marker held for review, and 

21 re-queueing those every sync burns LLM calls and overwrites pending 

22 review content. Above ``cfg.wiki_ingest_update_cap`` touched pages 

23 the auto-update bails out and logs a manual-update hint instead. 

24 The caller is a sync and holds the sync mark, so no source key moves 

25 during the update. 

26 """ 

27 if config is None: 

28 config = cfg 

29 if not config.wiki or not changed_sources: 

30 return 

31 from lilbee.data.store import SearchChunk 

32 from lilbee.wiki import append_wiki_log, build_wiki, update_wiki_index 

33 from lilbee.wiki.entity_extractor import get_entity_extractor 

34 from lilbee.wiki.shared import WIKI_BUILD_LOCK, WikiLogAction 

35 from lilbee.wiki.stats import BuildStats 

36 

37 svc = get_services() 

38 extractor = get_entity_extractor(config.wiki_entity_mode, svc.provider, config) 

39 

40 chunks: list[SearchChunk] = [] 

41 for record in svc.store.get_sources(): 

42 chunks.extend(svc.store.get_chunks_by_source(record["filename"])) 

43 entities = await asyncio.to_thread(extractor.extract, chunks) 

44 

45 touched = [ 

46 entity 

47 for entity in entities 

48 if any(ref.source in changed_sources for ref in entity.chunk_refs) 

49 ] 

50 

51 if not touched: 

52 return 

53 

54 if len(touched) > config.wiki_ingest_update_cap: 

55 # warning, not info: the default LILBEE_LOG_LEVEL is WARNING, so 

56 # log.info would silently drop the manual-update hint and the user 

57 # would see no signal at all during `lilbee sync` when the cap trips. 

58 log.warning( 

59 "Wiki auto-update skipped: %d pages touched (cap %d). " 

60 "Run 'lilbee wiki update' for a full rebuild.", 

61 len(touched), 

62 config.wiki_ingest_update_cap, 

63 ) 

64 

65 def _log_skip() -> None: 

66 with WIKI_BUILD_LOCK: 

67 append_wiki_log( 

68 WikiLogAction.INGEST, 

69 f"skipped: {len(touched)} pages exceeds cap {config.wiki_ingest_update_cap}", 

70 config, 

71 ) 

72 

73 # log.md is shared with the builders, so the append takes the mutex in a 

74 # worker thread for the same reason the regenerate path does. 

75 await asyncio.to_thread(_log_skip) 

76 return 

77 

78 # extract_concepts=False so an incremental sync does not churn 

79 # concept slugs. Concept curation is a deliberate, user-invoked 

80 # refresh (full `lilbee wiki build`). 

81 def _regenerate() -> None: 

82 stats = BuildStats() 

83 with WIKI_BUILD_LOCK: 

84 pages = build_wiki( 

85 touched, svc.provider, svc.store, config, extract_concepts=False, stats=stats 

86 ) 

87 update_wiki_index(config) 

88 append_wiki_log( 

89 WikiLogAction.INGEST, 

90 f"{len(pages)} pages regenerated for {', '.join(sorted(changed_sources))}; " 

91 f"{stats.summary_line()}", 

92 config, 

93 ) 

94 

95 # The mutex is taken in the worker thread: acquiring it here would block the 

96 # event loop for the length of another surface's build. 

97 await asyncio.to_thread(_regenerate)