Coverage for src/lilbee/core/config/defaults.py: 100%
39 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-08-14 11:46 +0000
1"""Default values and constants for :mod:`lilbee.core.config`.
3Holds frozen literal data: directory ignore lists, the NER label allow-list,
4LanceDB table names, the default HTTP timeout and context size, the crawl URL
5exclusion patterns (grouped per category), the default RAG and general system
6prompts, and the CORS allow-origin regex.
7"""
9from __future__ import annotations
11from collections.abc import Mapping
12from types import MappingProxyType
14DEFAULT_IGNORE_DIRS = frozenset(
15 {
16 "node_modules",
17 "__pycache__",
18 "venv",
19 "build",
20 "dist",
21 "target",
22 "vendor",
23 "_build",
24 "coverage",
25 "htmlcov",
26 }
27)
29# spaCy NER labels that map onto something wiki-shaped. Excludes
30# QUANTITY / ORDINAL / CARDINAL / DATE / TIME / MONEY / PERCENT /
31# LANGUAGE / LAW because pages for "42" or "2021" are never useful, and
32# NORP (nationalities / political / religious groups) because its surfaces
33# are adjectival (Saturnian, American) and make poor page subjects; opt
34# back in via the config override. FAC (buildings / airports) stays:
35# corpora routinely surface them as wiki-worthy topics.
36DEFAULT_ALLOWED_NER_LABELS = frozenset(
37 {"PERSON", "ORG", "GPE", "LOC", "EVENT", "WORK_OF_ART", "PRODUCT", "FAC"}
38)
40# Timeout for backend catalog / management HTTP calls.
41DEFAULT_HTTP_TIMEOUT = 30.0
43# Safe default + cap for chat-mode n_ctx; full 128K+ training contexts OOM laptops.
44DEFAULT_NUM_CTX = 8192
46CHUNKS_TABLE = "chunks"
47SOURCES_TABLE = "_sources"
48CITATIONS_TABLE = "_citations"
49MEMORIES_TABLE = "_memories"
50META_TABLE = "_meta"
51PAGE_TEXTS_TABLE = "_page_texts"
52CONCEPT_NODES_TABLE = "concept_nodes"
53CONCEPT_EDGES_TABLE = "concept_edges"
54CHUNK_CONCEPTS_TABLE = "chunk_concepts"
55ENTITIES_TABLE = "entities"
56ENTITY_SCHEMA_TABLE = "_entity_schema"
57# Per-(subject, source) wiki mention evidence. The wiki stub index is a
58# corpus-wide aggregate over this table, so a subject named below the floor in
59# each separately-synced source still crosses it once its rows are all present.
60WIKI_MENTIONS_TABLE = "_wiki_mentions"
62# Tables an ingest writes per source, and the column holding the source key.
63INGEST_SOURCE_COLUMNS: Mapping[str, str] = MappingProxyType(
64 {
65 CHUNKS_TABLE: "source",
66 PAGE_TEXTS_TABLE: "source",
67 CHUNK_CONCEPTS_TABLE: "chunk_source",
68 ENTITIES_TABLE: "source",
69 CITATIONS_TABLE: "source_filename",
70 SOURCES_TABLE: "filename",
71 }
72)
74# Default URL-exclusion regexes for recursive crawls. Grouped by source
75# CMS / category. User overrides come from LILBEE_CRAWL_EXCLUDE_PATTERNS
76# (newline-separated) or config.toml.
78# WordPress scaffolding: admin UIs, APIs, RPC, numeric permalinks, Elementor.
79_WP_EXCLUDE: tuple[str, ...] = (
80 r"/wp-admin/",
81 r"/wp-login(\.php)?",
82 r"/wp-json/",
83 r"/xmlrpc\.php",
84 r"/wp-cron\.php",
85 r"/wp-includes/",
86 r"/wp-content/uploads/",
87 r"\?p=\d+",
88 r"\?page_id=\d+",
89 r"\?cat=\d+",
90 r"/elementor-\d+",
91 r"\?elementor_library",
92)
94# Pagination and archive permalinks (WP + other CMSes share this shape).
95_ARCHIVE_EXCLUDE: tuple[str, ...] = (
96 r"/page/\d+/?$",
97 r"\?paged?=\d+",
98 r"/20\d{2}(/\d{2}(/\d{2})?)?/?$",
99 r"/tag/",
100 r"/category/",
101 r"/author/",
102 r"/archives?/?$",
103 r"/comment-page-\d+",
104)
106# Syndication feeds (content-duplicated in HTML pages).
107_FEED_EXCLUDE: tuple[str, ...] = (
108 r"/feed/?$",
109 r"/feed/atom/?$",
110 r"/feed/rdf/?$",
111 r"/comments/feed/?$",
112 r"/rss/?$",
113)
115# Duplicate views of the same canonical page (AMP, print, preview).
116_DUPLICATE_VIEW_EXCLUDE: tuple[str, ...] = (
117 r"/amp/?$",
118 r"\?amp=",
119 r"\?print=",
120 r"/print/?$",
121 r"\?preview=",
122)
124# WP attachment URLs (point at media, not content pages).
125_ATTACHMENT_EXCLUDE: tuple[str, ...] = (
126 r"/attachment/",
127 r"\?attachment_id=",
128)
130# Regexes against the whole URL, not globs, so a bare prefix also matches
131# longer words: /cart excluded /cartography. Require a segment boundary.
132_PATH_BOUNDARY = r"(?:/|\?|#|$)"
135def _whole_segments(*paths: str) -> tuple[str, ...]:
136 """Anchor each path prefix so it matches a whole segment, not a word."""
137 return tuple(path + _PATH_BOUNDARY for path in paths)
140# Auth and account flows (generic across CMSes and e-commerce platforms).
141_AUTH_EXCLUDE: tuple[str, ...] = _whole_segments(
142 r"/login",
143 r"/logout",
144 r"/register",
145 r"/signup",
146 r"/signin",
147 r"/account",
148 r"/profile",
149 r"/password-reset",
150 r"/forgot-password",
151)
152_AUTH_EXCLUDE = (*_AUTH_EXCLUDE, r"/my-account/")
154# E-commerce transactional flows (cart / checkout / compare / etc.).
155_ECOMMERCE_EXCLUDE: tuple[str, ...] = _whole_segments(
156 r"/cart",
157 r"/checkout",
158 r"/wishlist",
159 r"/orders?",
160 r"/compare",
161)
162_ECOMMERCE_EXCLUDE = (
163 *_ECOMMERCE_EXCLUDE,
164 r"/products\.json",
165 r"/collections/.+/products/.+\?page=",
166)
168# Marketing / tracking query parameters (utm_*, fbclid, gclid, etc.).
169# Vendor campaign tokens only. Dropping ?utm_source= is free (the canonical
170# URL is in the frontier too), but ?ref= and ?share= are ordinary content
171# links on docs and forum platforms.
172_TRACKING_EXCLUDE: tuple[str, ...] = (
173 (
174 r"[?&]("
175 r"utm_[a-z_]+"
176 r"|fbclid|gclid|msclkid|yclid"
177 r"|mc_cid|mc_eid"
178 r"|_hsenc|_hsmi|hsCtaTracking"
179 r"|mkt_tok|mkt_[a-z_]+"
180 r"|trk|trkInfo"
181 r"|dm_i"
182 r"|vero_id|vero_conv"
183 r"|oly_anon_id|oly_enc_id"
184 r"|igshid"
185 r"|pk_campaign|pk_source|pk_medium|pk_[a-z_]+"
186 r"|_ga"
187 r"|affiliate|aff_id|aff_ref|aff|partner"
188 r"|srsltid"
189 r"|replytocom"
190 r")="
191 ),
192)
194# Site-meta URLs and non-HTML resources; skipped before fetch.
195_META_EXCLUDE: tuple[str, ...] = (
196 r"/sitemap[^/]*\.xml",
197 r"/robots\.txt",
198 r"/humans\.txt",
199 r"/favicon\.ico",
200 r"/\.well-known/",
201 r"\.(jpe?g|png|gif|webp|avif|svg|ico|pdf|docx?|xlsx?|pptx?|zip|tar|gz|mp3|mp4|webm|ogg|ttf|woff2?|css|js|map|json|xml)(\?.*)?$",
202)
204# Mediawiki/Wikipedia navlinks that dominate BFS before the article body.
205_MEDIAWIKI_EXCLUDE: tuple[str, ...] = (
206 r"/wiki/Main_Page$",
207 r"/wiki/Wikipedia:",
208 r"/wiki/Portal:",
209 r"/wiki/Help:",
210 r"/wiki/Special:",
211 r"/wiki/Category:",
212 r"/wiki/Template:",
213 r"/wiki/Template_talk:",
214 r"/wiki/Talk:",
215 r"/wiki/File:",
216 r"/wiki/File_talk:",
217 r"/wiki/User:",
218 r"/wiki/User_talk:",
219 r"/w/index\.php",
220)
222DEFAULT_CRAWL_EXCLUDE_PATTERNS: tuple[str, ...] = (
223 *_WP_EXCLUDE,
224 *_ARCHIVE_EXCLUDE,
225 *_FEED_EXCLUDE,
226 *_DUPLICATE_VIEW_EXCLUDE,
227 *_ATTACHMENT_EXCLUDE,
228 *_AUTH_EXCLUDE,
229 *_ECOMMERCE_EXCLUDE,
230 *_TRACKING_EXCLUDE,
231 *_META_EXCLUDE,
232 *_MEDIAWIKI_EXCLUDE,
233)
236DEFAULT_RAG_SYSTEM_PROMPT = (
237 "You are a precise assistant answering from the user's own documents. "
238 "Ground every claim in the numbered context passages and nothing else; if "
239 "they don't cover the question, say so plainly instead of guessing or "
240 "answering from general knowledge. Synthesize across passages rather than "
241 "leaning on one, and if they disagree, note the conflict. Cite inline by "
242 "placing the passage number in brackets right after the claim it supports "
243 "(e.g. [1] or [2][5]), and cite only passages you actually used. Do not "
244 "write a Sources, References, or Bibliography list at the end; the app adds "
245 "the real source list for you. Prefer exact values, names, and short quotes "
246 "from the context over paraphrase. Handle any material: prose, notes, "
247 "tables, transcripts, or code; for code, prefer a working example. When "
248 "asked how to do something, lay the answer out as ordered steps; if the "
249 "context covers the procedure only partially, give the steps it contains "
250 "and name what's missing rather than glossing over the gap. Match the "
251 "answer's length to the question: exhaustive requests deserve every "
252 "relevant detail the context offers."
253)
255DEFAULT_GENERAL_SYSTEM_PROMPT = (
256 "You are a helpful, direct assistant. Answer the user's question from "
257 "general knowledge. Keep responses concise unless asked to elaborate. "
258 "For code, prefer working examples over abstract explanations."
259)
261# CORS allow-origin regex: Obsidian (desktop + iOS) and localhost loopback.
262# Mutating endpoints still require auth regardless of origin.
263DEFAULT_CORS_ORIGIN_REGEX = (
264 r"^(app://obsidian\.md"
265 r"|capacitor://localhost"
266 r"|https?://localhost(:\d+)?"
267 r"|https?://127\.0\.0\.1(:\d+)?"
268 r"|https?://\[::1\](:\d+)?)$"
269)