Coverage for src/lilbee/core/config/defaults.py: 100%
56 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-11-06 08:17 +0000
1"""Default values and constants for :mod:`lilbee.core.config`.
3Holds frozen literal data: the config file name, directory ignore lists, the
4NER label allow-list, LanceDB table names, the default HTTP timeout and context
5size, the crawl URL exclusion patterns (grouped per category), the default RAG
6and general system prompts, and the CORS allow-origin regex.
7"""
9from __future__ import annotations
11from typing import Any
12from urllib.parse import quote
14from lilbee.core.config import mediawiki_namespaces as wiki_namespaces
16# Upper bounds for the numbers lilbee passes to the crawl engine, shared by every entry point.
17MAX_CRAWL_DEPTH = 100
18MAX_CRAWL_PAGES = 10_000_000
19MAX_CRAWL_WAIT_SECONDS = 3600.0
20MAX_CRAWL_CONCURRENCY = 256
22DEFAULT_IGNORE_DIRS = frozenset(
23 {
24 "node_modules",
25 "__pycache__",
26 "venv",
27 "build",
28 "dist",
29 "target",
30 "vendor",
31 "_build",
32 "coverage",
33 "htmlcov",
34 }
35)
37# spaCy NER labels that map onto something wiki-shaped. Excludes
38# QUANTITY / ORDINAL / CARDINAL / DATE / TIME / MONEY / PERCENT /
39# LANGUAGE / LAW because pages for "42" or "2021" are never useful, and
40# NORP (nationalities / political / religious groups) because its surfaces
41# are adjectival (Saturnian, American) and make poor page subjects; opt
42# back in via the config override. FAC (buildings / airports) stays:
43# corpora routinely surface them as wiki-worthy topics.
44DEFAULT_ALLOWED_NER_LABELS = frozenset(
45 {"PERSON", "ORG", "GPE", "LOC", "EVENT", "WORK_OF_ART", "PRODUCT", "FAC"}
46)
48# Timeout for backend catalog / management HTTP calls.
49DEFAULT_HTTP_TIMEOUT = 30.0
51# Safe default + cap for chat-mode n_ctx; full 128K+ training contexts OOM laptops.
52DEFAULT_NUM_CTX = 8192
54# Idle minutes before the engine unloads its weights; also read by the engine reaper.
55ENGINE_IDLE_TTL_MINUTES_DEFAULT = 5
56CONFIG_FILE_NAME = "config.toml"
57ENV_PREFIX = "LILBEE_"
58SKIP_TOML_ENV = "LILBEE_SKIP_TOML_CONFIG"
60CHUNKS_TABLE = "chunks"
61SOURCES_TABLE = "_sources"
62CITATIONS_TABLE = "_citations"
63MEMORIES_TABLE = "_memories"
64# Which file each crawled page owns; kept, like memories, through a rebuild.
65CRAWL_PAGES_TABLE = "_crawl_pages"
66META_TABLE = "_meta"
67PAGE_TEXTS_TABLE = "_page_texts"
68CONCEPT_NODES_TABLE = "concept_nodes"
69CONCEPT_EDGES_TABLE = "concept_edges"
70CHUNK_CONCEPTS_TABLE = "chunk_concepts"
71ENTITIES_TABLE = "entities"
72ENTITY_SCHEMA_TABLE = "_entity_schema"
73# Per-(subject, source) wiki mention evidence. The wiki stub index is a
74# corpus-wide aggregate over this table, so a subject named below the floor in
75# each separately-synced source still crosses it once its rows are all present.
76WIKI_MENTIONS_TABLE = "_wiki_mentions"
78# Default URL-exclusion regexes for recursive crawls. Grouped by source
79# CMS / category. User overrides come from LILBEE_CRAWL_EXCLUDE_PATTERNS
80# (newline-separated) or config.toml.
82# WordPress scaffolding: admin UIs, APIs, RPC, numeric permalinks, Elementor.
83_WP_EXCLUDE: tuple[str, ...] = (
84 r"/wp-admin/",
85 r"/wp-login(\.php)?",
86 r"/wp-json/",
87 r"/xmlrpc\.php",
88 r"/wp-cron\.php",
89 r"/wp-includes/",
90 r"/wp-content/uploads/",
91 r"\?p=\d+",
92 r"\?page_id=\d+",
93 r"\?cat=\d+",
94 r"/elementor-\d+",
95 r"\?elementor_library",
96)
98# Pagination and archive permalinks (WP + other CMSes share this shape).
99_ARCHIVE_EXCLUDE: tuple[str, ...] = (
100 r"/page/\d+/?$",
101 r"\?paged?=\d+",
102 r"/20\d{2}(/\d{2}(/\d{2})?)?/?$",
103 r"/tag/",
104 r"/category/",
105 r"/author/",
106 r"/archives?/?$",
107 r"/comment-page-\d+",
108)
110# Syndication feeds (content-duplicated in HTML pages).
111_FEED_EXCLUDE: tuple[str, ...] = (
112 r"/feed/?$",
113 r"/feed/atom/?$",
114 r"/feed/rdf/?$",
115 r"/comments/feed/?$",
116 r"/rss/?$",
117)
119# Duplicate views of the same canonical page (AMP, print, preview).
120_DUPLICATE_VIEW_EXCLUDE: tuple[str, ...] = (
121 r"/amp/?$",
122 r"\?amp=",
123 r"\?print=",
124 r"/print/?$",
125 r"\?preview=",
126)
128# WP attachment URLs (point at media, not content pages).
129_ATTACHMENT_EXCLUDE: tuple[str, ...] = (
130 r"/attachment/",
131 r"\?attachment_id=",
132)
134# Regexes against the whole URL, not globs. A generic name matches only as the
135# whole first path segment of a site: /cart and /cart/items, but neither
136# /cartography nor /docs/payments/cart.
137_SITE_ROOT = r"^https?://[^/?#]+"
138_PATH_BOUNDARY = r"(?:/|\?|#|$)"
141def _first_segments(*paths: str) -> tuple[str, ...]:
142 """Anchor each path so it matches only the whole first path segment of a site."""
143 return tuple(_SITE_ROOT + path + _PATH_BOUNDARY for path in paths)
146# Auth and account flows (generic across CMSes and e-commerce platforms).
147_AUTH_EXCLUDE: tuple[str, ...] = _first_segments(
148 r"/login",
149 r"/logout",
150 r"/register",
151 r"/signup",
152 r"/signin",
153 r"/account",
154 r"/profile",
155 r"/password-reset",
156 r"/forgot-password",
157)
158_AUTH_EXCLUDE = (*_AUTH_EXCLUDE, r"/my-account/")
160# E-commerce transactional flows (cart / checkout / compare / etc.).
161_ECOMMERCE_EXCLUDE: tuple[str, ...] = _first_segments(
162 r"/cart",
163 r"/checkout",
164 r"/wishlist",
165 r"/orders?",
166 r"/compare",
167)
168_ECOMMERCE_EXCLUDE = (
169 *_ECOMMERCE_EXCLUDE,
170 r"/products\.json",
171 r"/collections/.+/products/.+\?page=",
172)
174# Site-meta URLs and non-HTML resources; skipped before fetch.
175_META_EXCLUDE: tuple[str, ...] = (
176 r"/sitemap[^/]*\.xml",
177 r"/robots\.txt",
178 r"/humans\.txt",
179 r"/favicon\.ico",
180 r"/\.well-known/",
181 r"\.(jpe?g|png|gif|webp|avif|svg|ico|pdf|docx?|xlsx?|pptx?|zip|tar|gz|mp3|mp4|webm|ogg|ttf|woff2?|css|js|map|json|xml)(\?.*)?$",
182)
185def _namespace_pattern(names: tuple[str, ...]) -> str:
186 """The pattern for a wiki address in one of the namespaces *names*, in any language.
188 crawlberg matches a pattern against the address with every character outside ASCII
189 percent-encoded as UTF-8, so the pattern holds each name in that form.
190 """
191 return rf"/wiki/(?:{'|'.join(dict.fromkeys(quote(name) for name in names))}):"
194# Mediawiki/Wikipedia navlinks that dominate BFS before the article body. A wiki shows each
195# namespace under its own language, so each pattern lists the names of the largest Wikipedias.
196# A pattern needs the namespace name before the first colon: an article title that holds a
197# colon, such as Re:Zero or Mission:_Impossible, is content and is crawled.
198# Help: is a content namespace and is crawled.
199_MEDIAWIKI_EXCLUDE: tuple[str, ...] = (
200 r"/wiki/Main_Page$",
201 _namespace_pattern(wiki_namespaces.PROJECT),
202 _namespace_pattern(wiki_namespaces.PORTAL),
203 _namespace_pattern(wiki_namespaces.SPECIAL),
204 _namespace_pattern(wiki_namespaces.CATEGORY),
205 _namespace_pattern(wiki_namespaces.TEMPLATE),
206 _namespace_pattern(wiki_namespaces.TEMPLATE_TALK),
207 _namespace_pattern(wiki_namespaces.TALK),
208 _namespace_pattern(wiki_namespaces.FILE),
209 _namespace_pattern(wiki_namespaces.FILE_TALK),
210 _namespace_pattern(wiki_namespaces.USER),
211 _namespace_pattern(wiki_namespaces.USER_TALK),
212 r"/w/index\.php",
213)
216def exclude_pattern_lines(value: Any) -> Any:
217 """The patterns in *value*: a string is one pattern per line, any other value passes through.
219 Regex uses commas (``{2,4}``) and pipes (alternation), so a newline is the only separator.
220 """
221 if isinstance(value, str):
222 return [line.strip() for line in value.splitlines() if line.strip()]
223 return value
226DEFAULT_CRAWL_EXCLUDE_PATTERNS: tuple[str, ...] = (
227 *_WP_EXCLUDE,
228 *_ARCHIVE_EXCLUDE,
229 *_FEED_EXCLUDE,
230 *_DUPLICATE_VIEW_EXCLUDE,
231 *_ATTACHMENT_EXCLUDE,
232 *_AUTH_EXCLUDE,
233 *_ECOMMERCE_EXCLUDE,
234 *_META_EXCLUDE,
235 *_MEDIAWIKI_EXCLUDE,
236)
239DEFAULT_RAG_SYSTEM_PROMPT = (
240 "You are a precise assistant answering from the user's own documents. "
241 "Ground every claim in the numbered context passages and nothing else; if "
242 "they don't cover the question, say so plainly instead of guessing or "
243 "answering from general knowledge. Synthesize across passages rather than "
244 "leaning on one, and if they disagree, note the conflict. Cite inline by "
245 "placing the passage number in brackets right after the claim it supports "
246 "(e.g. [1] or [2][5]), and cite only passages you actually used. Do not "
247 "write a Sources, References, or Bibliography list at the end; the app adds "
248 "the real source list for you. Prefer exact values, names, and short quotes "
249 "from the context over paraphrase. Handle any material: prose, notes, "
250 "tables, transcripts, or code; for code, prefer a working example. When "
251 "asked how to do something, lay the answer out as ordered steps; if the "
252 "context covers the procedure only partially, give the steps it contains "
253 "and name what's missing rather than glossing over the gap. Match the "
254 "answer's length to the question: exhaustive requests deserve every "
255 "relevant detail the context offers."
256)
258DEFAULT_GENERAL_SYSTEM_PROMPT = (
259 "You are a helpful, direct assistant. Answer the user's question from "
260 "general knowledge. Keep responses concise unless asked to elaborate. "
261 "For code, prefer working examples over abstract explanations."
262)
264# CORS allow-origin regex: Obsidian (desktop + iOS) and localhost loopback.
265# Mutating endpoints still require auth regardless of origin.
266DEFAULT_CORS_ORIGIN_REGEX = (
267 r"^(app://obsidian\.md"
268 r"|capacitor://localhost"
269 r"|https?://localhost(:\d+)?"
270 r"|https?://127\.0\.0\.1(:\d+)?"
271 r"|https?://\[::1\](:\d+)?)$"
272)
275def env_var_name(field: str) -> str:
276 """The ``LILBEE_<FIELD>`` environment variable that sets a config field."""
277 return f"{ENV_PREFIX}{field.upper()}"