Phase 3: 攝入管線 — kb.py 共用模組 + ingest.py（重試/fallback/branch+PR）+ selfcheck
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
LittleYellow
186
tools/kb.py
Normal file
186
tools/kb.py
Normal file
@@ -0,0 +1,186 @@
|
||||
"""共用模組：config 載入驗證、本地 Ollama 呼叫（重試 + fallback）、
|
||||
wiki 頁 frontmatter 解析/寫出、index.md 重建、log.md 追加。
|
||||
|
||||
供 ingest.py / lint.py / search.py / mcp/server.py 使用。
|
||||
"""
|
||||
import datetime
|
||||
import json
|
||||
import pathlib
|
||||
import os
|
||||
import re
|
||||
import urllib.request
|
||||
|
||||
import yaml
|
||||
|
||||
ROOT = pathlib.Path(__file__).resolve().parents[1]
|
||||
PAGE_DIRS = {"summary": "wiki/summaries", "entity": "wiki/entities", "concept": "wiki/concepts"}
|
||||
PAGE_TYPES = set(PAGE_DIRS)
|
||||
STATUSES = {"draft", "reviewed", "stale"}
|
||||
REQUIRED_FM = ("type", "title", "description", "tags", "timestamp", "sources", "status")
|
||||
|
||||
|
||||
# ---------- config ----------
|
||||
|
||||
def load_config(root=ROOT):
|
||||
"""讀取 config/models.yaml。缺欄位報錯，不使用隱含預設值（AGENTS.md §1.2）。"""
|
||||
path = root / "config" / "models.yaml"
|
||||
cfg = yaml.safe_load(path.read_text(encoding="utf-8"))
|
||||
try:
|
||||
cfg["ollama"]["base_url"]
|
||||
cfg["ollama"]["timeout_seconds"]
|
||||
for task in ("ingest", "lint"):
|
||||
for key in ("model", "temperature", "max_retries"):
|
||||
cfg["tasks"][task][key]
|
||||
cfg["tasks"]["fallback"]["model"]
|
||||
cfg["limits"]["mcp_response_max_tokens"]
|
||||
cfg["limits"]["ingest_chunk_max_chars"]
|
||||
except (KeyError, TypeError) as e:
|
||||
raise SystemExit(f"config/models.yaml 缺欄位：{e}")
|
||||
for task, spec in cfg["tasks"].items():
|
||||
if spec["model"] == "CHANGE_ME":
|
||||
raise SystemExit(f"config/models.yaml：tasks.{task}.model 尚未設定")
|
||||
if spec["model"].endswith(":cloud"):
|
||||
raise SystemExit(f"tasks.{task}.model 為 :cloud 模型，違反資料落地（AGENTS.md §1.1）")
|
||||
# 環境變數是唯一允許的覆寫來源（AGENTS.md §1.2）
|
||||
cfg["ollama"]["base_url"] = os.environ.get("OLLAMA_BASE_URL", cfg["ollama"]["base_url"])
|
||||
return cfg
|
||||
|
||||
|
||||
# ---------- Ollama ----------
|
||||
|
||||
def ollama_chat(cfg, model, prompt, temperature, json_format=False):
|
||||
"""單次呼叫本地 Ollama /api/chat（stdlib urllib，不需額外依賴）。"""
|
||||
body = {
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"stream": False,
|
||||
"options": {"temperature": temperature},
|
||||
}
|
||||
if json_format:
|
||||
body["format"] = "json"
|
||||
req = urllib.request.Request(
|
||||
cfg["ollama"]["base_url"].rstrip("/") + "/api/chat",
|
||||
data=json.dumps(body).encode("utf-8"),
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=cfg["ollama"]["timeout_seconds"]) as r:
|
||||
return json.loads(r.read())["message"]["content"]
|
||||
|
||||
|
||||
def llm_json(cfg, task, prompt, validate):
|
||||
"""要求 JSON 輸出的呼叫：主模型重試 max_retries 次，仍失敗切 fallback 模型。
|
||||
|
||||
本地模型輸出不穩定是預期情況（AGENTS.md 前提），故每次失敗把錯誤附回
|
||||
prompt 再試。validate(obj) 回傳錯誤訊息字串或 None。
|
||||
回傳 (obj, 實際使用的 model)。全部失敗拋 RuntimeError。
|
||||
"""
|
||||
spec = cfg["tasks"][task]
|
||||
attempts = []
|
||||
for model in (spec["model"], cfg["tasks"]["fallback"]["model"]):
|
||||
hint = ""
|
||||
for _ in range(spec["max_retries"]):
|
||||
try:
|
||||
raw = ollama_chat(cfg, model, prompt + hint, spec["temperature"],
|
||||
json_format=True)
|
||||
obj = json.loads(raw)
|
||||
err = validate(obj)
|
||||
if err is None:
|
||||
return obj, model
|
||||
except Exception as e:
|
||||
err = f"{type(e).__name__}: {e}"
|
||||
attempts.append(f"{model}: {err}")
|
||||
hint = f"\n\n注意：上次輸出無效（{err}）。請只輸出符合規格的 JSON，不得有其他文字。"
|
||||
raise RuntimeError("LLM 輸出驗證失敗（含 fallback）：" + " | ".join(attempts[-4:]))
|
||||
|
||||
|
||||
# ---------- wiki 頁 ----------
|
||||
|
||||
def parse_page(text):
|
||||
"""回傳 (frontmatter dict, body)。格式不符拋 ValueError。"""
|
||||
m = re.match(r"^---\r?\n(.*?)\r?\n---\r?\n(.*)$", text, re.S)
|
||||
if not m:
|
||||
raise ValueError("缺少 YAML frontmatter")
|
||||
meta = yaml.safe_load(m.group(1))
|
||||
if not isinstance(meta, dict):
|
||||
raise ValueError("frontmatter 不是 mapping")
|
||||
return meta, m.group(2).strip()
|
||||
|
||||
|
||||
def dump_page(meta, body):
|
||||
fm = yaml.safe_dump(meta, allow_unicode=True, sort_keys=False)
|
||||
return f"---\n{fm}---\n\n{body.strip()}\n"
|
||||
|
||||
|
||||
def validate_meta(meta):
|
||||
"""依 AGENTS.md §3.2 檢查 frontmatter，回傳錯誤訊息列表。"""
|
||||
errs = [f"缺欄位 {k}" for k in REQUIRED_FM if k not in meta]
|
||||
if errs:
|
||||
return errs
|
||||
if meta["type"] not in PAGE_TYPES:
|
||||
errs.append(f"type 不合法：{meta['type']}")
|
||||
if meta["status"] not in STATUSES:
|
||||
errs.append(f"status 不合法：{meta['status']}")
|
||||
if not isinstance(meta["tags"], list) or not meta["tags"]:
|
||||
errs.append("tags 須為非空列表")
|
||||
if not isinstance(meta["sources"], list) or not meta["sources"]:
|
||||
errs.append("sources 須為非空列表")
|
||||
for s in meta.get("sources") or []:
|
||||
if not re.match(r"^raw/(converted|originals)/.+#[0-9a-f]{8}$", str(s)):
|
||||
errs.append(f"source_ref 格式不符：{s}")
|
||||
if not re.match(r"^\d{4}-\d{2}-\d{2}$", str(meta["timestamp"])):
|
||||
errs.append(f"timestamp 須為 YYYY-MM-DD：{meta['timestamp']}")
|
||||
return errs
|
||||
|
||||
|
||||
def safe_slug(s, fallback):
|
||||
s = re.sub(r"[^a-z0-9]+", "-", str(s).lower()).strip("-")[:60]
|
||||
return s or fallback
|
||||
|
||||
|
||||
def iter_pages(root=ROOT):
|
||||
for sub in PAGE_DIRS.values():
|
||||
d = root / sub
|
||||
if d.is_dir():
|
||||
yield from sorted(d.glob("*.md"))
|
||||
|
||||
|
||||
# ---------- index.md / log.md ----------
|
||||
|
||||
def rebuild_index(root=ROOT):
|
||||
"""由全部 wiki 頁 frontmatter 決定性重建 index.md（AGENTS.md §10）。"""
|
||||
groups = {"summary": [], "entity": [], "concept": []}
|
||||
for p in iter_pages(root):
|
||||
meta, _ = parse_page(p.read_text(encoding="utf-8"))
|
||||
rel = p.relative_to(root).as_posix()
|
||||
tags = " ".join(f"#{t}" for t in meta["tags"])
|
||||
groups[meta["type"]].append(
|
||||
(str(meta["title"]), f"- [{meta['title']}]({rel}) — {meta['description']} ｜ {tags}"))
|
||||
def section(key):
|
||||
lines = [line for _, line in sorted(groups[key])]
|
||||
return "\n".join(lines) if lines else "（尚無頁面）"
|
||||
text = f"""# 知識庫索引
|
||||
|
||||
<!-- 目錄式導航（AGENTS.md §10）：每頁一行，由 ingest 管線自動重建，格式：
|
||||
- [標題](路徑) — 一句摘要 ｜ #tag1 #tag2
|
||||
查詢工作流一律先讀本檔定位候選頁（最多 10 頁）。 -->
|
||||
|
||||
## 摘要頁（summaries）
|
||||
|
||||
{section('summary')}
|
||||
|
||||
## 實體頁（entities）
|
||||
|
||||
{section('entity')}
|
||||
|
||||
## 概念頁（concepts）
|
||||
|
||||
{section('concept')}
|
||||
"""
|
||||
(root / "index.md").write_text(text, encoding="utf-8")
|
||||
|
||||
|
||||
def append_log(root, op, title, lines):
|
||||
entry = f"\n## [{datetime.date.today().isoformat()}] {op} | {title}\n\n"
|
||||
entry += "".join(f"- {l}\n" for l in lines)
|
||||
with open(root / "log.md", "a", encoding="utf-8") as f:
|
||||
f.write(entry)
|
||||
Reference in New Issue
Block a user