0b4c2bbb15
兩個 bug 同源於自檢測資「照能過建、不照現實建」： - ingest preflight：git status --porcelain 預設會把含中文或空格的路徑加引號 轉義，raw/ 白名單比對因此失效，中文檔名的 convert 產出被誤判為 raw/ 以外的 未提交變更而中止攝入。舊測資用 ASCII 檔名，整條路徑從未被走過。 - search：BM25Okapi 的 idf 在 df >= N/2 時 <= 0（rank_bm25 對負 idf 的替代值 epsilon x average_idf 在 average_idf 為負時同樣為負），與 search() 的 s > 0 過濾相乘，會讓「每頁都提到的核心詞」查詢全數落空——知識庫愈小、詞愈核心 愈嚴重，MCP 的 search_wiki 一併受害。改用 Lucene 式恆正 idf。 舊測資每個查詢詞都只出現在一頁（df=1），恰好避開此配置。 變更： - tools/ingest.py：preflight 比對前剝除 git 的轉義引號 - tools/search.py：_BM25 子類覆寫 _calc_idf 為 log(1 + (N-df+0.5)/(df+0.5)) - tools/selfcheck_ingest.py：新增 preflight 中文檔名放行 / 非 raw 擋下的檢查 - tools/selfcheck_search.py：新增與既有頁共用核心詞的測資，鎖住 idf 退化 - tools/convert/selfcheck.py、tools/selfcheck_lint.py：測資檔名改中文＋空格 - AGENTS.md §13.4：新增「測資照現實建，不是照能過建」規則 - README.md §3.2：補 convert 產出應保持未提交、由 ingest 一併 commit 的流程 驗證：四個自檢全數通過；分別移除兩個修法後對應檢查會失敗。 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
158 lines
7.4 KiB
Python
158 lines
7.4 KiB
Python
"""Phase 3 自檢：以假 LLM + 沙盒 git repo 驗證攝入管線的完整 plumbing——
|
||
分支建立、頁面產出、frontmatter 合規、index/log 更新、冪等跳過、就地編輯合併，
|
||
以及 kb.llm_json 的重試與 fallback 切換。不需要 Ollama 在線。
|
||
|
||
執行：.venv/Scripts/python tools/selfcheck_ingest.py
|
||
"""
|
||
import hashlib
|
||
import pathlib
|
||
import shutil
|
||
import subprocess
|
||
import sys
|
||
import tempfile
|
||
|
||
sys.path.insert(0, str(pathlib.Path(__file__).parent))
|
||
import kb
|
||
import ingest
|
||
|
||
CFG = """ollama: {base_url: "http://localhost:11434", timeout_seconds: 5}
|
||
tasks:
|
||
ingest: {model: "fake:test", temperature: 0.2, max_retries: 2}
|
||
lint: {model: "fake:test", temperature: 0.1, max_retries: 2}
|
||
fallback: {model: "fake:fb"}
|
||
limits: {mcp_response_max_tokens: 2000, ingest_chunk_max_chars: 200}
|
||
"""
|
||
|
||
|
||
def fake_llm(cfg, task, prompt, validate):
|
||
if prompt.startswith("以下是一份長文件的第"):
|
||
obj = {"summary_markdown": "- 分段重點"}
|
||
elif '"updated_markdown"' in prompt:
|
||
obj = {"updated_markdown": "- 壓測逾時\n- 新來源事實",
|
||
"updated_description": "支付閘道模組（已更新）"}
|
||
else:
|
||
obj = {"title": "支付閘道壓測報告", "description": "壓測發現逾時缺陷",
|
||
"slug": "gateway-load-test", "tags": ["load-test", "gateway"],
|
||
"summary_markdown": "- 逾時缺陷於壓測重現",
|
||
"knowledge_items": [{"type": "entity", "slug": "payment-gateway",
|
||
"title": "支付閘道", "description": "支付閘道模組",
|
||
"tags": ["gateway"], "facts_markdown": "- 壓測逾時"}]}
|
||
err = validate(obj) # 假輸出也要通過真 validator，確保契約一致
|
||
assert err is None, err
|
||
return obj, "fake:test"
|
||
|
||
|
||
def git(root, *args):
|
||
return subprocess.run(["git", *args], cwd=root, check=True, text=True,
|
||
encoding="utf-8", capture_output=True).stdout.strip()
|
||
|
||
|
||
def add_doc(root, name, text):
|
||
p = root / "raw" / "converted" / name
|
||
p.write_text(text, encoding="utf-8")
|
||
sha = hashlib.sha256(text.encode("utf-8")).hexdigest()
|
||
import json
|
||
mp = root / "raw" / "manifest.json"
|
||
m = json.loads(mp.read_text(encoding="utf-8"))
|
||
m["files"][f"raw/converted/{name}"] = {
|
||
"kind": "converted", "sha256": sha, "original_path": f"raw/originals/{name}",
|
||
"original_sha256": sha, "converter": "from_docx", "converted_at": "2026-07-14T00:00:00"}
|
||
mp.write_text(json.dumps(m, ensure_ascii=False, indent=2), encoding="utf-8")
|
||
return sha
|
||
|
||
|
||
def main():
|
||
tmp = pathlib.Path(tempfile.mkdtemp(prefix="ppqa-ingest-"))
|
||
real_llm, real_chat = kb.llm_json, kb.ollama_chat
|
||
try:
|
||
root = tmp / "repo"
|
||
for sub in ("config", "raw/originals", "raw/converted",
|
||
"wiki/summaries", "wiki/entities", "wiki/concepts"):
|
||
(root / sub).mkdir(parents=True)
|
||
(root / "config" / "models.yaml").write_text(CFG, encoding="utf-8")
|
||
(root / "raw" / "manifest.json").write_text('{"version": 1, "files": {}}',
|
||
encoding="utf-8")
|
||
(root / "index.md").write_text("# 知識庫索引\n", encoding="utf-8")
|
||
(root / "log.md").write_text("# 操作日誌\n", encoding="utf-8")
|
||
git(root, "init", "-q", "-b", "main")
|
||
git(root, "config", "user.name", "selfcheck")
|
||
git(root, "config", "user.email", "selfcheck@local")
|
||
text = "支付閘道於壓力測試下出現逾時缺陷，交易峰值時重試機制未生效。\n\n" * 12
|
||
add_doc(root, "doc1.md", text) # > 200 chars → 走分段路徑
|
||
git(root, "add", "-A")
|
||
git(root, "commit", "-q", "-m", "init")
|
||
|
||
# 1) 攝入：建分支、產頁、commit、回到 main
|
||
kb.llm_json = fake_llm
|
||
ingest.main(["raw/converted/doc1.md", "--root", str(root)])
|
||
assert git(root, "branch", "--show-current") == "main"
|
||
branch = git(root, "branch", "--list", "ingest/*").strip("* ").strip()
|
||
assert branch, "未建立 ingest 分支"
|
||
assert not git(root, "status", "--porcelain"), "工作區不乾淨"
|
||
git(root, "checkout", "-q", branch)
|
||
summary = list((root / "wiki/summaries").glob("*.md"))
|
||
entity = root / "wiki/entities/payment-gateway.md"
|
||
assert len(summary) == 1 and entity.is_file()
|
||
meta, _ = kb.parse_page(entity.read_text(encoding="utf-8"))
|
||
assert kb.validate_meta(meta) == [], kb.validate_meta(meta)
|
||
assert "payment-gateway" in (root / "index.md").read_text(encoding="utf-8")
|
||
assert "] ingest |" in (root / "log.md").read_text(encoding="utf-8")
|
||
print("PASS: 攝入 → 分支 + summary/entity 頁 + index/log")
|
||
|
||
# 2) 模擬 PR 合併後：冪等跳過；新來源觸發既有頁就地編輯
|
||
git(root, "checkout", "-q", "main")
|
||
git(root, "merge", "-q", branch)
|
||
ingest.main(["raw/converted/doc1.md", "--root", str(root)]) # 應 skip
|
||
assert git(root, "branch", "--show-current") == "main"
|
||
add_doc(root, "doc2.md", "第二份文件：支付閘道逾時已修復並通過回歸測試。")
|
||
git(root, "add", "-A")
|
||
git(root, "commit", "-q", "-m", "doc2")
|
||
ingest.main(["raw/converted/doc2.md", "--root", str(root)])
|
||
b2 = [b.strip("* ").strip() for b in
|
||
git(root, "branch", "--list", "ingest/*").splitlines() if branch not in b]
|
||
git(root, "checkout", "-q", b2[0])
|
||
meta, body = kb.parse_page(entity.read_text(encoding="utf-8"))
|
||
assert len(meta["sources"]) == 2 and "已更新" in meta["description"], meta
|
||
assert "新來源事實" in body
|
||
git(root, "checkout", "-q", "main")
|
||
print("PASS: 冪等跳過 + 既有 entity 頁就地編輯（sources 追加）")
|
||
|
||
# 3) preflight：raw/ 下未提交的 convert 產出（含中文檔名）不得被誤判為髒污
|
||
(root / "raw" / "converted" / "測試報告.md").write_text("中文", encoding="utf-8")
|
||
ingest.git_preflight(root) # 不應拋錯
|
||
stray = root / "wiki" / "concepts" / "stray.md"
|
||
stray.write_text("x", encoding="utf-8")
|
||
try:
|
||
ingest.git_preflight(root)
|
||
raise AssertionError("raw/ 以外的未提交變更應中止")
|
||
except SystemExit:
|
||
pass
|
||
stray.unlink()
|
||
print("PASS: preflight 放行 raw/（含中文檔名）、擋下 raw/ 以外的未提交變更")
|
||
|
||
# 4) kb.llm_json：重試後切 fallback；全失敗拋錯
|
||
kb.llm_json = real_llm
|
||
cfg = kb.load_config(root)
|
||
calls = {"n": 0}
|
||
def flaky(cfg_, model, prompt, temperature, json_format=False):
|
||
calls["n"] += 1
|
||
return "not-json" if calls["n"] < 3 else '{"summary_markdown": "ok"}'
|
||
kb.ollama_chat = flaky
|
||
obj, model = kb.llm_json(cfg, "ingest", "p", ingest.validate_chunk)
|
||
assert obj["summary_markdown"] == "ok" and model == "fake:fb" and calls["n"] == 3
|
||
kb.ollama_chat = lambda *a, **k: "junk"
|
||
try:
|
||
kb.llm_json(cfg, "ingest", "p", ingest.validate_chunk)
|
||
raise AssertionError("應拋 RuntimeError")
|
||
except RuntimeError as e:
|
||
assert "fallback" in str(e)
|
||
print("PASS: 重試 + fallback 切換 + 全失敗報錯")
|
||
print("ALL PASS")
|
||
finally:
|
||
kb.llm_json, kb.ollama_chat = real_llm, real_chat
|
||
shutil.rmtree(tmp, ignore_errors=True)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|