yellowadmin/pp-qa-km
1
0
Fork 0
Code Issues 5 Pull Requests Actions Packages Projects Releases Wiki Activity

修復中文路徑 preflight 與 BM25 核心詞落空，測資改照現實建

Browse Source 
兩個 bug 同源於自檢測資「照能過建、不照現實建」：

- ingest preflight：git status --porcelain 預設會把含中文或空格的路徑加引號
  轉義，raw/ 白名單比對因此失效，中文檔名的 convert 產出被誤判為 raw/ 以外的
  未提交變更而中止攝入。舊測資用 ASCII 檔名，整條路徑從未被走過。

- search：BM25Okapi 的 idf 在 df >= N/2 時 <= 0（rank_bm25 對負 idf 的替代值
  epsilon x average_idf 在 average_idf 為負時同樣為負），與 search() 的 s > 0
  過濾相乘，會讓「每頁都提到的核心詞」查詢全數落空——知識庫愈小、詞愈核心
  愈嚴重，MCP 的 search_wiki 一併受害。改用 Lucene 式恆正 idf。
  舊測資每個查詢詞都只出現在一頁（df=1），恰好避開此配置。

變更：
- tools/ingest.py：preflight 比對前剝除 git 的轉義引號
- tools/search.py：_BM25 子類覆寫 _calc_idf 為 log(1 + (N-df+0.5)/(df+0.5))
- tools/selfcheck_ingest.py：新增 preflight 中文檔名放行 / 非 raw 擋下的檢查
- tools/selfcheck_search.py：新增與既有頁共用核心詞的測資，鎖住 idf 退化
- tools/convert/selfcheck.py、tools/selfcheck_lint.py：測資檔名改中文＋空格
- AGENTS.md §13.4：新增「測資照現實建，不是照能過建」規則
- README.md §3.2：補 convert 產出應保持未提交、由 ingest 一併 commit 的流程

驗證：四個自檢全數通過；分別移除兩個修法後對應檢查會失敗。

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
LittleYellow
2026-07-22 07:21:45 +08:00
parent cfd1876359
commit 0b4c2bbb15
8 changed files with 108 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
tools/convert/selfcheck.py
View File

@@ -24,6 +24,14 @@ import kb
import to_image
# 測資檔名一律用「中文 + 空格」——真實來源檔就長這樣，而 git status --porcelain
# 對這兩者都會加引號轉義。ASCII 檔名的測資曾讓 ingest 的 preflight bug 溜過（§12）。
DOCX, XLSX, PDF, SCAN = ("壓測 報告.docx", "測試案例 清單.xlsx",
"規格 說明.pdf", "掃描件 無文字層.pdf")
PPTX, HTML, TXT, DOC = ("結案 簡報.pptx", "指引 快照.html",
"不支援 筆記.txt", "舊版 報告.doc")
def make_fixtures(d):
import docx as docxlib
doc = docxlib.Document()
@@ -34,7 +42,7 @@ def make_fixtures(d):
t.rows[0].cells[1].text = "結果"
t.rows[1].cells[0].text = "TC-001"
t.rows[1].cells[1].text = "FAIL"
doc.save(d / "report.docx")
doc.save(d / DOCX)
import openpyxl
wb = openpyxl.Workbook()
@@ -42,17 +50,17 @@ def make_fixtures(d):
ws.title = "測項"
ws.append(["編號", "說明"])
ws.append(["TC-001", "逾時 | 重試"])
wb.save(d / "cases.xlsx")
wb.save(d / XLSX)
import fitz
pdf = fitz.open()
page = pdf.new_page()
page.insert_text((72, 72), "Payment gateway timeout defect reproduced under load test.")
pdf.save(d / "text.pdf")
pdf.save(d / PDF)
pdf.close()
scanned = fitz.open()
scanned.new_page() # 無文字層 → 應判 needs_ocr
scanned.save(d / "scanned.pdf")
scanned.save(d / SCAN)
scanned.close()
from pptx import Presentation
@@ -60,9 +68,9 @@ def make_fixtures(d):
slide = prs.slides.add_slide(prs.slide_layouts[1])
slide.shapes.title.text = "結案報告"
slide.notes_slide.notes_text_frame.text = "備註：法遵項目全數通過"
prs.save(d / "closing.pptx")
prs.save(d / PPTX)
(d / "page.html").write_text(
(d / HTML).write_text(
"<html><body><article><h1>AML 測試指引</h1>"
+ "<p>可疑交易監控測試需涵蓋大額交易與分散交易兩種樣態。</p>" * 8
+ "</article></body></html>", encoding="utf-8")
@@ -75,17 +83,17 @@ def main():
fx.mkdir()
make_fixtures(fx)
md, st = from_docx.convert(fx / "report.docx")
md, st = from_docx.convert(fx / DOCX)
assert st == "ok" and "# 支付閘道測試報告" in md and "| TC-001 | FAIL |" in md, md
md, st = from_xlsx.convert(fx / "cases.xlsx")
md, st = from_xlsx.convert(fx / XLSX)
assert st == "ok" and "## 工作表：測項" in md and "逾時 \\| 重試" in md, md
md, st = from_pdf.convert(fx / "text.pdf")
md, st = from_pdf.convert(fx / PDF)
assert st == "ok" and "<!-- page 1 -->" in md and "timeout defect" in md, md
_, st = from_pdf.convert(fx / "scanned.pdf")
_, st = from_pdf.convert(fx / SCAN)
assert st == "needs_ocr", st
md, st = from_pptx.convert(fx / "closing.pptx")
md, st = from_pptx.convert(fx / PPTX)
assert st == "ok" and "## Slide 1" in md and "法遵項目全數通過" in md, md
md, st = from_web.extract((fx / "page.html").read_text(encoding="utf-8"))
md, st = from_web.extract((fx / HTML).read_text(encoding="utf-8"))
assert st == "ok" and "可疑交易監控" in md, md
print("PASS: 5 個轉換器 + needs_ocr 偵測")
@@ -95,22 +103,22 @@ def main():
(root / sub).mkdir(parents=True)
(root / "raw/manifest.json").write_text('{"version": 1, "files": {}}', encoding="utf-8")
convert.main([str(fx / "report.docx"), str(fx / "scanned.pdf"),
str(fx / "page.html"), "--root", str(root)])
convert.main([str(fx / DOCX), str(fx / SCAN),
str(fx / HTML), "--root", str(root)])
m = json.loads((root / "raw/manifest.json").read_text(encoding="utf-8"))
origs = {k: v for k, v in m["files"].items() if v["kind"] == "original"}
convs = {k: v for k, v in m["files"].items() if v["kind"] == "converted"}
assert len(origs) == 3 and len(convs) == 2, m
assert origs["raw/originals/scanned.pdf"]["status"] == "needs_ocr"
assert origs[f"raw/originals/{SCAN}"]["status"] == "needs_ocr"
for k, v in convs.items():
assert (root / k).is_file(), k
assert m["files"][v["original_path"]]["sha256"] == v["original_sha256"]
assert (root / "raw/originals/report.docx").is_file()
assert (root / f"raw/originals/{DOCX}").is_file()
print("PASS: convert.py 端到端 + manifest 對應")
# 冪等：同檔再跑一次 → skip，manifest 不變
before = (root / "raw/manifest.json").read_text(encoding="utf-8")
convert.main([str(fx / "report.docx"), "--root", str(root)])
convert.main([str(fx / DOCX), "--root", str(root)])
assert (root / "raw/manifest.json").read_text(encoding="utf-8") == before
print("PASS: 冪等（同 hash 跳過）")
@@ -119,7 +127,7 @@ def main():
for sub in ("raw/originals", "raw/converted"):
(root2 / sub).mkdir(parents=True)
(root2 / "raw/manifest.json").write_text('{"version": 1, "files": {}}', encoding="utf-8")
convert.main([str(fx / "report.docx"), "--dry-run", "--root", str(root2)])
convert.main([str(fx / DOCX), "--dry-run", "--root", str(root2)])
m2 = json.loads((root2 / "raw/manifest.json").read_text(encoding="utf-8"))
assert m2["files"] == {} and not list((root2 / "raw/originals").iterdir())
print("PASS: --dry-run 不落地")
@@ -129,13 +137,13 @@ def main():
for sub in ("raw/originals", "raw/converted"):
(root3 / sub).mkdir(parents=True)
(root3 / "raw/manifest.json").write_text('{"version": 1, "files": {}}', encoding="utf-8")
(fx / "notes.txt").write_text("不支援的格式", encoding="utf-8") # → ValueError
(fx / TXT).write_text("不支援的格式", encoding="utf-8") # → ValueError
err = io.StringIO()
code = None
with contextlib.redirect_stderr(err):
try:
convert.main([str(fx / "report.docx"), str(fx / "notes.txt"),
str(fx / "missing.docx"), # 不存在 → FileNotFoundError
convert.main([str(fx / DOCX), str(fx / TXT),
str(fx / "不存在 檔案.docx"), # 不存在 → FileNotFoundError
"--root", str(root3)])
except SystemExit as e:
code = e.code
@@ -144,7 +152,7 @@ def main():
assert "ValueError" in log and "FileNotFoundError" in log, log # 錯誤帶例外型別
assert "2 個項目轉換失敗" in log, log # 尾端彙總
m3 = json.loads((root3 / "raw/manifest.json").read_text(encoding="utf-8"))
assert (root3 / "raw/originals/report.docx").is_file() # 好檔仍轉換並落地
assert (root3 / f"raw/originals/{DOCX}").is_file() # 好檔仍轉換並落地
assert any(v["kind"] == "converted" for v in m3["files"].values()), m3
print("PASS: 失敗回報（例外型別 + 尾端彙總，批次不中斷）")
@@ -153,21 +161,21 @@ def main():
for sub in ("raw/originals", "raw/converted"):
(root4 / sub).mkdir(parents=True)
(root4 / "raw/manifest.json").write_text('{"version": 1, "files": {}}', encoding="utf-8")
(fx / "old.doc").write_bytes(b"\xd0\xcf\x11\xe0legacy-stub") # 非有效 .doc，僅測路由/錯誤
(fx / DOC).write_bytes(b"\xd0\xcf\x11\xe0legacy-stub") # 非有效 .doc，僅測路由/錯誤
err = io.StringIO()
code = None
with contextlib.redirect_stderr(err):
try:
convert.main([str(fx / "old.doc"), "--root", str(root4)])
convert.main([str(fx / DOC), "--root", str(root4)])
except SystemExit as e:
code = e.code
log = err.getvalue()
assert code == 1 and "old.doc" in log, log
assert code == 1 and DOC in log, log
assert "不支援的格式" not in log and "RuntimeError" in log, log # 已路由 legacy，乾淨報錯
print("PASS: 舊格式路由至 legacy（soffice 缺失/升版失敗皆乾淨報錯）")
# A: to_image render（deterministic，真跑）
imgs = to_image.render_pdf_pages(fx / "text.pdf", tmp / "imgs")
imgs = to_image.render_pdf_pages(fx / PDF, tmp / "imgs")
assert imgs and all(p.is_file() and p.stat().st_size > 0 for p in imgs), imgs
print(f"PASS: to_image render（{len(imgs)} 頁 PNG）")
@@ -212,7 +220,7 @@ def main():
(root_v / "raw/manifest.json").write_text('{"version": 1, "files": {}}', encoding="utf-8")
kb.ollama_chat = fake_chat
try:
convert.main([str(fx / "scanned.pdf"), "--vision", "--root", str(root_v)])
convert.main([str(fx / SCAN), "--vision", "--root", str(root_v)])
finally:
kb.ollama_chat = orig_chat
mv = json.loads((root_v / "raw/manifest.json").read_text(encoding="utf-8"))

4
tools/ingest.py
View File

@@ -191,8 +191,10 @@ def git_preflight(root):
branch = run_git(root, "branch", "--show-current", capture=True).stdout.strip()
if branch != "main":
raise SystemExit(f"須在 main 分支執行（目前：{branch}）")
# git 預設 core.quotepath=true，含非 ASCII（中文檔名）或空格的路徑會被引號包住並轉義，
# 故比對前先剝除開頭引號，否則 convert 產出的中文檔名會被誤判為 raw/ 以外的變更。
dirty = [l for l in run_git(root, "status", "--porcelain", capture=True)
.stdout.splitlines() if l.strip() and not l[3:].startswith("raw/")]
.stdout.splitlines() if l.strip() and not l[3:].lstrip('"').startswith("raw/")]
if dirty:
raise SystemExit("工作區有 raw/ 以外的未提交變更，請先處理：\n" + "\n".join(dirty))

17
tools/search.py
View File

@@ -5,6 +5,7 @@
"""
import argparse
import json
import math
import pathlib
import re
import sys
@@ -27,6 +28,20 @@ def tokenize(text):
return tokens
class _BM25(BM25Okapi):
"""改用 Lucene 式 idf：log(1 + (N-df+0.5)/(df+0.5))，恆為正。
BM25Okapi 原式在 df >= N/2 時 idf <= 0（rank_bm25 對負 idf 的替代值
epsilon × average_idf 在 average_idf 為負時同樣是負的），與 search() 的
`s > 0` 過濾相乘，會讓「每頁都提到的核心詞」查詢全數落空——知識庫愈小、
詞愈核心愈嚴重。恆正 idf 讓常見詞只是權重低，而不是被整批濾掉。
"""
def _calc_idf(self, nd):
for word, freq in nd.items():
self.idf[word] = math.log(1 + (self.corpus_size - freq + 0.5) / (freq + 0.5))
def build_corpus(root):
docs = []
for p in kb.iter_pages(root):
@@ -47,7 +62,7 @@ def search(root, query, k=10):
docs = build_corpus(pathlib.Path(root))
if not docs:
return []
bm = BM25Okapi([d["tokens"] for d in docs])
bm = _BM25([d["tokens"] for d in docs])
scores = bm.get_scores(tokenize(query))
ranked = sorted(zip(docs, scores), key=lambda x: -x[1])[:k]
return [{"path": d["path"], "title": d["title"], "description": d["description"],

15
tools/selfcheck_ingest.py
View File

@@ -117,7 +117,20 @@ def main():
git(root, "checkout", "-q", "main")
print("PASS: 冪等跳過 + 既有 entity 頁就地編輯（sources 追加）")
# 3) kb.llm_json：重試後切 fallback；全失敗拋錯
# 3) preflight：raw/ 下未提交的 convert 產出（含中文檔名）不得被誤判為髒污
(root / "raw" / "converted" / "測試報告.md").write_text("中文", encoding="utf-8")
ingest.git_preflight(root) # 不應拋錯
stray = root / "wiki" / "concepts" / "stray.md"
stray.write_text("x", encoding="utf-8")
try:
ingest.git_preflight(root)
raise AssertionError("raw/ 以外的未提交變更應中止")
except SystemExit:
pass
stray.unlink()
print("PASS: preflight 放行 raw/（含中文檔名）、擋下 raw/ 以外的未提交變更")
# 4) kb.llm_json：重試後切 fallback；全失敗拋錯
kb.llm_json = real_llm
cfg = kb.load_config(root)
calls = {"n": 0}

30
tools/selfcheck_lint.py
View File

@@ -23,6 +23,8 @@ limits: {mcp_response_max_tokens: 2000, ingest_chunk_max_chars: 12000}
"""
SHA_A = "abcd1234" + "0" * 56
SHA_B = "beef5678" + "0" * 56
SRC_OK = "raw/converted/好來源 報告.md"
SRC_UNCOVERED = "raw/converted/未覆蓋 來源.md"
def page(root, rel, meta, body="內文。"):
@@ -36,20 +38,22 @@ def build(root):
for sub in ("config", "raw/converted", "wiki/summaries", "wiki/entities", "wiki/concepts"):
(root / sub).mkdir(parents=True)
(root / "config/models.yaml").write_text(CFG, encoding="utf-8")
# 來源檔名一律「中文 + 空格」——真實 QA 文件就長這樣（§12：測資要照現實建，不是照能過建）
(root / "raw/manifest.json").write_text(json.dumps({"version": 1, "files": {
"raw/converted/good-src.md": {"kind": "converted", "sha256": SHA_A,
"original_path": "raw/originals/g.docx",
"original_sha256": SHA_A, "converter": "from_docx",
"converted_at": "2026-07-01T00:00:00"},
"raw/converted/uncovered.md": {"kind": "converted", "sha256": SHA_B,
"original_path": "raw/originals/u.docx",
"original_sha256": SHA_B, "converter": "from_docx",
"converted_at": "2026-07-01T00:00:00"},
"raw/originals/scan.pdf": {"kind": "original", "sha256": SHA_B,
"media_type": "pdf", "added_at": "2026-07-01T00:00:00",
"status": "needs_ocr"},
SRC_OK: {"kind": "converted", "sha256": SHA_A,
"original_path": "raw/originals/好來源 報告.docx",
"original_sha256": SHA_A, "converter": "from_docx",
"converted_at": "2026-07-01T00:00:00"},
SRC_UNCOVERED: {"kind": "converted", "sha256": SHA_B,
"original_path": "raw/originals/未覆蓋 來源.docx",
"original_sha256": SHA_B, "converter": "from_docx",
"converted_at": "2026-07-01T00:00:00"},
"raw/originals/掃描件 無文字層.pdf": {"kind": "original", "sha256": SHA_B,
"media_type": "pdf",
"added_at": "2026-07-01T00:00:00",
"status": "needs_ocr"},
}}), encoding="utf-8")
ref = f"raw/converted/good-src.md#{SHA_A[:8]}"
ref = f"{SRC_OK}#{SHA_A[:8]}"
good = {"type": "entity", "title": "支付閘道", "description": "支付閘道模組",
"tags": ["gateway"], "timestamp": today, "sources": [ref], "status": "draft"}
page(root, "wiki/entities/good.md", good)
@@ -90,7 +94,7 @@ def main():
assert code == 1, code
report = next((root / "reports").glob("lint-*.md")).read_text(encoding="utf-8")
for expected in ("缺欄位 tags", "source_ref 不在 manifest",
"old.md", "建議人工複審", "uncovered.md", "攝入缺口",
"old.md", "建議人工複審", "未覆蓋 來源.md", "攝入缺口",
"needs_ocr", "bad-schema.md` — 不被 index.md"):
assert expected in report, f"報告缺少：{expected}\n{report}"
assert "good.md" not in report

12
tools/selfcheck_search.py
View File

@@ -44,6 +44,10 @@ def main():
["aml", "compliance"], "大額交易與分散交易樣態的監控測試要求。")
page(root, "wiki/summaries/long-doc.md", "長文件摘要", "截斷測試用",
["test"], "逾時。" * 500)
# 與 payment-gateway 共用核心詞「支付閘道」→ df=2, N=4，正是 BM25Okapi
# 原式 idf 歸零的配置；沒有這頁就測不出核心詞落空的 bug
page(root, "wiki/summaries/gateway-report.md", "支付閘道版本沿革", "支付閘道改版紀錄",
["gateway"], "支付閘道於 2026 年改版，新增分期付款。")
hits = search.search(root, "支付閘道 逾時")
assert hits and hits[0]["path"] == "wiki/entities/payment-gateway.md", hits
@@ -53,6 +57,14 @@ def main():
assert search.search(root, "zzz-nonexistent-term") == []
print("PASS: BM25 檢索排序（繁中 bigram）")
# 核心詞（出現在多頁）不得因 idf <= 0 被整批濾掉
core = search.search(root, "支付閘道")
paths = {h["path"] for h in core}
assert {"wiki/entities/payment-gateway.md",
"wiki/summaries/gateway-report.md"} <= paths, core
assert all(h["score"] > 0 for h in core), core
print("PASS: 核心詞跨頁共用仍可檢索（idf 恆正）")
os.environ["PP_QA_ROOT"] = str(root)
import importlib
sys.path.insert(0, str(pathlib.Path(__file__).parents[1] / "mcp"))