From 0b4c2bbb15279c7fcc020a4732f5fdd6f0e6b03b Mon Sep 17 00:00:00 2001 From: LittleYellow Date: Wed, 22 Jul 2026 07:21:45 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=BE=A9=E4=B8=AD=E6=96=87=E8=B7=AF?= =?UTF-8?q?=E5=BE=91=20preflight=20=E8=88=87=20BM25=20=E6=A0=B8=E5=BF=83?= =?UTF-8?q?=E8=A9=9E=E8=90=BD=E7=A9=BA=EF=BC=8C=E6=B8=AC=E8=B3=87=E6=94=B9?= =?UTF-8?q?=E7=85=A7=E7=8F=BE=E5=AF=A6=E5=BB=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 兩個 bug 同源於自檢測資「照能過建、不照現實建」： - ingest preflight：git status --porcelain 預設會把含中文或空格的路徑加引號 轉義，raw/ 白名單比對因此失效，中文檔名的 convert 產出被誤判為 raw/ 以外的 未提交變更而中止攝入。舊測資用 ASCII 檔名，整條路徑從未被走過。 - search：BM25Okapi 的 idf 在 df >= N/2 時 <= 0（rank_bm25 對負 idf 的替代值 epsilon x average_idf 在 average_idf 為負時同樣為負），與 search() 的 s > 0 過濾相乘，會讓「每頁都提到的核心詞」查詢全數落空——知識庫愈小、詞愈核心 愈嚴重，MCP 的 search_wiki 一併受害。改用 Lucene 式恆正 idf。 舊測資每個查詢詞都只出現在一頁（df=1），恰好避開此配置。 變更： - tools/ingest.py：preflight 比對前剝除 git 的轉義引號 - tools/search.py：_BM25 子類覆寫 _calc_idf 為 log(1 + (N-df+0.5)/(df+0.5)) - tools/selfcheck_ingest.py：新增 preflight 中文檔名放行 / 非 raw 擋下的檢查 - tools/selfcheck_search.py：新增與既有頁共用核心詞的測資，鎖住 idf 退化 - tools/convert/selfcheck.py、tools/selfcheck_lint.py：測資檔名改中文＋空格 - AGENTS.md §13.4：新增「測資照現實建，不是照能過建」規則 - README.md §3.2：補 convert 產出應保持未提交、由 ingest 一併 commit 的流程 驗證：四個自檢全數通過；分別移除兩個修法後對應檢查會失敗。 Co-Authored-By: Claude Opus 4.8 --- AGENTS.md | 4 +++ README.md | 7 +++++ tools/convert/selfcheck.py | 62 +++++++++++++++++++++----------------- tools/ingest.py | 4 ++- tools/search.py | 17 ++++++++++- tools/selfcheck_ingest.py | 15 ++++++++- tools/selfcheck_lint.py | 30 ++++++++++-------- tools/selfcheck_search.py | 12 ++++++++ 8 files changed, 108 insertions(+), 43 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index fdf768f..5aee872 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -272,6 +272,10 @@ output**）。單筆修正只治標；規則修正才治本。 （global lock、O(n²) 掃描、naive heuristic），註解要寫明天花板與升級路徑。 - 非平凡邏輯要留下**一個**可執行的檢查——邏輯壞掉就會失敗的最小東西 （assert 式自檢或一個小測試檔；不用框架、不用 fixtures）。 +- **測資照現實建，不是照「能過」建**：來源檔名用中文＋空格（真實 QA 文件就長這樣）、 + 語料要有跨頁共用的核心詞。便利值會變成盲點的形狀——ASCII 檔名讓 `git status` + 的路徑轉義 bug 溜過，互不重疊的語料讓 BM25 的 idf 退化 bug 溜過。 + 新增檢查時先問：**這組測資和真實輸入差在哪？差異處就是沒被測到的地方。** - 把任務轉成可驗證的目標，迭代到通過： - 「加驗證」→ 先寫無效輸入的測試，再讓它通過。 - 「修 bug」→ 先寫重現 bug 的測試，再讓它通過。 diff --git a/README.md b/README.md index 47e0f92..ae2dda8 100644 --- a/README.md +++ b/README.md @@ -99,6 +99,13 @@ ollama list # 對照 tasks.ingest.model / tasks.lint.model / tasks.fallba **前置條件**：須在 `main` 分支、且工作區除 `raw/` 外無未提交變更，否則會報錯中止。 +> **`convert` 的產出請保持未提交，不要另開分支 commit。** 除 `raw/` 外無未提交變更的條件 +> 刻意把 `raw/` 排除在外——正確流程是在 `main` 上 `convert`、確認 `raw/converted/*.md` 內容 +> 無誤後直接 `ingest`，由 ingest 把來源與編譯出的 wiki 頁**一起** commit 進同一個分支。這樣 +> PR 的 diff 同時含來源與產出，審核者才能對照 LLM 有無編譯錯誤。若先開分支 commit 了轉換檔， +> 切回 `main` 時那些檔案會隨分支離開工作區；此時請先 `git merge` 該分支回 `main` 再攝入 +> （`raw/` 不受 §1.5 HITL 閘門管轄，該條只規範 wiki 層變更）。 + 執行後：變更落在 `ingest/<時間戳>-` 分支。有 remote 時自動 push 並印出 PR 建立網址； 無 remote / 離線時分支保留本地，請手動 push 後開 PR。**經人工審核合併後才進 main。** diff --git a/tools/convert/selfcheck.py b/tools/convert/selfcheck.py index 43601e9..ad84361 100644 --- a/tools/convert/selfcheck.py +++ b/tools/convert/selfcheck.py @@ -24,6 +24,14 @@ import kb import to_image +# 測資檔名一律用「中文 + 空格」——真實來源檔就長這樣，而 git status --porcelain +# 對這兩者都會加引號轉義。ASCII 檔名的測資曾讓 ingest 的 preflight bug 溜過（§12）。 +DOCX, XLSX, PDF, SCAN = ("壓測 報告.docx", "測試案例 清單.xlsx", + "規格 說明.pdf", "掃描件 無文字層.pdf") +PPTX, HTML, TXT, DOC = ("結案 簡報.pptx", "指引 快照.html", + "不支援 筆記.txt", "舊版 報告.doc") + + def make_fixtures(d): import docx as docxlib doc = docxlib.Document() @@ -34,7 +42,7 @@ def make_fixtures(d): t.rows[0].cells[1].text = "結果" t.rows[1].cells[0].text = "TC-001" t.rows[1].cells[1].text = "FAIL" - doc.save(d / "report.docx") + doc.save(d / DOCX) import openpyxl wb = openpyxl.Workbook() @@ -42,17 +50,17 @@ def make_fixtures(d): ws.title = "測項" ws.append(["編號", "說明"]) ws.append(["TC-001", "逾時 | 重試"]) - wb.save(d / "cases.xlsx") + wb.save(d / XLSX) import fitz pdf = fitz.open() page = pdf.new_page() page.insert_text((72, 72), "Payment gateway timeout defect reproduced under load test.") - pdf.save(d / "text.pdf") + pdf.save(d / PDF) pdf.close() scanned = fitz.open() scanned.new_page() # 無文字層 → 應判 needs_ocr - scanned.save(d / "scanned.pdf") + scanned.save(d / SCAN) scanned.close() from pptx import Presentation @@ -60,9 +68,9 @@ def make_fixtures(d): slide = prs.slides.add_slide(prs.slide_layouts[1]) slide.shapes.title.text = "結案報告" slide.notes_slide.notes_text_frame.text = "備註：法遵項目全數通過" - prs.save(d / "closing.pptx") + prs.save(d / PPTX) - (d / "page.html").write_text( + (d / HTML).write_text( "

AML 測試指引

" + "

可疑交易監控測試需涵蓋大額交易與分散交易兩種樣態。

" * 8 + "
", encoding="utf-8") @@ -75,17 +83,17 @@ def main(): fx.mkdir() make_fixtures(fx) - md, st = from_docx.convert(fx / "report.docx") + md, st = from_docx.convert(fx / DOCX) assert st == "ok" and "# 支付閘道測試報告" in md and "| TC-001 | FAIL |" in md, md - md, st = from_xlsx.convert(fx / "cases.xlsx") + md, st = from_xlsx.convert(fx / XLSX) assert st == "ok" and "## 工作表：測項" in md and "逾時 \\| 重試" in md, md - md, st = from_pdf.convert(fx / "text.pdf") + md, st = from_pdf.convert(fx / PDF) assert st == "ok" and "" in md and "timeout defect" in md, md - _, st = from_pdf.convert(fx / "scanned.pdf") + _, st = from_pdf.convert(fx / SCAN) assert st == "needs_ocr", st - md, st = from_pptx.convert(fx / "closing.pptx") + md, st = from_pptx.convert(fx / PPTX) assert st == "ok" and "## Slide 1" in md and "法遵項目全數通過" in md, md - md, st = from_web.extract((fx / "page.html").read_text(encoding="utf-8")) + md, st = from_web.extract((fx / HTML).read_text(encoding="utf-8")) assert st == "ok" and "可疑交易監控" in md, md print("PASS: 5 個轉換器 + needs_ocr 偵測") @@ -95,22 +103,22 @@ def main(): (root / sub).mkdir(parents=True) (root / "raw/manifest.json").write_text('{"version": 1, "files": {}}', encoding="utf-8") - convert.main([str(fx / "report.docx"), str(fx / "scanned.pdf"), - str(fx / "page.html"), "--root", str(root)]) + convert.main([str(fx / DOCX), str(fx / SCAN), + str(fx / HTML), "--root", str(root)]) m = json.loads((root / "raw/manifest.json").read_text(encoding="utf-8")) origs = {k: v for k, v in m["files"].items() if v["kind"] == "original"} convs = {k: v for k, v in m["files"].items() if v["kind"] == "converted"} assert len(origs) == 3 and len(convs) == 2, m - assert origs["raw/originals/scanned.pdf"]["status"] == "needs_ocr" + assert origs[f"raw/originals/{SCAN}"]["status"] == "needs_ocr" for k, v in convs.items(): assert (root / k).is_file(), k assert m["files"][v["original_path"]]["sha256"] == v["original_sha256"] - assert (root / "raw/originals/report.docx").is_file() + assert (root / f"raw/originals/{DOCX}").is_file() print("PASS: convert.py 端到端 + manifest 對應") # 冪等：同檔再跑一次 → skip，manifest 不變 before = (root / "raw/manifest.json").read_text(encoding="utf-8") - convert.main([str(fx / "report.docx"), "--root", str(root)]) + convert.main([str(fx / DOCX), "--root", str(root)]) assert (root / "raw/manifest.json").read_text(encoding="utf-8") == before print("PASS: 冪等（同 hash 跳過）") @@ -119,7 +127,7 @@ def main(): for sub in ("raw/originals", "raw/converted"): (root2 / sub).mkdir(parents=True) (root2 / "raw/manifest.json").write_text('{"version": 1, "files": {}}', encoding="utf-8") - convert.main([str(fx / "report.docx"), "--dry-run", "--root", str(root2)]) + convert.main([str(fx / DOCX), "--dry-run", "--root", str(root2)]) m2 = json.loads((root2 / "raw/manifest.json").read_text(encoding="utf-8")) assert m2["files"] == {} and not list((root2 / "raw/originals").iterdir()) print("PASS: --dry-run 不落地") @@ -129,13 +137,13 @@ def main(): for sub in ("raw/originals", "raw/converted"): (root3 / sub).mkdir(parents=True) (root3 / "raw/manifest.json").write_text('{"version": 1, "files": {}}', encoding="utf-8") - (fx / "notes.txt").write_text("不支援的格式", encoding="utf-8") # → ValueError + (fx / TXT).write_text("不支援的格式", encoding="utf-8") # → ValueError err = io.StringIO() code = None with contextlib.redirect_stderr(err): try: - convert.main([str(fx / "report.docx"), str(fx / "notes.txt"), - str(fx / "missing.docx"), # 不存在 → FileNotFoundError + convert.main([str(fx / DOCX), str(fx / TXT), + str(fx / "不存在 檔案.docx"), # 不存在 → FileNotFoundError "--root", str(root3)]) except SystemExit as e: code = e.code @@ -144,7 +152,7 @@ def main(): assert "ValueError" in log and "FileNotFoundError" in log, log # 錯誤帶例外型別 assert "2 個項目轉換失敗" in log, log # 尾端彙總 m3 = json.loads((root3 / "raw/manifest.json").read_text(encoding="utf-8")) - assert (root3 / "raw/originals/report.docx").is_file() # 好檔仍轉換並落地 + assert (root3 / f"raw/originals/{DOCX}").is_file() # 好檔仍轉換並落地 assert any(v["kind"] == "converted" for v in m3["files"].values()), m3 print("PASS: 失敗回報（例外型別 + 尾端彙總，批次不中斷）") @@ -153,21 +161,21 @@ def main(): for sub in ("raw/originals", "raw/converted"): (root4 / sub).mkdir(parents=True) (root4 / "raw/manifest.json").write_text('{"version": 1, "files": {}}', encoding="utf-8") - (fx / "old.doc").write_bytes(b"\xd0\xcf\x11\xe0legacy-stub") # 非有效 .doc，僅測路由/錯誤 + (fx / DOC).write_bytes(b"\xd0\xcf\x11\xe0legacy-stub") # 非有效 .doc，僅測路由/錯誤 err = io.StringIO() code = None with contextlib.redirect_stderr(err): try: - convert.main([str(fx / "old.doc"), "--root", str(root4)]) + convert.main([str(fx / DOC), "--root", str(root4)]) except SystemExit as e: code = e.code log = err.getvalue() - assert code == 1 and "old.doc" in log, log + assert code == 1 and DOC in log, log assert "不支援的格式" not in log and "RuntimeError" in log, log # 已路由 legacy，乾淨報錯 print("PASS: 舊格式路由至 legacy（soffice 缺失/升版失敗皆乾淨報錯）") # A: to_image render（deterministic，真跑） - imgs = to_image.render_pdf_pages(fx / "text.pdf", tmp / "imgs") + imgs = to_image.render_pdf_pages(fx / PDF, tmp / "imgs") assert imgs and all(p.is_file() and p.stat().st_size > 0 for p in imgs), imgs print(f"PASS: to_image render（{len(imgs)} 頁 PNG）") @@ -212,7 +220,7 @@ def main(): (root_v / "raw/manifest.json").write_text('{"version": 1, "files": {}}', encoding="utf-8") kb.ollama_chat = fake_chat try: - convert.main([str(fx / "scanned.pdf"), "--vision", "--root", str(root_v)]) + convert.main([str(fx / SCAN), "--vision", "--root", str(root_v)]) finally: kb.ollama_chat = orig_chat mv = json.loads((root_v / "raw/manifest.json").read_text(encoding="utf-8")) diff --git a/tools/ingest.py b/tools/ingest.py index f9ec135..e065bf8 100644 --- a/tools/ingest.py +++ b/tools/ingest.py @@ -191,8 +191,10 @@ def git_preflight(root): branch = run_git(root, "branch", "--show-current", capture=True).stdout.strip() if branch != "main": raise SystemExit(f"須在 main 分支執行（目前：{branch}）") + # git 預設 core.quotepath=true，含非 ASCII（中文檔名）或空格的路徑會被引號包住並轉義， + # 故比對前先剝除開頭引號，否則 convert 產出的中文檔名會被誤判為 raw/ 以外的變更。 dirty = [l for l in run_git(root, "status", "--porcelain", capture=True) - .stdout.splitlines() if l.strip() and not l[3:].startswith("raw/")] + .stdout.splitlines() if l.strip() and not l[3:].lstrip('"').startswith("raw/")] if dirty: raise SystemExit("工作區有 raw/ 以外的未提交變更，請先處理：\n" + "\n".join(dirty)) diff --git a/tools/search.py b/tools/search.py index f8d7a4c..0b6cb46 100644 --- a/tools/search.py +++ b/tools/search.py @@ -5,6 +5,7 @@ """ import argparse import json +import math import pathlib import re import sys @@ -27,6 +28,20 @@ def tokenize(text): return tokens +class _BM25(BM25Okapi): + """改用 Lucene 式 idf：log(1 + (N-df+0.5)/(df+0.5))，恆為正。 + + BM25Okapi 原式在 df >= N/2 時 idf <= 0（rank_bm25 對負 idf 的替代值 + epsilon × average_idf 在 average_idf 為負時同樣是負的），與 search() 的 + `s > 0` 過濾相乘，會讓「每頁都提到的核心詞」查詢全數落空——知識庫愈小、 + 詞愈核心愈嚴重。恆正 idf 讓常見詞只是權重低，而不是被整批濾掉。 + """ + + def _calc_idf(self, nd): + for word, freq in nd.items(): + self.idf[word] = math.log(1 + (self.corpus_size - freq + 0.5) / (freq + 0.5)) + + def build_corpus(root): docs = [] for p in kb.iter_pages(root): @@ -47,7 +62,7 @@ def search(root, query, k=10): docs = build_corpus(pathlib.Path(root)) if not docs: return [] - bm = BM25Okapi([d["tokens"] for d in docs]) + bm = _BM25([d["tokens"] for d in docs]) scores = bm.get_scores(tokenize(query)) ranked = sorted(zip(docs, scores), key=lambda x: -x[1])[:k] return [{"path": d["path"], "title": d["title"], "description": d["description"], diff --git a/tools/selfcheck_ingest.py b/tools/selfcheck_ingest.py index b2914f2..514fa4c 100644 --- a/tools/selfcheck_ingest.py +++ b/tools/selfcheck_ingest.py @@ -117,7 +117,20 @@ def main(): git(root, "checkout", "-q", "main") print("PASS: 冪等跳過 + 既有 entity 頁就地編輯（sources 追加）") - # 3) kb.llm_json：重試後切 fallback；全失敗拋錯 + # 3) preflight：raw/ 下未提交的 convert 產出（含中文檔名）不得被誤判為髒污 + (root / "raw" / "converted" / "測試報告.md").write_text("中文", encoding="utf-8") + ingest.git_preflight(root) # 不應拋錯 + stray = root / "wiki" / "concepts" / "stray.md" + stray.write_text("x", encoding="utf-8") + try: + ingest.git_preflight(root) + raise AssertionError("raw/ 以外的未提交變更應中止") + except SystemExit: + pass + stray.unlink() + print("PASS: preflight 放行 raw/（含中文檔名）、擋下 raw/ 以外的未提交變更") + + # 4) kb.llm_json：重試後切 fallback；全失敗拋錯 kb.llm_json = real_llm cfg = kb.load_config(root) calls = {"n": 0} diff --git a/tools/selfcheck_lint.py b/tools/selfcheck_lint.py index fd484b4..45d8736 100644 --- a/tools/selfcheck_lint.py +++ b/tools/selfcheck_lint.py @@ -23,6 +23,8 @@ limits: {mcp_response_max_tokens: 2000, ingest_chunk_max_chars: 12000} """ SHA_A = "abcd1234" + "0" * 56 SHA_B = "beef5678" + "0" * 56 +SRC_OK = "raw/converted/好來源 報告.md" +SRC_UNCOVERED = "raw/converted/未覆蓋 來源.md" def page(root, rel, meta, body="內文。"): @@ -36,20 +38,22 @@ def build(root): for sub in ("config", "raw/converted", "wiki/summaries", "wiki/entities", "wiki/concepts"): (root / sub).mkdir(parents=True) (root / "config/models.yaml").write_text(CFG, encoding="utf-8") + # 來源檔名一律「中文 + 空格」——真實 QA 文件就長這樣（§12：測資要照現實建，不是照能過建） (root / "raw/manifest.json").write_text(json.dumps({"version": 1, "files": { - "raw/converted/good-src.md": {"kind": "converted", "sha256": SHA_A, - "original_path": "raw/originals/g.docx", - "original_sha256": SHA_A, "converter": "from_docx", - "converted_at": "2026-07-01T00:00:00"}, - "raw/converted/uncovered.md": {"kind": "converted", "sha256": SHA_B, - "original_path": "raw/originals/u.docx", - "original_sha256": SHA_B, "converter": "from_docx", - "converted_at": "2026-07-01T00:00:00"}, - "raw/originals/scan.pdf": {"kind": "original", "sha256": SHA_B, - "media_type": "pdf", "added_at": "2026-07-01T00:00:00", - "status": "needs_ocr"}, + SRC_OK: {"kind": "converted", "sha256": SHA_A, + "original_path": "raw/originals/好來源 報告.docx", + "original_sha256": SHA_A, "converter": "from_docx", + "converted_at": "2026-07-01T00:00:00"}, + SRC_UNCOVERED: {"kind": "converted", "sha256": SHA_B, + "original_path": "raw/originals/未覆蓋 來源.docx", + "original_sha256": SHA_B, "converter": "from_docx", + "converted_at": "2026-07-01T00:00:00"}, + "raw/originals/掃描件 無文字層.pdf": {"kind": "original", "sha256": SHA_B, + "media_type": "pdf", + "added_at": "2026-07-01T00:00:00", + "status": "needs_ocr"}, }}), encoding="utf-8") - ref = f"raw/converted/good-src.md#{SHA_A[:8]}" + ref = f"{SRC_OK}#{SHA_A[:8]}" good = {"type": "entity", "title": "支付閘道", "description": "支付閘道模組", "tags": ["gateway"], "timestamp": today, "sources": [ref], "status": "draft"} page(root, "wiki/entities/good.md", good) @@ -90,7 +94,7 @@ def main(): assert code == 1, code report = next((root / "reports").glob("lint-*.md")).read_text(encoding="utf-8") for expected in ("缺欄位 tags", "source_ref 不在 manifest", - "old.md", "建議人工複審", "uncovered.md", "攝入缺口", + "old.md", "建議人工複審", "未覆蓋 來源.md", "攝入缺口", "needs_ocr", "bad-schema.md` — 不被 index.md"): assert expected in report, f"報告缺少：{expected}\n{report}" assert "good.md" not in report diff --git a/tools/selfcheck_search.py b/tools/selfcheck_search.py index 12182c7..5e5d39c 100644 --- a/tools/selfcheck_search.py +++ b/tools/selfcheck_search.py @@ -44,6 +44,10 @@ def main(): ["aml", "compliance"], "大額交易與分散交易樣態的監控測試要求。") page(root, "wiki/summaries/long-doc.md", "長文件摘要", "截斷測試用", ["test"], "逾時。" * 500) + # 與 payment-gateway 共用核心詞「支付閘道」→ df=2, N=4，正是 BM25Okapi + # 原式 idf 歸零的配置；沒有這頁就測不出核心詞落空的 bug + page(root, "wiki/summaries/gateway-report.md", "支付閘道版本沿革", "支付閘道改版紀錄", + ["gateway"], "支付閘道於 2026 年改版，新增分期付款。") hits = search.search(root, "支付閘道 逾時") assert hits and hits[0]["path"] == "wiki/entities/payment-gateway.md", hits @@ -53,6 +57,14 @@ def main(): assert search.search(root, "zzz-nonexistent-term") == [] print("PASS: BM25 檢索排序（繁中 bigram）") + # 核心詞（出現在多頁）不得因 idf <= 0 被整批濾掉 + core = search.search(root, "支付閘道") + paths = {h["path"] for h in core} + assert {"wiki/entities/payment-gateway.md", + "wiki/summaries/gateway-report.md"} <= paths, core + assert all(h["score"] > 0 for h in core), core + print("PASS: 核心詞跨頁共用仍可檢索（idf 恆正）") + os.environ["PP_QA_ROOT"] = str(root) import importlib sys.path.insert(0, str(pathlib.Path(__file__).parents[1] / "mcp"))