修復中文路徑 preflight 與 BM25 核心詞落空,測資改照現實建
兩個 bug 同源於自檢測資「照能過建、不照現實建」: - ingest preflight:git status --porcelain 預設會把含中文或空格的路徑加引號 轉義,raw/ 白名單比對因此失效,中文檔名的 convert 產出被誤判為 raw/ 以外的 未提交變更而中止攝入。舊測資用 ASCII 檔名,整條路徑從未被走過。 - search:BM25Okapi 的 idf 在 df >= N/2 時 <= 0(rank_bm25 對負 idf 的替代值 epsilon x average_idf 在 average_idf 為負時同樣為負),與 search() 的 s > 0 過濾相乘,會讓「每頁都提到的核心詞」查詢全數落空——知識庫愈小、詞愈核心 愈嚴重,MCP 的 search_wiki 一併受害。改用 Lucene 式恆正 idf。 舊測資每個查詢詞都只出現在一頁(df=1),恰好避開此配置。 變更: - tools/ingest.py:preflight 比對前剝除 git 的轉義引號 - tools/search.py:_BM25 子類覆寫 _calc_idf 為 log(1 + (N-df+0.5)/(df+0.5)) - tools/selfcheck_ingest.py:新增 preflight 中文檔名放行 / 非 raw 擋下的檢查 - tools/selfcheck_search.py:新增與既有頁共用核心詞的測資,鎖住 idf 退化 - tools/convert/selfcheck.py、tools/selfcheck_lint.py:測資檔名改中文+空格 - AGENTS.md §13.4:新增「測資照現實建,不是照能過建」規則 - README.md §3.2:補 convert 產出應保持未提交、由 ingest 一併 commit 的流程 驗證:四個自檢全數通過;分別移除兩個修法後對應檢查會失敗。 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -23,6 +23,8 @@ limits: {mcp_response_max_tokens: 2000, ingest_chunk_max_chars: 12000}
|
||||
"""
|
||||
SHA_A = "abcd1234" + "0" * 56
|
||||
SHA_B = "beef5678" + "0" * 56
|
||||
SRC_OK = "raw/converted/好來源 報告.md"
|
||||
SRC_UNCOVERED = "raw/converted/未覆蓋 來源.md"
|
||||
|
||||
|
||||
def page(root, rel, meta, body="內文。"):
|
||||
@@ -36,20 +38,22 @@ def build(root):
|
||||
for sub in ("config", "raw/converted", "wiki/summaries", "wiki/entities", "wiki/concepts"):
|
||||
(root / sub).mkdir(parents=True)
|
||||
(root / "config/models.yaml").write_text(CFG, encoding="utf-8")
|
||||
# 來源檔名一律「中文 + 空格」——真實 QA 文件就長這樣(§12:測資要照現實建,不是照能過建)
|
||||
(root / "raw/manifest.json").write_text(json.dumps({"version": 1, "files": {
|
||||
"raw/converted/good-src.md": {"kind": "converted", "sha256": SHA_A,
|
||||
"original_path": "raw/originals/g.docx",
|
||||
"original_sha256": SHA_A, "converter": "from_docx",
|
||||
"converted_at": "2026-07-01T00:00:00"},
|
||||
"raw/converted/uncovered.md": {"kind": "converted", "sha256": SHA_B,
|
||||
"original_path": "raw/originals/u.docx",
|
||||
"original_sha256": SHA_B, "converter": "from_docx",
|
||||
"converted_at": "2026-07-01T00:00:00"},
|
||||
"raw/originals/scan.pdf": {"kind": "original", "sha256": SHA_B,
|
||||
"media_type": "pdf", "added_at": "2026-07-01T00:00:00",
|
||||
"status": "needs_ocr"},
|
||||
SRC_OK: {"kind": "converted", "sha256": SHA_A,
|
||||
"original_path": "raw/originals/好來源 報告.docx",
|
||||
"original_sha256": SHA_A, "converter": "from_docx",
|
||||
"converted_at": "2026-07-01T00:00:00"},
|
||||
SRC_UNCOVERED: {"kind": "converted", "sha256": SHA_B,
|
||||
"original_path": "raw/originals/未覆蓋 來源.docx",
|
||||
"original_sha256": SHA_B, "converter": "from_docx",
|
||||
"converted_at": "2026-07-01T00:00:00"},
|
||||
"raw/originals/掃描件 無文字層.pdf": {"kind": "original", "sha256": SHA_B,
|
||||
"media_type": "pdf",
|
||||
"added_at": "2026-07-01T00:00:00",
|
||||
"status": "needs_ocr"},
|
||||
}}), encoding="utf-8")
|
||||
ref = f"raw/converted/good-src.md#{SHA_A[:8]}"
|
||||
ref = f"{SRC_OK}#{SHA_A[:8]}"
|
||||
good = {"type": "entity", "title": "支付閘道", "description": "支付閘道模組",
|
||||
"tags": ["gateway"], "timestamp": today, "sources": [ref], "status": "draft"}
|
||||
page(root, "wiki/entities/good.md", good)
|
||||
@@ -90,7 +94,7 @@ def main():
|
||||
assert code == 1, code
|
||||
report = next((root / "reports").glob("lint-*.md")).read_text(encoding="utf-8")
|
||||
for expected in ("缺欄位 tags", "source_ref 不在 manifest",
|
||||
"old.md", "建議人工複審", "uncovered.md", "攝入缺口",
|
||||
"old.md", "建議人工複審", "未覆蓋 來源.md", "攝入缺口",
|
||||
"needs_ocr", "bad-schema.md` — 不被 index.md"):
|
||||
assert expected in report, f"報告缺少:{expected}\n{report}"
|
||||
assert "good.md" not in report
|
||||
|
||||
Reference in New Issue
Block a user