yellowadmin/pp-qa-km
1
0
Fork 0
Code Issues 5 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: diag_refs 對帳診斷 + ingest 進度可見性/Ctrl-C 安全 (Closes #2) (#5)

Browse Source 
## 摘要

本 PR 含兩個獨立關注點,各一 commit。

### 1. `diag_refs` — source_ref↔manifest 對帳診斷 (`04f4fd5`) — 回應 #4

lint 的「source_ref 不在 manifest」只報對不上、不辨成因。新增 `tools/diag_refs.py`,把每個對不上的 source_ref 分成五類(hash 不符／路徑字串岔開／檔名 hash 後綴岔開／檔在磁碟未登記／完全無對應),各附修法,讓「補帳本(§14.3)」與「修規則(§12)」的相反處置不再混為一談。純唯讀、結束碼 `0/1`,可當搬機或改 manifest 後的閘門。附 selfcheck 驗五型別分類、結束碼與零寫檔。README §3.3/§6/§7/§8 同步。

### 2. ingest 進度可見性 + Ctrl-C 安全回復 (`4add7fa`) — Closes #2

`--all-pending` 長時間本地 Ollama 攝入全程靜默,使用者不確定有沒有在跑、又不敢中斷:

- **進度可見**:每份來源 `[i/N]`、長文件逐段 `分段 k/n`、每個 item `↳ 新增/更新`、完成 `✓`,全 `flush=True` 即時顯示。
- **Ctrl-C 安全**:獨立攔 `KeyboardInterrupt`(它不是 `Exception` 子類,原本會略過善後)。因 commit 在迴圈之後,**main 必然未受影響**;中斷時印可照做的回復指令並以結束碼 130 收場。

## 測試

- `python tools/selfcheck_diag_refs.py` — ALL PASS
- `python tools/selfcheck_ingest.py` — ALL PASS(含進度標記、中斷後 main 未動、回復指令實測可回乾淨 main)
- README markdownlint 0 issues

Closes #2

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-authored-by: LittleYellow <crazytea@gmail.com>
Reviewed-on: #5
This commit was merged in pull request #5.
This commit is contained in:
yellowadmin
2026-07-24 05:42:35 +00:00
parent 356ae9367c
commit b90727fc11
5 changed files with 334 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

134
tools/diag_refs.py Normal file
View File

@@ -0,0 +1,134 @@
"""source_ref ↔ manifest 對帳診斷（AGENTS.md §7 schema / §14 對帳的輔助工具）。
lint.py 對「source_ref 不在 manifest」只報一句，無法區分成因；本工具把每個
對不上的 source_ref 分類成可據以修復的型別（見 CAT_TITLE / FIX），讓「帳本掉
條目」「路徑字串岔開」「檔名 hash 後綴岔開」「檔真的遺失」彼此不再混為一談。
純唯讀——不改任何檔（含 manifest），有對不上以結束碼 1 表示（可當搬機／改
manifest 後的閘門）。分類邏輯與 lint 一致：以最後一個 '#' 切出 path 與 sha8。
用法：python tools/diag_refs.py [--root DIR]
結束碼：0 = 全部解得開，1 = 有對不上。
"""
import argparse
import json
import pathlib
import re
import sys
sys.path.insert(0, str(pathlib.Path(__file__).parent))
import kb
CAT_TITLE = {
"hash-mismatch": "Hash 不符（帳本有此路徑，但 sha256 與 ref 不同）",
"normalize": "路徑字串岔開（正規化後與帳本 key 相同：斜線／前綴／大小寫）",
"stem": "檔名 hash 後綴岔開（帳本有同檔名幹、hash 後綴不同的 key）",
"unregistered": "檔在磁碟、帳本無此條目",
"absent": "完全無對應（帳本、磁碟、檔名幹皆無）",
}
FIX = {
"hash-mismatch": "raw 被就地改動或重轉（違反 §1.4）；還原檔案，或連同 source_ref 走 PR（§14.2）。",
"normalize": "修產生岔開字串的那一個環節（路徑正規化），別逐頁改 source_ref（§12 change the ruler）。",
"stem": "source_ref 指到不同 hash 後綴的檔名；核對正確檔名，別改帳本去遷就（§14.2）。",
"unregistered": "帳本掉了條目、檔還在；走 branch+PR 補登 manifest（§14.3），別動 wiki 頁。",
"absent": "source_ref 可能攝入時寫歪，或原始檔真的遺失；上報 HITL 據實記錄（§14.2）。",
}
# 分類輸出順序（愈可機械修復的排愈前）
ORDER = ["hash-mismatch", "normalize", "stem", "unregistered", "absent"]
def _norm(path):
"""正規化路徑字串以偵測「同一檔、不同寫法」：反斜線→斜線、去開頭 ./、轉小寫。"""
s = path.replace("\\", "/")
if s.startswith("./"):
s = s[2:]
return s.lower()
def _stem(path):
"""取檔名幹：去目錄、去 .md、去結尾的 -<8 碼 hex>（convert 的檔名 hash 後綴）。"""
name = path.replace("\\", "/").rsplit("/", 1)[-1]
name = re.sub(r"\.md$", "", name, flags=re.I)
return re.sub(r"-[0-9a-f]{8}$", "", name, flags=re.I).lower()
def classify_ref(source_ref, files, exists):
"""把單一 source_ref 分類。files：manifest['files']（path→entry）；
exists(path)→bool 判磁碟是否有該檔。回傳 (category, detail)。
category 為 "resolved" 或 CAT_TITLE 的任一鍵。"""
path, _, sha8 = str(source_ref).rpartition("#")
entry = files.get(path)
if entry is not None:
if entry["sha256"].startswith(sha8):
return "resolved", path
return "hash-mismatch", f"帳本 sha256={entry['sha256'][:8]} ≠ ref #{sha8}"
norm = _norm(path)
for k in files:
if _norm(k) == norm:
return "normalize", f"帳本 key = '{k}'"
stem = _stem(path)
for k, e in files.items():
if e.get("kind") == "converted" and _stem(k) == stem:
return "stem", f"帳本 key = '{k}'"
if exists(path):
return "unregistered", f"磁碟有 '{path}'，帳本無此 key"
return "absent", f"'{path}' 帳本無、磁碟無、無同名幹 key"
def collect(root, files, exists):
"""掃全庫 wiki 頁，回傳 (findings, n_refs)。findings：category→[(page, ref, detail)]。"""
findings = {c: [] for c in ORDER}
n_refs = 0
for p in kb.iter_pages(root):
rel = p.relative_to(root).as_posix()
try:
meta, _ = kb.parse_page(p.read_text(encoding="utf-8"))
except Exception as e:
findings["absent"].append((rel, "(無法解析頁面)", str(e)))
continue
for s in meta.get("sources") or []:
n_refs += 1
cat, detail = classify_ref(s, files, exists)
if cat != "resolved":
findings[cat].append((rel, str(s), detail))
return findings, n_refs
def format_report(findings, n_refs):
total = sum(len(v) for v in findings.values())
lines = [f"檢查 source_ref：{n_refs}，對不上：{total}", ""]
if total == 0:
lines.append("全部解得開（source_ref 與 manifest 一致）。")
return "\n".join(lines)
for cat in ORDER:
items = findings[cat]
if not items:
continue
lines.append(f"## {CAT_TITLE[cat]}（{len(items)}）")
lines.append(f"→ 修法：{FIX[cat]}")
for page, ref, detail in items:
lines.append(f" - [{page}] {ref}")
lines.append(f" {detail}")
lines.append("")
return "\n".join(lines)
def main(argv=None):
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument("--root", default=str(kb.ROOT))
a = ap.parse_args(argv)
root = pathlib.Path(a.root).resolve()
mpath = root / "raw" / "manifest.json"
try:
files = json.loads(mpath.read_text(encoding="utf-8"))["files"]
except (OSError, KeyError, json.JSONDecodeError) as e:
raise SystemExit(f"讀不到 / 解析不了 manifest：{mpath}（{e}）")
findings, n_refs = collect(root, files, lambda p: (root / p).is_file())
print(format_report(findings, n_refs))
if sum(len(v) for v in findings.values()):
raise SystemExit(1)
if __name__ == "__main__":
main()

31
tools/ingest.py
View File

@@ -123,6 +123,8 @@ def summarize(cfg, filename, text):
parts = split_chunks(text, limit)
partials = []
for i, part in enumerate(parts, 1):
# 長文件的分段迴圈是最容易「長時間靜默」的地方（issue #2）：逐段回報進度
print(f" 分段 {i}/{len(parts)} …", flush=True)
obj, _ = kb.llm_json(cfg, "ingest",
CHUNK_PROMPT.format(i=i, n=len(parts), content=part),
validate_chunk)
@@ -170,14 +172,16 @@ def upsert_item(root, cfg, item, source_ref, today, changed):
if source_ref not in meta["sources"]:
meta["sources"].append(source_ref)
existing.write_text(kb.dump_page(meta, obj["updated_markdown"]), encoding="utf-8")
changed.append(f"更新 {existing.relative_to(root).as_posix()}")
msg = f"更新 {existing.relative_to(root).as_posix()}"
else:
p = root / kb.PAGE_DIRS[item["type"]] / f"{slug}.md"
meta = {"type": item["type"], "title": item["title"],
"description": item["description"], "tags": item["tags"],
"timestamp": today, "sources": [source_ref], "status": "draft"}
p.write_text(kb.dump_page(meta, item["facts_markdown"]), encoding="utf-8")
changed.append(f"新增 {p.relative_to(root).as_posix()}")
msg = f"新增 {p.relative_to(root).as_posix()}"
changed.append(msg)
print(f" ↳ {msg}", flush=True)
# ---------- git ----------
@@ -240,7 +244,7 @@ def main(argv=None):
skipped = [t for t in targets if conv[t]["sha256"][:8] in done]
targets = [t for t in targets if conv[t]["sha256"][:8] not in done]
for t in skipped:
print(f"skip: {t} 已有 summary 引用（--force 可重跑）")
print(f"skip: {t} 已有 summary 引用（--force 可重跑）", flush=True)
if not targets:
print("沒有待攝入的來源。")
return
@@ -261,21 +265,24 @@ def main(argv=None):
run_git(root, "checkout", "-q", "-b", branch)
ok, failed, changed = [], [], []
try:
for t in targets:
for i, t in enumerate(targets, 1):
try:
print(f"[{i}/{len(targets)}] 攝入 {t} …", flush=True)
text = (root / t).read_text(encoding="utf-8")
sha8 = conv[t]["sha256"][:8]
source_ref = f"{t}#{sha8}"
obj, model = summarize(cfg, pathlib.Path(conv[t]["original_path"]).name, text)
p = write_summary_page(root, obj, source_ref, sha8, today)
changed.append(f"新增 {p.relative_to(root).as_posix()}")
msg = f"新增 {p.relative_to(root).as_posix()}"
changed.append(msg)
print(f" ↳ {msg}", flush=True)
for item in obj["knowledge_items"]:
upsert_item(root, cfg, item, source_ref, today, changed)
ok.append((t, model))
print(f"ingested: {t}（model={model}）")
print(f" ✓ {t} 完成（model={model}）", flush=True)
except Exception as e:
failed.append((t, e))
print(f"error: {t}: {e}", file=sys.stderr)
print(f" ✗ {t}: {e}", file=sys.stderr, flush=True)
if not ok:
run_git(root, "checkout", "-q", "main")
run_git(root, "branch", "-q", "-D", branch)
@@ -297,6 +304,16 @@ def main(argv=None):
print(f"完成：{len(ok)} 成功、{len(failed)} 失敗。變更在分支 {branch}，經 PR 審核後合併。")
except SystemExit:
raise
except KeyboardInterrupt:
# KeyboardInterrupt 不是 Exception 子類，會略過下方善後——故獨立攔截。
# 中斷時尚未 commit（commit 在迴圈之後），main 必然未受影響；給一條可照做的回復路徑。
print(f"\n已中斷（Ctrl-C）。main 未受影響——本次半成品都在分支 {branch}、尚未 commit。\n"
f"回到乾淨的 main：\n"
f" git checkout main && git stash -u && git branch -D {branch}\n"
f"（git stash -u 收起分支上未提交的半成品含未追蹤頁面；確定不要再 git stash drop）\n"
f"續跑 python tools/ingest.py --all-pending 會自動從尚未攝入的來源接續。",
file=sys.stderr, flush=True)
raise SystemExit(130)
except Exception:
print(f"攝入中斷。目前在分支 {branch}，工作區可能有未提交變更，"
"請人工檢查（不自動清除以免遺失資料）。", file=sys.stderr)

114
tools/selfcheck_diag_refs.py Normal file
View File

@@ -0,0 +1,114 @@
"""Phase 4 自檢：沙盒植入五種 source_ref 岔開型別（hash 不符、路徑正規化、
檔名 hash 後綴岔開、檔在磁碟未登記、完全無對應）＋一個正常解得開的 ref，
驗證 diag_refs 逐一分類正確、報告涵蓋各型別、結束碼 1、且不修改任何檔案。
另驗全部解得開時結束碼 0。
執行：.venv/Scripts/python tools/selfcheck_diag_refs.py
"""
import io
import json
import pathlib
import shutil
import sys
import tempfile
from contextlib import redirect_stdout
sys.path.insert(0, str(pathlib.Path(__file__).parent))
import kb
import diag_refs
SHA_A = "abcd1234" + "0" * 56
SHA_B = "beef5678" + "0" * 56
# 帳本 key 一律 posix 正斜線＋中文/連字號檔名（真實 convert 產出的形狀）
K_OK = "raw/converted/www-pluspay-faq-06b02fd9.md"
K_STEM = "raw/converted/www-pluspay-terms-11112222.md" # 同幹、不同 hash 後綴
FILES = {
K_OK: {"kind": "converted", "sha256": SHA_A, "original_path": "raw/originals/faq.html",
"original_sha256": SHA_A, "converter": "from_web", "converted_at": "2026-07-01T00:00:00"},
K_STEM: {"kind": "converted", "sha256": SHA_B, "original_path": "raw/originals/terms.html",
"original_sha256": SHA_B, "converter": "from_web", "converted_at": "2026-07-01T00:00:00"},
}
# ref → 預期分類。unregistered 的檔會實際落地；absent 的不落地。
CASES = {
"resolved": f"{K_OK}#abcd1234",
"hash-mismatch": f"{K_OK}#deadbeef",
"normalize": r"raw\converted\www-pluspay-faq-06b02fd9.md#abcd1234", # 反斜線
"stem": "raw/converted/www-pluspay-terms-99998888.md#cccccccc", # 同幹、異後綴、非 key
"unregistered": "raw/converted/www-pluspay-instructions-77776666.md#77777777",
"absent": "raw/converted/www-pluspay-ghost-00001111.md#cafebabe",
}
def unit():
"""classify_ref 純函式逐型別驗證。"""
on_disk = {"raw/converted/www-pluspay-instructions-77776666.md"}
exists = lambda p: p in on_disk
for expect, ref in CASES.items():
cat, detail = diag_refs.classify_ref(ref, FILES, exists)
assert cat == expect, f"{ref!r} 應分類為 {expect}，實得 {cat}（{detail}）"
print("PASS: classify_ref 六型別（含 resolved）分類正確")
def build(root):
for sub in ("raw/converted", "wiki/summaries", "wiki/entities", "wiki/concepts"):
(root / sub).mkdir(parents=True)
(root / "raw/manifest.json").write_text(
json.dumps({"version": 1, "files": FILES}), encoding="utf-8")
# unregistered 的檔要真的存在於磁碟，才會被判為「檔在、帳本無」而非 absent
(root / "raw/converted/www-pluspay-instructions-77776666.md").write_text(
"# 未登記\n", encoding="utf-8")
today = "2026-07-01"
for i, (name, ref) in enumerate(CASES.items()):
meta = {"type": "entity", "title": f"頁{name}", "description": "測試頁",
"tags": ["t"], "timestamp": today, "sources": [ref], "status": "draft"}
(root / f"wiki/entities/p{i}.md").write_text(kb.dump_page(meta, "內文。"), encoding="utf-8")
def run(argv):
buf = io.StringIO()
code = 0
try:
with redirect_stdout(buf):
diag_refs.main(argv)
except SystemExit as e:
code = e.code
return code, buf.getvalue()
def main():
unit()
tmp = pathlib.Path(tempfile.mkdtemp(prefix="ppqa-diag-"))
try:
root = tmp / "repo"
build(root)
before = {p: p.read_bytes() for p in root.rglob("*")
if p.is_file()}
code, out = run(["--root", str(root)])
assert code == 1, f"有岔開應以結束碼 1，實得 {code}\n{out}"
for cat in ("hash-mismatch", "normalize", "stem", "unregistered", "absent"):
assert diag_refs.CAT_TITLE[cat].split("（")[0] in out, f"報告缺型別 {cat}\n{out}"
assert "對不上：5" in out, out
after = {p: p.read_bytes() for p in root.rglob("*") if p.is_file()}
assert before == after, "diag_refs 修改了檔案！"
print("PASS: 整合——五型別全報、結束碼 1、零寫檔")
# 全部解得開 → 結束碼 0
good = {"type": "entity", "title": "好頁", "description": "d", "tags": ["t"],
"timestamp": "2026-07-01", "sources": [f"{K_OK}#abcd1234"], "status": "draft"}
for p in (root / "wiki/entities").glob("*.md"):
p.unlink()
(root / "wiki/entities/ok.md").write_text(kb.dump_page(good, "內文。"), encoding="utf-8")
code, out = run(["--root", str(root)])
assert code == 0, f"全部解得開應以結束碼 0，實得 {code}\n{out}"
assert "全部解得開" in out, out
print("PASS: 全部解得開時結束碼 0")
print("ALL PASS")
finally:
shutil.rmtree(tmp, ignore_errors=True)
if __name__ == "__main__":
main()

50
tools/selfcheck_ingest.py
View File

@@ -5,11 +5,13 @@
執行：.venv/Scripts/python tools/selfcheck_ingest.py
"""
import hashlib
import io
import pathlib
import shutil
import subprocess
import sys
import tempfile
from contextlib import redirect_stderr, redirect_stdout
sys.path.insert(0, str(pathlib.Path(__file__).parent))
import kb
@@ -82,9 +84,14 @@ def main():
git(root, "add", "-A")
git(root, "commit", "-q", "-m", "init")
# 1) 攝入：建分支、產頁、commit、回到 main
# 1) 攝入：建分支、產頁、commit、回到 main（並驗進度輸出可見，issue #2）
kb.llm_json = fake_llm
ingest.main(["raw/converted/doc1.md", "--root", str(root)])
buf = io.StringIO()
with redirect_stdout(buf):
ingest.main(["raw/converted/doc1.md", "--root", str(root)])
out = buf.getvalue()
for marker in ("[1/1] 攝入", "分段 1/", "↳ 新增", "✓ ", "完成"):
assert marker in out, f"進度輸出缺少 {marker!r}\n{out}"
assert git(root, "branch", "--show-current") == "main"
branch = git(root, "branch", "--list", "ingest/*").strip("* ").strip()
assert branch, "未建立 ingest 分支"
@@ -147,6 +154,45 @@ def main():
except RuntimeError as e:
assert "fallback" in str(e)
print("PASS: 重試 + fallback 切換 + 全失敗報錯")
# 5) Ctrl-C 安全（issue #2）：中斷時 main 不受影響、以 130 收場，
# 且文件宣稱的回復指令真的能回到乾淨 main。第 2 份來源開始摘要時中斷，
# 此時第 1 份的頁面已寫入（未追蹤、未 commit）——正是使用者最怕的半成品狀態。
add_doc(root, "docA.md", "來源A：待攝入。")
add_doc(root, "docB.md", "來源B：待攝入。")
git(root, "add", "-A")
git(root, "commit", "-q", "-m", "docA/docB")
head_before = git(root, "rev-parse", "main") # 攝入不得動到的 main 狀態
state = {"n": 0}
def boom(cfg, task, prompt, validate):
state["n"] += 1
if state["n"] >= 2: # 第 2 份來源的 summarize 呼叫
raise KeyboardInterrupt
return ({"title": "來源A", "description": "d", "slug": "fresh-a",
"tags": ["x"], "summary_markdown": "- a", "knowledge_items": []},
"fake:test")
kb.llm_json = boom
err = io.StringIO()
try:
with redirect_stdout(io.StringIO()), redirect_stderr(err):
ingest.main(["raw/converted/docA.md", "raw/converted/docB.md",
"--root", str(root)])
raise AssertionError("KeyboardInterrupt 應轉為 SystemExit(130)")
except SystemExit as e:
assert e.code == 130, e.code
msg = err.getvalue()
assert "git checkout main" in msg and "main 未受影響" in msg, msg
assert git(root, "rev-parse", "main") == head_before, "main 被動到了！"
cur = git(root, "branch", "--show-current")
assert cur.startswith("ingest/"), f"中斷後應停在 ingest 分支，實得 {cur!r}"
assert git(root, "status", "--porcelain"), "應有未提交的半成品"
# 照文件指令回復，斷言真的回到乾淨 main（驗「結果可用」，非「字串存在」，§13.4）
git(root, "checkout", "main")
git(root, "stash", "-u")
git(root, "branch", "-D", cur)
assert git(root, "branch", "--show-current") == "main"
assert not git(root, "status", "--porcelain"), "回復後工作區仍不乾淨"
print("PASS: Ctrl-C → main 未受影響、130 收場、回復指令實測可回乾淨 main")
print("ALL PASS")
finally:
kb.llm_json, kb.ollama_chat = real_llm, real_chat