feat: diag_refs 對帳診斷 + ingest 進度可見性/Ctrl-C 安全 (Closes #2) #5

Merged
yellowadmin merged 4 commits from feat/convert-verify-reconcile into main 2026-07-24 05:42:35 +00:00
3 changed files with 73 additions and 9 deletions
Showing only changes of commit 4add7fa9d9 - Show all commits

View File

@@ -210,6 +210,7 @@ mcp/server.py # MCP 薄殼
| lint 報 `source_ref 不在 manifest` | 跑 `tools/diag_refs.py` 分類成因:帳本掉條目→補登走 PR§14.3路徑字串岔開→修規則§12別逐頁改。 |
| `LLM 輸出驗證失敗(含 fallback` | 本地模型連主模型 + fallback 都無法產出合規 JSON檢查 Ollama 是否在線、模型是否拉好。 |
| `push 失敗(無 remote 或離線)` | 正常;分支已保留本地,手動 push 後開 PR。 |
| ingest 跑很久、看不到進度/想中斷 | 已逐來源 `[i/N]` 逐分段回報進度。Ctrl-C 安全main 不受影響,中斷時會印回復指令(`git checkout main && git stash -u && git branch -D <branch>`),續跑 `--all-pending` 自動接續。 |
---

View File

@@ -123,6 +123,8 @@ def summarize(cfg, filename, text):
parts = split_chunks(text, limit)
partials = []
for i, part in enumerate(parts, 1):
# 長文件的分段迴圈是最容易「長時間靜默」的地方issue #2逐段回報進度
print(f" 分段 {i}/{len(parts)}", flush=True)
obj, _ = kb.llm_json(cfg, "ingest",
CHUNK_PROMPT.format(i=i, n=len(parts), content=part),
validate_chunk)
@@ -170,14 +172,16 @@ def upsert_item(root, cfg, item, source_ref, today, changed):
if source_ref not in meta["sources"]:
meta["sources"].append(source_ref)
existing.write_text(kb.dump_page(meta, obj["updated_markdown"]), encoding="utf-8")
changed.append(f"更新 {existing.relative_to(root).as_posix()}")
msg = f"更新 {existing.relative_to(root).as_posix()}"
else:
p = root / kb.PAGE_DIRS[item["type"]] / f"{slug}.md"
meta = {"type": item["type"], "title": item["title"],
"description": item["description"], "tags": item["tags"],
"timestamp": today, "sources": [source_ref], "status": "draft"}
p.write_text(kb.dump_page(meta, item["facts_markdown"]), encoding="utf-8")
changed.append(f"新增 {p.relative_to(root).as_posix()}")
msg = f"新增 {p.relative_to(root).as_posix()}"
changed.append(msg)
print(f"{msg}", flush=True)
# ---------- git ----------
@@ -240,7 +244,7 @@ def main(argv=None):
skipped = [t for t in targets if conv[t]["sha256"][:8] in done]
targets = [t for t in targets if conv[t]["sha256"][:8] not in done]
for t in skipped:
print(f"skip: {t} 已有 summary 引用(--force 可重跑)")
print(f"skip: {t} 已有 summary 引用(--force 可重跑)", flush=True)
if not targets:
print("沒有待攝入的來源。")
return
@@ -261,21 +265,24 @@ def main(argv=None):
run_git(root, "checkout", "-q", "-b", branch)
ok, failed, changed = [], [], []
try:
for t in targets:
for i, t in enumerate(targets, 1):
try:
print(f"[{i}/{len(targets)}] 攝入 {t}", flush=True)
text = (root / t).read_text(encoding="utf-8")
sha8 = conv[t]["sha256"][:8]
source_ref = f"{t}#{sha8}"
obj, model = summarize(cfg, pathlib.Path(conv[t]["original_path"]).name, text)
p = write_summary_page(root, obj, source_ref, sha8, today)
changed.append(f"新增 {p.relative_to(root).as_posix()}")
msg = f"新增 {p.relative_to(root).as_posix()}"
changed.append(msg)
print(f"{msg}", flush=True)
for item in obj["knowledge_items"]:
upsert_item(root, cfg, item, source_ref, today, changed)
ok.append((t, model))
print(f"ingested: {t}model={model}")
print(f" {t} 完成model={model}", flush=True)
except Exception as e:
failed.append((t, e))
print(f"error: {t}: {e}", file=sys.stderr)
print(f" {t}: {e}", file=sys.stderr, flush=True)
if not ok:
run_git(root, "checkout", "-q", "main")
run_git(root, "branch", "-q", "-D", branch)
@@ -297,6 +304,16 @@ def main(argv=None):
print(f"完成:{len(ok)} 成功、{len(failed)} 失敗。變更在分支 {branch},經 PR 審核後合併。")
except SystemExit:
raise
except KeyboardInterrupt:
# KeyboardInterrupt 不是 Exception 子類,會略過下方善後——故獨立攔截。
# 中斷時尚未 commitcommit 在迴圈之後main 必然未受影響;給一條可照做的回復路徑。
print(f"\n已中斷Ctrl-C。main 未受影響——本次半成品都在分支 {branch}、尚未 commit。\n"
f"回到乾淨的 main\n"
f" git checkout main && git stash -u && git branch -D {branch}\n"
f"git stash -u 收起分支上未提交的半成品含未追蹤頁面;確定不要再 git stash drop\n"
f"續跑 python tools/ingest.py --all-pending 會自動從尚未攝入的來源接續。",
file=sys.stderr, flush=True)
raise SystemExit(130)
except Exception:
print(f"攝入中斷。目前在分支 {branch},工作區可能有未提交變更,"
"請人工檢查(不自動清除以免遺失資料)。", file=sys.stderr)

View File

@@ -5,11 +5,13 @@
執行:.venv/Scripts/python tools/selfcheck_ingest.py
"""
import hashlib
import io
import pathlib
import shutil
import subprocess
import sys
import tempfile
from contextlib import redirect_stderr, redirect_stdout
sys.path.insert(0, str(pathlib.Path(__file__).parent))
import kb
@@ -82,9 +84,14 @@ def main():
git(root, "add", "-A")
git(root, "commit", "-q", "-m", "init")
# 1) 攝入建分支、產頁、commit、回到 main
# 1) 攝入建分支、產頁、commit、回到 main並驗進度輸出可見issue #2
kb.llm_json = fake_llm
ingest.main(["raw/converted/doc1.md", "--root", str(root)])
buf = io.StringIO()
with redirect_stdout(buf):
ingest.main(["raw/converted/doc1.md", "--root", str(root)])
out = buf.getvalue()
for marker in ("[1/1] 攝入", "分段 1/", "↳ 新增", "", "完成"):
assert marker in out, f"進度輸出缺少 {marker!r}\n{out}"
assert git(root, "branch", "--show-current") == "main"
branch = git(root, "branch", "--list", "ingest/*").strip("* ").strip()
assert branch, "未建立 ingest 分支"
@@ -147,6 +154,45 @@ def main():
except RuntimeError as e:
assert "fallback" in str(e)
print("PASS: 重試 + fallback 切換 + 全失敗報錯")
# 5) Ctrl-C 安全issue #2中斷時 main 不受影響、以 130 收場,
# 且文件宣稱的回復指令真的能回到乾淨 main。第 2 份來源開始摘要時中斷,
# 此時第 1 份的頁面已寫入(未追蹤、未 commit——正是使用者最怕的半成品狀態。
add_doc(root, "docA.md", "來源A待攝入。")
add_doc(root, "docB.md", "來源B待攝入。")
git(root, "add", "-A")
git(root, "commit", "-q", "-m", "docA/docB")
head_before = git(root, "rev-parse", "main") # 攝入不得動到的 main 狀態
state = {"n": 0}
def boom(cfg, task, prompt, validate):
state["n"] += 1
if state["n"] >= 2: # 第 2 份來源的 summarize 呼叫
raise KeyboardInterrupt
return ({"title": "來源A", "description": "d", "slug": "fresh-a",
"tags": ["x"], "summary_markdown": "- a", "knowledge_items": []},
"fake:test")
kb.llm_json = boom
err = io.StringIO()
try:
with redirect_stdout(io.StringIO()), redirect_stderr(err):
ingest.main(["raw/converted/docA.md", "raw/converted/docB.md",
"--root", str(root)])
raise AssertionError("KeyboardInterrupt 應轉為 SystemExit(130)")
except SystemExit as e:
assert e.code == 130, e.code
msg = err.getvalue()
assert "git checkout main" in msg and "main 未受影響" in msg, msg
assert git(root, "rev-parse", "main") == head_before, "main 被動到了!"
cur = git(root, "branch", "--show-current")
assert cur.startswith("ingest/"), f"中斷後應停在 ingest 分支,實得 {cur!r}"
assert git(root, "status", "--porcelain"), "應有未提交的半成品"
# 照文件指令回復,斷言真的回到乾淨 main驗「結果可用」非「字串存在」§13.4
git(root, "checkout", "main")
git(root, "stash", "-u")
git(root, "branch", "-D", cur)
assert git(root, "branch", "--show-current") == "main"
assert not git(root, "status", "--porcelain"), "回復後工作區仍不乾淨"
print("PASS: Ctrl-C → main 未受影響、130 收場、回復指令實測可回乾淨 main")
print("ALL PASS")
finally:
kb.llm_json, kb.ollama_chat = real_llm, real_chat