feat: diag_refs 對帳診斷 + ingest 進度可見性/Ctrl-C 安全 (Closes #2) #5
@@ -210,6 +210,7 @@ mcp/server.py # MCP 薄殼
|
||||
| lint 報 `source_ref 不在 manifest` | 跑 `tools/diag_refs.py` 分類成因:帳本掉條目→補登走 PR(§14.3);路徑字串岔開→修規則(§12),別逐頁改。 |
|
||||
| `LLM 輸出驗證失敗(含 fallback)` | 本地模型連主模型 + fallback 都無法產出合規 JSON;檢查 Ollama 是否在線、模型是否拉好。 |
|
||||
| `push 失敗(無 remote 或離線)` | 正常;分支已保留本地,手動 push 後開 PR。 |
|
||||
| ingest 跑很久、看不到進度/想中斷 | 已逐來源 `[i/N]` +逐分段回報進度。Ctrl-C 安全:main 不受影響,中斷時會印回復指令(`git checkout main && git stash -u && git branch -D <branch>`),續跑 `--all-pending` 自動接續。 |
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -123,6 +123,8 @@ def summarize(cfg, filename, text):
|
||||
parts = split_chunks(text, limit)
|
||||
partials = []
|
||||
for i, part in enumerate(parts, 1):
|
||||
# 長文件的分段迴圈是最容易「長時間靜默」的地方(issue #2):逐段回報進度
|
||||
print(f" 分段 {i}/{len(parts)} …", flush=True)
|
||||
obj, _ = kb.llm_json(cfg, "ingest",
|
||||
CHUNK_PROMPT.format(i=i, n=len(parts), content=part),
|
||||
validate_chunk)
|
||||
@@ -170,14 +172,16 @@ def upsert_item(root, cfg, item, source_ref, today, changed):
|
||||
if source_ref not in meta["sources"]:
|
||||
meta["sources"].append(source_ref)
|
||||
existing.write_text(kb.dump_page(meta, obj["updated_markdown"]), encoding="utf-8")
|
||||
changed.append(f"更新 {existing.relative_to(root).as_posix()}")
|
||||
msg = f"更新 {existing.relative_to(root).as_posix()}"
|
||||
else:
|
||||
p = root / kb.PAGE_DIRS[item["type"]] / f"{slug}.md"
|
||||
meta = {"type": item["type"], "title": item["title"],
|
||||
"description": item["description"], "tags": item["tags"],
|
||||
"timestamp": today, "sources": [source_ref], "status": "draft"}
|
||||
p.write_text(kb.dump_page(meta, item["facts_markdown"]), encoding="utf-8")
|
||||
changed.append(f"新增 {p.relative_to(root).as_posix()}")
|
||||
msg = f"新增 {p.relative_to(root).as_posix()}"
|
||||
changed.append(msg)
|
||||
print(f" ↳ {msg}", flush=True)
|
||||
|
||||
|
||||
# ---------- git ----------
|
||||
@@ -240,7 +244,7 @@ def main(argv=None):
|
||||
skipped = [t for t in targets if conv[t]["sha256"][:8] in done]
|
||||
targets = [t for t in targets if conv[t]["sha256"][:8] not in done]
|
||||
for t in skipped:
|
||||
print(f"skip: {t} 已有 summary 引用(--force 可重跑)")
|
||||
print(f"skip: {t} 已有 summary 引用(--force 可重跑)", flush=True)
|
||||
if not targets:
|
||||
print("沒有待攝入的來源。")
|
||||
return
|
||||
@@ -261,21 +265,24 @@ def main(argv=None):
|
||||
run_git(root, "checkout", "-q", "-b", branch)
|
||||
ok, failed, changed = [], [], []
|
||||
try:
|
||||
for t in targets:
|
||||
for i, t in enumerate(targets, 1):
|
||||
try:
|
||||
print(f"[{i}/{len(targets)}] 攝入 {t} …", flush=True)
|
||||
text = (root / t).read_text(encoding="utf-8")
|
||||
sha8 = conv[t]["sha256"][:8]
|
||||
source_ref = f"{t}#{sha8}"
|
||||
obj, model = summarize(cfg, pathlib.Path(conv[t]["original_path"]).name, text)
|
||||
p = write_summary_page(root, obj, source_ref, sha8, today)
|
||||
changed.append(f"新增 {p.relative_to(root).as_posix()}")
|
||||
msg = f"新增 {p.relative_to(root).as_posix()}"
|
||||
changed.append(msg)
|
||||
print(f" ↳ {msg}", flush=True)
|
||||
for item in obj["knowledge_items"]:
|
||||
upsert_item(root, cfg, item, source_ref, today, changed)
|
||||
ok.append((t, model))
|
||||
print(f"ingested: {t}(model={model})")
|
||||
print(f" ✓ {t} 完成(model={model})", flush=True)
|
||||
except Exception as e:
|
||||
failed.append((t, e))
|
||||
print(f"error: {t}: {e}", file=sys.stderr)
|
||||
print(f" ✗ {t}: {e}", file=sys.stderr, flush=True)
|
||||
if not ok:
|
||||
run_git(root, "checkout", "-q", "main")
|
||||
run_git(root, "branch", "-q", "-D", branch)
|
||||
@@ -297,6 +304,16 @@ def main(argv=None):
|
||||
print(f"完成:{len(ok)} 成功、{len(failed)} 失敗。變更在分支 {branch},經 PR 審核後合併。")
|
||||
except SystemExit:
|
||||
raise
|
||||
except KeyboardInterrupt:
|
||||
# KeyboardInterrupt 不是 Exception 子類,會略過下方善後——故獨立攔截。
|
||||
# 中斷時尚未 commit(commit 在迴圈之後),main 必然未受影響;給一條可照做的回復路徑。
|
||||
print(f"\n已中斷(Ctrl-C)。main 未受影響——本次半成品都在分支 {branch}、尚未 commit。\n"
|
||||
f"回到乾淨的 main:\n"
|
||||
f" git checkout main && git stash -u && git branch -D {branch}\n"
|
||||
f"(git stash -u 收起分支上未提交的半成品含未追蹤頁面;確定不要再 git stash drop)\n"
|
||||
f"續跑 python tools/ingest.py --all-pending 會自動從尚未攝入的來源接續。",
|
||||
file=sys.stderr, flush=True)
|
||||
raise SystemExit(130)
|
||||
except Exception:
|
||||
print(f"攝入中斷。目前在分支 {branch},工作區可能有未提交變更,"
|
||||
"請人工檢查(不自動清除以免遺失資料)。", file=sys.stderr)
|
||||
|
||||
@@ -5,11 +5,13 @@
|
||||
執行:.venv/Scripts/python tools/selfcheck_ingest.py
|
||||
"""
|
||||
import hashlib
|
||||
import io
|
||||
import pathlib
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from contextlib import redirect_stderr, redirect_stdout
|
||||
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).parent))
|
||||
import kb
|
||||
@@ -82,9 +84,14 @@ def main():
|
||||
git(root, "add", "-A")
|
||||
git(root, "commit", "-q", "-m", "init")
|
||||
|
||||
# 1) 攝入:建分支、產頁、commit、回到 main
|
||||
# 1) 攝入:建分支、產頁、commit、回到 main(並驗進度輸出可見,issue #2)
|
||||
kb.llm_json = fake_llm
|
||||
buf = io.StringIO()
|
||||
with redirect_stdout(buf):
|
||||
ingest.main(["raw/converted/doc1.md", "--root", str(root)])
|
||||
out = buf.getvalue()
|
||||
for marker in ("[1/1] 攝入", "分段 1/", "↳ 新增", "✓ ", "完成"):
|
||||
assert marker in out, f"進度輸出缺少 {marker!r}\n{out}"
|
||||
assert git(root, "branch", "--show-current") == "main"
|
||||
branch = git(root, "branch", "--list", "ingest/*").strip("* ").strip()
|
||||
assert branch, "未建立 ingest 分支"
|
||||
@@ -147,6 +154,45 @@ def main():
|
||||
except RuntimeError as e:
|
||||
assert "fallback" in str(e)
|
||||
print("PASS: 重試 + fallback 切換 + 全失敗報錯")
|
||||
|
||||
# 5) Ctrl-C 安全(issue #2):中斷時 main 不受影響、以 130 收場,
|
||||
# 且文件宣稱的回復指令真的能回到乾淨 main。第 2 份來源開始摘要時中斷,
|
||||
# 此時第 1 份的頁面已寫入(未追蹤、未 commit)——正是使用者最怕的半成品狀態。
|
||||
add_doc(root, "docA.md", "來源A:待攝入。")
|
||||
add_doc(root, "docB.md", "來源B:待攝入。")
|
||||
git(root, "add", "-A")
|
||||
git(root, "commit", "-q", "-m", "docA/docB")
|
||||
head_before = git(root, "rev-parse", "main") # 攝入不得動到的 main 狀態
|
||||
state = {"n": 0}
|
||||
def boom(cfg, task, prompt, validate):
|
||||
state["n"] += 1
|
||||
if state["n"] >= 2: # 第 2 份來源的 summarize 呼叫
|
||||
raise KeyboardInterrupt
|
||||
return ({"title": "來源A", "description": "d", "slug": "fresh-a",
|
||||
"tags": ["x"], "summary_markdown": "- a", "knowledge_items": []},
|
||||
"fake:test")
|
||||
kb.llm_json = boom
|
||||
err = io.StringIO()
|
||||
try:
|
||||
with redirect_stdout(io.StringIO()), redirect_stderr(err):
|
||||
ingest.main(["raw/converted/docA.md", "raw/converted/docB.md",
|
||||
"--root", str(root)])
|
||||
raise AssertionError("KeyboardInterrupt 應轉為 SystemExit(130)")
|
||||
except SystemExit as e:
|
||||
assert e.code == 130, e.code
|
||||
msg = err.getvalue()
|
||||
assert "git checkout main" in msg and "main 未受影響" in msg, msg
|
||||
assert git(root, "rev-parse", "main") == head_before, "main 被動到了!"
|
||||
cur = git(root, "branch", "--show-current")
|
||||
assert cur.startswith("ingest/"), f"中斷後應停在 ingest 分支,實得 {cur!r}"
|
||||
assert git(root, "status", "--porcelain"), "應有未提交的半成品"
|
||||
# 照文件指令回復,斷言真的回到乾淨 main(驗「結果可用」,非「字串存在」,§13.4)
|
||||
git(root, "checkout", "main")
|
||||
git(root, "stash", "-u")
|
||||
git(root, "branch", "-D", cur)
|
||||
assert git(root, "branch", "--show-current") == "main"
|
||||
assert not git(root, "status", "--porcelain"), "回復後工作區仍不乾淨"
|
||||
print("PASS: Ctrl-C → main 未受影響、130 收場、回復指令實測可回乾淨 main")
|
||||
print("ALL PASS")
|
||||
finally:
|
||||
kb.llm_json, kb.ollama_chat = real_llm, real_chat
|
||||
|
||||
Reference in New Issue
Block a user