diff --git a/README.md b/README.md index fe064c2..296c4bb 100644 --- a/README.md +++ b/README.md @@ -210,6 +210,7 @@ mcp/server.py # MCP 薄殼 | lint 報 `source_ref 不在 manifest` | 跑 `tools/diag_refs.py` 分類成因:帳本掉條目→補登走 PR(§14.3);路徑字串岔開→修規則(§12),別逐頁改。 | | `LLM 輸出驗證失敗(含 fallback)` | 本地模型連主模型 + fallback 都無法產出合規 JSON;檢查 Ollama 是否在線、模型是否拉好。 | | `push 失敗(無 remote 或離線)` | 正常;分支已保留本地,手動 push 後開 PR。 | +| ingest 跑很久、看不到進度/想中斷 | 已逐來源 `[i/N]` +逐分段回報進度。Ctrl-C 安全:main 不受影響,中斷時會印回復指令(`git checkout main && git stash -u && git branch -D `),續跑 `--all-pending` 自動接續。 | --- diff --git a/tools/ingest.py b/tools/ingest.py index e065bf8..eacf11f 100644 --- a/tools/ingest.py +++ b/tools/ingest.py @@ -123,6 +123,8 @@ def summarize(cfg, filename, text): parts = split_chunks(text, limit) partials = [] for i, part in enumerate(parts, 1): + # 長文件的分段迴圈是最容易「長時間靜默」的地方(issue #2):逐段回報進度 + print(f" 分段 {i}/{len(parts)} …", flush=True) obj, _ = kb.llm_json(cfg, "ingest", CHUNK_PROMPT.format(i=i, n=len(parts), content=part), validate_chunk) @@ -170,14 +172,16 @@ def upsert_item(root, cfg, item, source_ref, today, changed): if source_ref not in meta["sources"]: meta["sources"].append(source_ref) existing.write_text(kb.dump_page(meta, obj["updated_markdown"]), encoding="utf-8") - changed.append(f"更新 {existing.relative_to(root).as_posix()}") + msg = f"更新 {existing.relative_to(root).as_posix()}" else: p = root / kb.PAGE_DIRS[item["type"]] / f"{slug}.md" meta = {"type": item["type"], "title": item["title"], "description": item["description"], "tags": item["tags"], "timestamp": today, "sources": [source_ref], "status": "draft"} p.write_text(kb.dump_page(meta, item["facts_markdown"]), encoding="utf-8") - changed.append(f"新增 {p.relative_to(root).as_posix()}") + msg = f"新增 {p.relative_to(root).as_posix()}" + changed.append(msg) + print(f" ↳ {msg}", flush=True) # ---------- git ---------- @@ -240,7 +244,7 @@ def main(argv=None): skipped = [t for t in targets if conv[t]["sha256"][:8] in done] targets = [t for t in targets if conv[t]["sha256"][:8] not in done] for t in skipped: - print(f"skip: {t} 已有 summary 引用(--force 可重跑)") + print(f"skip: {t} 已有 summary 引用(--force 可重跑)", flush=True) if not targets: print("沒有待攝入的來源。") return @@ -261,21 +265,24 @@ def main(argv=None): run_git(root, "checkout", "-q", "-b", branch) ok, failed, changed = [], [], [] try: - for t in targets: + for i, t in enumerate(targets, 1): try: + print(f"[{i}/{len(targets)}] 攝入 {t} …", flush=True) text = (root / t).read_text(encoding="utf-8") sha8 = conv[t]["sha256"][:8] source_ref = f"{t}#{sha8}" obj, model = summarize(cfg, pathlib.Path(conv[t]["original_path"]).name, text) p = write_summary_page(root, obj, source_ref, sha8, today) - changed.append(f"新增 {p.relative_to(root).as_posix()}") + msg = f"新增 {p.relative_to(root).as_posix()}" + changed.append(msg) + print(f" ↳ {msg}", flush=True) for item in obj["knowledge_items"]: upsert_item(root, cfg, item, source_ref, today, changed) ok.append((t, model)) - print(f"ingested: {t}(model={model})") + print(f" ✓ {t} 完成(model={model})", flush=True) except Exception as e: failed.append((t, e)) - print(f"error: {t}: {e}", file=sys.stderr) + print(f" ✗ {t}: {e}", file=sys.stderr, flush=True) if not ok: run_git(root, "checkout", "-q", "main") run_git(root, "branch", "-q", "-D", branch) @@ -297,6 +304,16 @@ def main(argv=None): print(f"完成:{len(ok)} 成功、{len(failed)} 失敗。變更在分支 {branch},經 PR 審核後合併。") except SystemExit: raise + except KeyboardInterrupt: + # KeyboardInterrupt 不是 Exception 子類,會略過下方善後——故獨立攔截。 + # 中斷時尚未 commit(commit 在迴圈之後),main 必然未受影響;給一條可照做的回復路徑。 + print(f"\n已中斷(Ctrl-C)。main 未受影響——本次半成品都在分支 {branch}、尚未 commit。\n" + f"回到乾淨的 main:\n" + f" git checkout main && git stash -u && git branch -D {branch}\n" + f"(git stash -u 收起分支上未提交的半成品含未追蹤頁面;確定不要再 git stash drop)\n" + f"續跑 python tools/ingest.py --all-pending 會自動從尚未攝入的來源接續。", + file=sys.stderr, flush=True) + raise SystemExit(130) except Exception: print(f"攝入中斷。目前在分支 {branch},工作區可能有未提交變更," "請人工檢查(不自動清除以免遺失資料)。", file=sys.stderr) diff --git a/tools/selfcheck_ingest.py b/tools/selfcheck_ingest.py index 514fa4c..5abb067 100644 --- a/tools/selfcheck_ingest.py +++ b/tools/selfcheck_ingest.py @@ -5,11 +5,13 @@ 執行:.venv/Scripts/python tools/selfcheck_ingest.py """ import hashlib +import io import pathlib import shutil import subprocess import sys import tempfile +from contextlib import redirect_stderr, redirect_stdout sys.path.insert(0, str(pathlib.Path(__file__).parent)) import kb @@ -82,9 +84,14 @@ def main(): git(root, "add", "-A") git(root, "commit", "-q", "-m", "init") - # 1) 攝入:建分支、產頁、commit、回到 main + # 1) 攝入:建分支、產頁、commit、回到 main(並驗進度輸出可見,issue #2) kb.llm_json = fake_llm - ingest.main(["raw/converted/doc1.md", "--root", str(root)]) + buf = io.StringIO() + with redirect_stdout(buf): + ingest.main(["raw/converted/doc1.md", "--root", str(root)]) + out = buf.getvalue() + for marker in ("[1/1] 攝入", "分段 1/", "↳ 新增", "✓ ", "完成"): + assert marker in out, f"進度輸出缺少 {marker!r}\n{out}" assert git(root, "branch", "--show-current") == "main" branch = git(root, "branch", "--list", "ingest/*").strip("* ").strip() assert branch, "未建立 ingest 分支" @@ -147,6 +154,45 @@ def main(): except RuntimeError as e: assert "fallback" in str(e) print("PASS: 重試 + fallback 切換 + 全失敗報錯") + + # 5) Ctrl-C 安全(issue #2):中斷時 main 不受影響、以 130 收場, + # 且文件宣稱的回復指令真的能回到乾淨 main。第 2 份來源開始摘要時中斷, + # 此時第 1 份的頁面已寫入(未追蹤、未 commit)——正是使用者最怕的半成品狀態。 + add_doc(root, "docA.md", "來源A:待攝入。") + add_doc(root, "docB.md", "來源B:待攝入。") + git(root, "add", "-A") + git(root, "commit", "-q", "-m", "docA/docB") + head_before = git(root, "rev-parse", "main") # 攝入不得動到的 main 狀態 + state = {"n": 0} + def boom(cfg, task, prompt, validate): + state["n"] += 1 + if state["n"] >= 2: # 第 2 份來源的 summarize 呼叫 + raise KeyboardInterrupt + return ({"title": "來源A", "description": "d", "slug": "fresh-a", + "tags": ["x"], "summary_markdown": "- a", "knowledge_items": []}, + "fake:test") + kb.llm_json = boom + err = io.StringIO() + try: + with redirect_stdout(io.StringIO()), redirect_stderr(err): + ingest.main(["raw/converted/docA.md", "raw/converted/docB.md", + "--root", str(root)]) + raise AssertionError("KeyboardInterrupt 應轉為 SystemExit(130)") + except SystemExit as e: + assert e.code == 130, e.code + msg = err.getvalue() + assert "git checkout main" in msg and "main 未受影響" in msg, msg + assert git(root, "rev-parse", "main") == head_before, "main 被動到了!" + cur = git(root, "branch", "--show-current") + assert cur.startswith("ingest/"), f"中斷後應停在 ingest 分支,實得 {cur!r}" + assert git(root, "status", "--porcelain"), "應有未提交的半成品" + # 照文件指令回復,斷言真的回到乾淨 main(驗「結果可用」,非「字串存在」,§13.4) + git(root, "checkout", "main") + git(root, "stash", "-u") + git(root, "branch", "-D", cur) + assert git(root, "branch", "--show-current") == "main" + assert not git(root, "status", "--porcelain"), "回復後工作區仍不乾淨" + print("PASS: Ctrl-C → main 未受影響、130 收場、回復指令實測可回乾淨 main") print("ALL PASS") finally: kb.llm_json, kb.ollama_chat = real_llm, real_chat