feat: ingest 進度可見性 + Ctrl-C 安全回復 (Closes #2)
issue #2:--all-pending 長時間本地 Ollama 攝入全程靜默,使用者不確定有沒有在 跑、又不敢中斷。三處補強: - 進度可見:每份來源印 [i/N] 起始行、長文件逐分段印「分段 k/n」、每個 knowledge-item 印「↳ 新增/更新」、完成印「✓」,全部 flush=True 即時顯示。 最會靜默的分段迴圈正是重點回報處。 - Ctrl-C 安全:KeyboardInterrupt 不是 Exception 子類,原本會略過善後、把使用者 留在 ingest 分支且無指引。改為獨立攔截——commit 在迴圈之後,故 main 必然未受 影響——印出可照做的回復指令(git checkout main && git stash -u && git branch -D <branch>),以結束碼 130 收場。 selfcheck_ingest 新增:驗進度標記出現;驗中斷後 main HEAD 未動、停在 ingest 分支、且照文件回復指令實測能回到乾淨 main(驗結果可用,非字串存在,§13.4)。 README §7 補一列。 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -123,6 +123,8 @@ def summarize(cfg, filename, text):
|
||||
parts = split_chunks(text, limit)
|
||||
partials = []
|
||||
for i, part in enumerate(parts, 1):
|
||||
# 長文件的分段迴圈是最容易「長時間靜默」的地方(issue #2):逐段回報進度
|
||||
print(f" 分段 {i}/{len(parts)} …", flush=True)
|
||||
obj, _ = kb.llm_json(cfg, "ingest",
|
||||
CHUNK_PROMPT.format(i=i, n=len(parts), content=part),
|
||||
validate_chunk)
|
||||
@@ -170,14 +172,16 @@ def upsert_item(root, cfg, item, source_ref, today, changed):
|
||||
if source_ref not in meta["sources"]:
|
||||
meta["sources"].append(source_ref)
|
||||
existing.write_text(kb.dump_page(meta, obj["updated_markdown"]), encoding="utf-8")
|
||||
changed.append(f"更新 {existing.relative_to(root).as_posix()}")
|
||||
msg = f"更新 {existing.relative_to(root).as_posix()}"
|
||||
else:
|
||||
p = root / kb.PAGE_DIRS[item["type"]] / f"{slug}.md"
|
||||
meta = {"type": item["type"], "title": item["title"],
|
||||
"description": item["description"], "tags": item["tags"],
|
||||
"timestamp": today, "sources": [source_ref], "status": "draft"}
|
||||
p.write_text(kb.dump_page(meta, item["facts_markdown"]), encoding="utf-8")
|
||||
changed.append(f"新增 {p.relative_to(root).as_posix()}")
|
||||
msg = f"新增 {p.relative_to(root).as_posix()}"
|
||||
changed.append(msg)
|
||||
print(f" ↳ {msg}", flush=True)
|
||||
|
||||
|
||||
# ---------- git ----------
|
||||
@@ -240,7 +244,7 @@ def main(argv=None):
|
||||
skipped = [t for t in targets if conv[t]["sha256"][:8] in done]
|
||||
targets = [t for t in targets if conv[t]["sha256"][:8] not in done]
|
||||
for t in skipped:
|
||||
print(f"skip: {t} 已有 summary 引用(--force 可重跑)")
|
||||
print(f"skip: {t} 已有 summary 引用(--force 可重跑)", flush=True)
|
||||
if not targets:
|
||||
print("沒有待攝入的來源。")
|
||||
return
|
||||
@@ -261,21 +265,24 @@ def main(argv=None):
|
||||
run_git(root, "checkout", "-q", "-b", branch)
|
||||
ok, failed, changed = [], [], []
|
||||
try:
|
||||
for t in targets:
|
||||
for i, t in enumerate(targets, 1):
|
||||
try:
|
||||
print(f"[{i}/{len(targets)}] 攝入 {t} …", flush=True)
|
||||
text = (root / t).read_text(encoding="utf-8")
|
||||
sha8 = conv[t]["sha256"][:8]
|
||||
source_ref = f"{t}#{sha8}"
|
||||
obj, model = summarize(cfg, pathlib.Path(conv[t]["original_path"]).name, text)
|
||||
p = write_summary_page(root, obj, source_ref, sha8, today)
|
||||
changed.append(f"新增 {p.relative_to(root).as_posix()}")
|
||||
msg = f"新增 {p.relative_to(root).as_posix()}"
|
||||
changed.append(msg)
|
||||
print(f" ↳ {msg}", flush=True)
|
||||
for item in obj["knowledge_items"]:
|
||||
upsert_item(root, cfg, item, source_ref, today, changed)
|
||||
ok.append((t, model))
|
||||
print(f"ingested: {t}(model={model})")
|
||||
print(f" ✓ {t} 完成(model={model})", flush=True)
|
||||
except Exception as e:
|
||||
failed.append((t, e))
|
||||
print(f"error: {t}: {e}", file=sys.stderr)
|
||||
print(f" ✗ {t}: {e}", file=sys.stderr, flush=True)
|
||||
if not ok:
|
||||
run_git(root, "checkout", "-q", "main")
|
||||
run_git(root, "branch", "-q", "-D", branch)
|
||||
@@ -297,6 +304,16 @@ def main(argv=None):
|
||||
print(f"完成:{len(ok)} 成功、{len(failed)} 失敗。變更在分支 {branch},經 PR 審核後合併。")
|
||||
except SystemExit:
|
||||
raise
|
||||
except KeyboardInterrupt:
|
||||
# KeyboardInterrupt 不是 Exception 子類,會略過下方善後——故獨立攔截。
|
||||
# 中斷時尚未 commit(commit 在迴圈之後),main 必然未受影響;給一條可照做的回復路徑。
|
||||
print(f"\n已中斷(Ctrl-C)。main 未受影響——本次半成品都在分支 {branch}、尚未 commit。\n"
|
||||
f"回到乾淨的 main:\n"
|
||||
f" git checkout main && git stash -u && git branch -D {branch}\n"
|
||||
f"(git stash -u 收起分支上未提交的半成品含未追蹤頁面;確定不要再 git stash drop)\n"
|
||||
f"續跑 python tools/ingest.py --all-pending 會自動從尚未攝入的來源接續。",
|
||||
file=sys.stderr, flush=True)
|
||||
raise SystemExit(130)
|
||||
except Exception:
|
||||
print(f"攝入中斷。目前在分支 {branch},工作區可能有未提交變更,"
|
||||
"請人工檢查(不自動清除以免遺失資料)。", file=sys.stderr)
|
||||
|
||||
Reference in New Issue
Block a user