feat: convert 新增 --verify 對帳模式 + 修復 Windows CRLF 使 hash 失準
- convert.py --verify:純唯讀 re-hash 全部登記檔並掃未登記檔, 報 missing/mismatch/unregistered,不一致退出碼 1(供刪除/竄改後稽核) - 修 write_text 在 Windows 轉 \n→\r\n 使磁碟 bytes 與登記 SHA-256 不符 (converted md 與 web 快照原始檔),改 write_bytes 寫入所登記的那份 bytes - selfcheck 補 clean/missing/mismatch/unregistered 四情境(全程唯讀斷言) - AGENTS.md §14 raw 完整性與修復流程 + §1.4 hash==磁碟 bytes 不變式 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -7,6 +7,7 @@
|
||||
用法:python tools/convert/convert.py <檔案路徑或URL>... [--vision] [--dry-run] [--root DIR]
|
||||
支援 .docx/.xlsx/.pdf/.pptx/.html 與 URL;舊版 .doc/.xls/.ppt 經 LibreOffice 升版;
|
||||
--vision 讓掃描/圖片型 PDF 走本地 vision 模型轉錄(見 from_vision.py)。
|
||||
--verify 對帳模式:re-hash 全部登記檔、掃出未登記檔,純唯讀不改檔,供刪除/竄改後稽核。
|
||||
"""
|
||||
import argparse
|
||||
import datetime
|
||||
@@ -87,7 +88,10 @@ def _register_pair(m, root, orig_rel, orig_entry, md_rel, md_text, converter, dr
|
||||
print(f"warning: {orig_rel} 轉換結果為空白,請人工檢查來源", file=sys.stderr)
|
||||
if not dry:
|
||||
md_path = root / md_rel
|
||||
md_path.write_text(md_text, encoding="utf-8")
|
||||
# write_bytes(非 write_text):登記的 sha256 算在 md_text.encode() 上,
|
||||
# 而 write_text 在 Windows 會把 \n 轉 \r\n,使磁碟 bytes 與登記 hash 不符,
|
||||
# 令日後「還原後 re-hash 比對」與 --verify 失準。寫入即登記的那份 bytes。
|
||||
md_path.write_bytes(md_text.encode("utf-8"))
|
||||
m["files"][orig_rel] = orig_entry
|
||||
m["files"][md_rel] = {
|
||||
"kind": "converted",
|
||||
@@ -130,6 +134,54 @@ def _vision_pdf(root, pdf_path, assets, conv_name):
|
||||
return "\n\n".join(blocks), "ok", "from_vision"
|
||||
|
||||
|
||||
def verify(root, m):
|
||||
"""對帳模式:re-hash 每個 manifest 條目對應的檔案,並掃出未登記的 raw 檔。
|
||||
純唯讀——不改任何檔案(含 manifest),只回報。有不一致以 SystemExit(1) 表示,
|
||||
可供 CI/排程當閘門用。
|
||||
|
||||
偵測三類意外:
|
||||
missing —— 帳本有登記,磁碟上檔案不見了(手動刪除)。
|
||||
mismatch —— 檔案還在但 sha256 與登記值不符(raw 層被就地改動,違反 §1.4)。
|
||||
unregistered —— raw/originals 或 raw/converted 下有檔卻不在帳本;
|
||||
converted/assets/* 由 markdown 連結而非帳本追蹤,故略過,.gitkeep 亦略過。
|
||||
"""
|
||||
missing, mismatch = [], []
|
||||
for rel, e in m["files"].items():
|
||||
p = root / rel
|
||||
if not p.is_file():
|
||||
missing.append(rel)
|
||||
elif hashlib.sha256(p.read_bytes()).hexdigest() != e["sha256"]:
|
||||
mismatch.append(rel)
|
||||
|
||||
unregistered = []
|
||||
for sub in ("originals", "converted"):
|
||||
base = root / "raw" / sub
|
||||
if not base.is_dir():
|
||||
continue
|
||||
for p in base.rglob("*"):
|
||||
if not p.is_file() or p.name == ".gitkeep":
|
||||
continue
|
||||
if "assets" in p.relative_to(base).parts: # 由 markdown 連結,不入帳本
|
||||
continue
|
||||
rel = p.relative_to(root).as_posix()
|
||||
if rel not in m["files"]:
|
||||
unregistered.append(rel)
|
||||
|
||||
for rel in missing:
|
||||
print(f"missing: {rel}(帳本有登記,磁碟上不見了)", file=sys.stderr)
|
||||
for rel in mismatch:
|
||||
print(f"mismatch: {rel}(sha256 與登記值不符,raw 層被就地改動)", file=sys.stderr)
|
||||
for rel in sorted(unregistered):
|
||||
print(f"unregistered: {rel}(磁碟上有檔,帳本未登記)", file=sys.stderr)
|
||||
|
||||
if missing or mismatch or unregistered:
|
||||
print(f"\nverify: {len(m['files'])} 筆登記 → "
|
||||
f"{len(missing)} missing / {len(mismatch)} mismatch / "
|
||||
f"{len(unregistered)} unregistered(純唯讀,未改任何檔)", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
print(f"verify: {len(m['files'])} 筆登記全部對得上,raw/ 無未登記檔案")
|
||||
|
||||
|
||||
def process_local(root, m, path, dry, vision=False):
|
||||
src = pathlib.Path(path).resolve()
|
||||
if not src.is_file():
|
||||
@@ -208,7 +260,9 @@ def process_url(root, m, url, dry):
|
||||
"added_at": _now(), "status": "converted",
|
||||
"source_url": url, "fetched_at": _now()}
|
||||
if not dry:
|
||||
(originals / name).write_text(html, encoding="utf-8") # 完整快照(來源可能消失)
|
||||
# write_bytes:同 §_register_pair,登記 hash 算在 html.encode() 上,
|
||||
# 避免 Windows 換行轉換讓快照 bytes 與登記 hash 不符(完整快照,來源可能消失)
|
||||
(originals / name).write_bytes(html.encode("utf-8"))
|
||||
md_text, status = from_web.extract(html)
|
||||
if status != "ok":
|
||||
orig_entry["status"] = "pending"
|
||||
@@ -223,7 +277,10 @@ def process_url(root, m, url, dry):
|
||||
|
||||
def main(argv=None):
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("inputs", nargs="+", help="檔案路徑或 http(s) URL")
|
||||
ap.add_argument("inputs", nargs="*", help="檔案路徑或 http(s) URL")
|
||||
ap.add_argument("--verify", action="store_true",
|
||||
help="對帳模式:re-hash 全部登記檔、掃出未登記檔,純唯讀不改檔;"
|
||||
"有不一致以退出碼 1 表示(不吃 inputs)")
|
||||
ap.add_argument("--dry-run", action="store_true")
|
||||
ap.add_argument("--vision", action="store_true",
|
||||
help="掃描/圖片型 PDF(needs_ocr)改走本地 vision 模型轉錄,"
|
||||
@@ -234,6 +291,11 @@ def main(argv=None):
|
||||
a = ap.parse_args(argv)
|
||||
root = pathlib.Path(a.root).resolve()
|
||||
m = load_manifest(root)
|
||||
if a.verify:
|
||||
verify(root, m)
|
||||
return
|
||||
if not a.inputs:
|
||||
ap.error("需指定至少一個檔案/URL,或改用 --verify 對帳")
|
||||
failed = []
|
||||
for item in a.inputs:
|
||||
try:
|
||||
|
||||
@@ -158,6 +158,45 @@ def main():
|
||||
assert (root / "raw/manifest.json").read_text(encoding="utf-8") == before
|
||||
print("PASS: 冪等(同 hash 跳過)")
|
||||
|
||||
# --verify 對帳(純唯讀):clean 通過、三類意外都抓到、且完全不改 manifest。
|
||||
# root 已有 docx/pdf/pptx 抽出的 assets(不入帳本)——clean 案例即證明 assets 不被誤報。
|
||||
def run_verify(rt):
|
||||
mm = json.loads((rt / "raw/manifest.json").read_text(encoding="utf-8"))
|
||||
err, code = io.StringIO(), 0
|
||||
with contextlib.redirect_stderr(err), contextlib.redirect_stdout(io.StringIO()):
|
||||
try:
|
||||
convert.verify(rt, mm)
|
||||
except SystemExit as e:
|
||||
code = e.code
|
||||
return code, err.getvalue()
|
||||
|
||||
before = (root / "raw/manifest.json").read_text(encoding="utf-8")
|
||||
code, _ = run_verify(root)
|
||||
assert code == 0, "clean 帳本應通過對帳(含未入帳本的 assets)"
|
||||
|
||||
rroot = tmp / "root_del" # missing:刪掉一個 converted .md
|
||||
shutil.copytree(root, rroot)
|
||||
gone = next(p for p in (rroot / "raw/converted").glob("*.md"))
|
||||
gone.unlink()
|
||||
code, log = run_verify(rroot)
|
||||
assert code == 1 and "missing" in log and gone.name in log, log
|
||||
|
||||
rroot = tmp / "root_mut" # mismatch:就地改動一個 original 的 bytes
|
||||
shutil.copytree(root, rroot)
|
||||
tampered = next(p for p in (rroot / "raw/originals").iterdir() if p.is_file())
|
||||
tampered.write_bytes(tampered.read_bytes() + b"tampered")
|
||||
code, log = run_verify(rroot)
|
||||
assert code == 1 and "mismatch" in log, log
|
||||
|
||||
rroot = tmp / "root_unreg" # unregistered:raw/originals 塞入未登記檔
|
||||
shutil.copytree(root, rroot)
|
||||
(rroot / "raw/originals/未登記 檔.docx").write_bytes(b"stray")
|
||||
code, log = run_verify(rroot)
|
||||
assert code == 1 and "unregistered" in log and "未登記 檔.docx" in log, log
|
||||
|
||||
assert (root / "raw/manifest.json").read_text(encoding="utf-8") == before # 全程唯讀
|
||||
print("PASS: --verify 對帳(clean/missing/mismatch/unregistered,純唯讀)")
|
||||
|
||||
# dry-run:全新沙盒不落地
|
||||
root2 = tmp / "root2"
|
||||
for sub in ("raw/originals", "raw/converted"):
|
||||
|
||||
Reference in New Issue
Block a user