feat: convert 新增 --verify 對帳模式 + 修復 Windows CRLF 使 hash 失準

- convert.py --verify:純唯讀 re-hash 全部登記檔並掃未登記檔,
  報 missing/mismatch/unregistered,不一致退出碼 1(供刪除/竄改後稽核)
- 修 write_text 在 Windows 轉 \n→\r\n 使磁碟 bytes 與登記 SHA-256 不符
  (converted md 與 web 快照原始檔),改 write_bytes 寫入所登記的那份 bytes
- selfcheck 補 clean/missing/mismatch/unregistered 四情境(全程唯讀斷言)
- AGENTS.md §14 raw 完整性與修復流程 + §1.4 hash==磁碟 bytes 不變式

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
LittleYellow
2026-07-23 08:21:39 +08:00
parent cb16e8688f
commit f9a0cd2369
3 changed files with 149 additions and 4 deletions

View File

@@ -7,6 +7,7 @@
用法python tools/convert/convert.py <檔案路徑或URL>... [--vision] [--dry-run] [--root DIR]
支援 .docx/.xlsx/.pdf/.pptx/.html 與 URL舊版 .doc/.xls/.ppt 經 LibreOffice 升版;
--vision 讓掃描/圖片型 PDF 走本地 vision 模型轉錄(見 from_vision.py
--verify 對帳模式re-hash 全部登記檔、掃出未登記檔,純唯讀不改檔,供刪除/竄改後稽核。
"""
import argparse
import datetime
@@ -87,7 +88,10 @@ def _register_pair(m, root, orig_rel, orig_entry, md_rel, md_text, converter, dr
print(f"warning: {orig_rel} 轉換結果為空白,請人工檢查來源", file=sys.stderr)
if not dry:
md_path = root / md_rel
md_path.write_text(md_text, encoding="utf-8")
# write_bytes非 write_text登記的 sha256 算在 md_text.encode() 上,
# 而 write_text 在 Windows 會把 \n 轉 \r\n使磁碟 bytes 與登記 hash 不符,
# 令日後「還原後 re-hash 比對」與 --verify 失準。寫入即登記的那份 bytes。
md_path.write_bytes(md_text.encode("utf-8"))
m["files"][orig_rel] = orig_entry
m["files"][md_rel] = {
"kind": "converted",
@@ -130,6 +134,54 @@ def _vision_pdf(root, pdf_path, assets, conv_name):
return "\n\n".join(blocks), "ok", "from_vision"
def verify(root, m):
"""對帳模式re-hash 每個 manifest 條目對應的檔案,並掃出未登記的 raw 檔。
純唯讀——不改任何檔案(含 manifest只回報。有不一致以 SystemExit(1) 表示,
可供 CI排程當閘門用。
偵測三類意外:
missing —— 帳本有登記,磁碟上檔案不見了(手動刪除)。
mismatch —— 檔案還在但 sha256 與登記值不符raw 層被就地改動,違反 §1.4)。
unregistered —— raw/originals 或 raw/converted 下有檔卻不在帳本;
converted/assets/* 由 markdown 連結而非帳本追蹤,故略過,.gitkeep 亦略過。
"""
missing, mismatch = [], []
for rel, e in m["files"].items():
p = root / rel
if not p.is_file():
missing.append(rel)
elif hashlib.sha256(p.read_bytes()).hexdigest() != e["sha256"]:
mismatch.append(rel)
unregistered = []
for sub in ("originals", "converted"):
base = root / "raw" / sub
if not base.is_dir():
continue
for p in base.rglob("*"):
if not p.is_file() or p.name == ".gitkeep":
continue
if "assets" in p.relative_to(base).parts: # 由 markdown 連結,不入帳本
continue
rel = p.relative_to(root).as_posix()
if rel not in m["files"]:
unregistered.append(rel)
for rel in missing:
print(f"missing: {rel}(帳本有登記,磁碟上不見了)", file=sys.stderr)
for rel in mismatch:
print(f"mismatch: {rel}sha256 與登記值不符raw 層被就地改動)", file=sys.stderr)
for rel in sorted(unregistered):
print(f"unregistered: {rel}(磁碟上有檔,帳本未登記)", file=sys.stderr)
if missing or mismatch or unregistered:
print(f"\nverify: {len(m['files'])} 筆登記 → "
f"{len(missing)} missing / {len(mismatch)} mismatch / "
f"{len(unregistered)} unregistered純唯讀未改任何檔", file=sys.stderr)
raise SystemExit(1)
print(f"verify: {len(m['files'])} 筆登記全部對得上raw/ 無未登記檔案")
def process_local(root, m, path, dry, vision=False):
src = pathlib.Path(path).resolve()
if not src.is_file():
@@ -208,7 +260,9 @@ def process_url(root, m, url, dry):
"added_at": _now(), "status": "converted",
"source_url": url, "fetched_at": _now()}
if not dry:
(originals / name).write_text(html, encoding="utf-8") # 完整快照(來源可能消失)
# write_bytes同 §_register_pair登記 hash 算在 html.encode() 上,
# 避免 Windows 換行轉換讓快照 bytes 與登記 hash 不符(完整快照,來源可能消失)
(originals / name).write_bytes(html.encode("utf-8"))
md_text, status = from_web.extract(html)
if status != "ok":
orig_entry["status"] = "pending"
@@ -223,7 +277,10 @@ def process_url(root, m, url, dry):
def main(argv=None):
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument("inputs", nargs="+", help="檔案路徑或 http(s) URL")
ap.add_argument("inputs", nargs="*", help="檔案路徑或 http(s) URL")
ap.add_argument("--verify", action="store_true",
help="對帳模式re-hash 全部登記檔、掃出未登記檔,純唯讀不改檔;"
"有不一致以退出碼 1 表示(不吃 inputs")
ap.add_argument("--dry-run", action="store_true")
ap.add_argument("--vision", action="store_true",
help="掃描/圖片型 PDFneeds_ocr改走本地 vision 模型轉錄,"
@@ -234,6 +291,11 @@ def main(argv=None):
a = ap.parse_args(argv)
root = pathlib.Path(a.root).resolve()
m = load_manifest(root)
if a.verify:
verify(root, m)
return
if not a.inputs:
ap.error("需指定至少一個檔案/URL或改用 --verify 對帳")
failed = []
for item in a.inputs:
try:

View File

@@ -158,6 +158,45 @@ def main():
assert (root / "raw/manifest.json").read_text(encoding="utf-8") == before
print("PASS: 冪等(同 hash 跳過)")
# --verify 對帳純唯讀clean 通過、三類意外都抓到、且完全不改 manifest。
# root 已有 docx/pdf/pptx 抽出的 assets不入帳本——clean 案例即證明 assets 不被誤報。
def run_verify(rt):
mm = json.loads((rt / "raw/manifest.json").read_text(encoding="utf-8"))
err, code = io.StringIO(), 0
with contextlib.redirect_stderr(err), contextlib.redirect_stdout(io.StringIO()):
try:
convert.verify(rt, mm)
except SystemExit as e:
code = e.code
return code, err.getvalue()
before = (root / "raw/manifest.json").read_text(encoding="utf-8")
code, _ = run_verify(root)
assert code == 0, "clean 帳本應通過對帳(含未入帳本的 assets"
rroot = tmp / "root_del" # missing刪掉一個 converted .md
shutil.copytree(root, rroot)
gone = next(p for p in (rroot / "raw/converted").glob("*.md"))
gone.unlink()
code, log = run_verify(rroot)
assert code == 1 and "missing" in log and gone.name in log, log
rroot = tmp / "root_mut" # mismatch就地改動一個 original 的 bytes
shutil.copytree(root, rroot)
tampered = next(p for p in (rroot / "raw/originals").iterdir() if p.is_file())
tampered.write_bytes(tampered.read_bytes() + b"tampered")
code, log = run_verify(rroot)
assert code == 1 and "mismatch" in log, log
rroot = tmp / "root_unreg" # unregisteredraw/originals 塞入未登記檔
shutil.copytree(root, rroot)
(rroot / "raw/originals/未登記 檔.docx").write_bytes(b"stray")
code, log = run_verify(rroot)
assert code == 1 and "unregistered" in log and "未登記 檔.docx" in log, log
assert (root / "raw/manifest.json").read_text(encoding="utf-8") == before # 全程唯讀
print("PASS: --verify 對帳clean/missing/mismatch/unregistered純唯讀")
# dry-run全新沙盒不落地
root2 = tmp / "root2"
for sub in ("raw/originals", "raw/converted"):