轉換工具強化：失敗診斷訊息 + 舊版格式升版 + 掃描 PDF vision fallback
- convert.py：轉換失敗改印例外型別、批次尾端彙總、--traceback 旗標； 空白轉換結果示警（避免純圖片 docx 等假成功靜默入庫） - from_legacy.py（B）：.doc/.xls/.ppt 經本地 LibreOffice headless 升版為 現代格式後再走既有轉換器；raw/originals 仍保存真正的舊格式原始檔 - to_image.py + from_vision.py（A）：needs_ocr 的掃描/圖片型 PDF 逐頁 render → 本地 vision 模型轉錄（--vision）；:cloud/未設定防護， 模型設定走 config/models.yaml 的 tasks.vision（預設不啟用） - kb.ollama_chat 加 images 參數（向後相容）；models.yaml 加 vision 範例 - selfcheck 補 legacy 路由、to_image 真測、from_vision 注入測、 --vision 端到端，全數 ALL PASS Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
LittleYellow
@@ -4,7 +4,9 @@
|
||||
原始檔存入 raw/originals/ 並登記 hash → 轉換 → Markdown 存入 raw/converted/
|
||||
並登記 hash 與原始檔對應 → 之後攝入管線只讀 converted 層。
|
||||
|
||||
用法：python tools/convert/convert.py <檔案路徑或URL>... [--dry-run] [--root DIR]
|
||||
用法：python tools/convert/convert.py <檔案路徑或URL>... [--vision] [--dry-run] [--root DIR]
|
||||
支援 .docx/.xlsx/.pdf/.pptx/.html 與 URL；舊版 .doc/.xls/.ppt 經 LibreOffice 升版；
|
||||
--vision 讓掃描/圖片型 PDF 走本地 vision 模型轉錄（見 from_vision.py）。
|
||||
"""
|
||||
import argparse
|
||||
import datetime
|
||||
@@ -12,11 +14,15 @@ import hashlib
|
||||
import json
|
||||
import pathlib
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import traceback
|
||||
import urllib.parse
|
||||
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).parent))
|
||||
import from_docx
|
||||
import from_legacy
|
||||
import from_pdf
|
||||
import from_pptx
|
||||
import from_web
|
||||
@@ -29,6 +35,10 @@ CONVERTERS = {
|
||||
".pdf": (from_pdf.convert, "from_pdf", "pdf"),
|
||||
".pptx": (from_pptx.convert, "from_pptx", "pptx"),
|
||||
}
|
||||
# 舊版 OLE 二進位 → 現代格式（經 LibreOffice 升版後走上表對應轉換器）。
|
||||
# <!-- ASSUMPTION: media_type 記真實舊格式（doc/xls/ppt），擴充 §9 的列舉；
|
||||
# converter 記為 "from_docx+libreoffice" 等，保留兩段式 provenance。 -->
|
||||
LEGACY_MAP = {".doc": ".docx", ".xls": ".xlsx", ".ppt": ".pptx"}
|
||||
|
||||
|
||||
def _now():
|
||||
@@ -72,6 +82,9 @@ def _dest_name(dir_, name, sha):
|
||||
|
||||
|
||||
def _register_pair(m, root, orig_rel, orig_entry, md_rel, md_text, converter, dry):
|
||||
if not md_text.strip():
|
||||
# 空轉換結果幾乎都是失敗（如純圖片 docx）；登記前示警，避免空頁靜默入庫
|
||||
print(f"warning: {orig_rel} 轉換結果為空白，請人工檢查來源", file=sys.stderr)
|
||||
if not dry:
|
||||
md_path = root / md_rel
|
||||
md_path.write_text(md_text, encoding="utf-8")
|
||||
@@ -87,49 +100,96 @@ def _register_pair(m, root, orig_rel, orig_entry, md_rel, md_text, converter, dr
|
||||
print(f"converted: {orig_rel} -> {md_rel}" + (" [dry-run]" if dry else ""))
|
||||
|
||||
|
||||
def process_local(root, m, path, dry):
|
||||
def _vision_pdf(root, pdf_path, assets, conv_name):
|
||||
"""--vision fallback（AGENTS.md §3 稽核原則）：needs_ocr 的 PDF 逐頁 render →
|
||||
本地 vision 模型轉錄。回傳 (markdown, status, converter)。
|
||||
|
||||
成功 → ('...', 'ok', 'from_vision')；vision 執行期失敗 → 降級回 needs_ocr（不遺失檔案）。
|
||||
設定錯誤（未設 vision / :cloud）由 from_vision 拋 SystemExit，向上傳播不靜默降級。
|
||||
"""
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1])) # tools/ 供 import kb
|
||||
import kb
|
||||
import to_image
|
||||
import from_vision
|
||||
cfg = kb.load_config(root)
|
||||
md_dir = root / "raw" / "converted"
|
||||
try:
|
||||
images = to_image.render_pdf_pages(pdf_path, assets)
|
||||
texts, model = from_vision.transcribe(cfg, images)
|
||||
except SystemExit:
|
||||
raise
|
||||
except Exception as e:
|
||||
print(f"warning: {pathlib.Path(pdf_path).name} vision 轉錄失敗，降級為 needs_ocr："
|
||||
f"{type(e).__name__}: {e}", file=sys.stderr)
|
||||
return "", "needs_ocr", conv_name
|
||||
blocks = []
|
||||
for i, (img, text) in enumerate(zip(images, texts), 1):
|
||||
rel = img.relative_to(md_dir).as_posix()
|
||||
blocks.append(f"<!-- page {i} (vision: {model}) -->\n\n\n\n{text}")
|
||||
return "\n\n".join(blocks), "ok", "from_vision"
|
||||
|
||||
|
||||
def process_local(root, m, path, dry, vision=False):
|
||||
src = pathlib.Path(path).resolve()
|
||||
if not src.is_file():
|
||||
raise FileNotFoundError(src)
|
||||
ext = src.suffix.lower()
|
||||
if ext in (".html", ".htm"):
|
||||
conv, conv_name, media = (
|
||||
lambda p, assets_dir=None, md_dir=None: from_web.extract(
|
||||
pathlib.Path(p).read_text(encoding="utf-8", errors="replace")),
|
||||
"from_web", "html")
|
||||
elif ext in CONVERTERS:
|
||||
conv, conv_name, media = CONVERTERS[ext]
|
||||
else:
|
||||
raise ValueError(f"不支援的格式：{src.name}")
|
||||
|
||||
sha = hashlib.sha256(src.read_bytes()).hexdigest()
|
||||
if _find_by_sha(m, sha):
|
||||
print(f"skip: {src.name} 已登記（hash 相同）")
|
||||
return
|
||||
|
||||
# raw/originals 一律保存「真正的原始檔」（舊格式也是），維持 provenance（§1.4）
|
||||
originals = root / "raw" / "originals"
|
||||
if src.parent == originals.resolve():
|
||||
dest = src
|
||||
else:
|
||||
dest = originals / _dest_name(originals, src.name, sha)
|
||||
if not dry and not dest.exists():
|
||||
dest.write_bytes(src.read_bytes())
|
||||
dest = src if src.parent == originals.resolve() else originals / _dest_name(originals, src.name, sha)
|
||||
orig_rel = f"raw/originals/{dest.name}"
|
||||
orig_entry = {"kind": "original", "sha256": sha, "media_type": media,
|
||||
"added_at": _now(), "status": "converted"}
|
||||
|
||||
md_name = f"{dest.stem}-{sha[:8]}.md"
|
||||
assets = None if dry else root / "raw" / "converted" / "assets" / f"{dest.stem}-{sha[:8]}"
|
||||
md_text, status = conv(str(src), assets_dir=assets, md_dir=root / "raw" / "converted")
|
||||
if status == "needs_ocr":
|
||||
orig_entry["status"] = "needs_ocr"
|
||||
if not dry:
|
||||
m["files"][orig_rel] = orig_entry
|
||||
print(f"needs_ocr: {orig_rel} 無文字層，已登記並跳過（不擅自 OCR）"
|
||||
+ (" [dry-run]" if dry else ""))
|
||||
return
|
||||
_register_pair(m, root, orig_rel, orig_entry, f"raw/converted/{md_name}",
|
||||
md_text, conv_name, dry)
|
||||
tmpdir = None
|
||||
try:
|
||||
if ext in LEGACY_MAP:
|
||||
if dry:
|
||||
print(f"legacy: {src.name} 需 LibreOffice 升版為 {LEGACY_MAP[ext]}（dry-run 不執行）")
|
||||
return
|
||||
tmpdir = tempfile.mkdtemp(prefix="ppqa-legacy-")
|
||||
upgraded = from_legacy.upgrade(src, tmpdir) # 找不到 soffice / 升版失敗 → RuntimeError
|
||||
conv, base_name, _ = CONVERTERS[upgraded.suffix.lower()]
|
||||
conv_name, media, conv_src = f"{base_name}+libreoffice", ext.lstrip("."), upgraded
|
||||
elif ext in (".html", ".htm"):
|
||||
conv = (lambda p, assets_dir=None, md_dir=None: from_web.extract(
|
||||
pathlib.Path(p).read_text(encoding="utf-8", errors="replace")))
|
||||
conv_name, media, conv_src = "from_web", "html", src
|
||||
elif ext in CONVERTERS:
|
||||
conv, conv_name, media = CONVERTERS[ext]
|
||||
conv_src = src
|
||||
else:
|
||||
raise ValueError(f"不支援的格式：{src.name}")
|
||||
|
||||
if not dry and dest is not src and not dest.exists():
|
||||
dest.write_bytes(src.read_bytes())
|
||||
orig_entry = {"kind": "original", "sha256": sha, "media_type": media,
|
||||
"added_at": _now(), "status": "converted"}
|
||||
|
||||
md_name = f"{dest.stem}-{sha[:8]}.md"
|
||||
assets = None if dry else root / "raw" / "converted" / "assets" / f"{dest.stem}-{sha[:8]}"
|
||||
md_text, status = conv(str(conv_src), assets_dir=assets,
|
||||
md_dir=root / "raw" / "converted")
|
||||
if status == "needs_ocr":
|
||||
if vision and not dry:
|
||||
md_text, status, conv_name = _vision_pdf(root, conv_src, assets, conv_name)
|
||||
if status == "needs_ocr":
|
||||
orig_entry["status"] = "needs_ocr"
|
||||
if not dry:
|
||||
m["files"][orig_rel] = orig_entry
|
||||
print(f"needs_ocr: {orig_rel} 無文字層，已登記並跳過"
|
||||
+ ("（--vision 轉錄未成功）" if vision and not dry else "（不擅自 OCR）")
|
||||
+ (" [dry-run]" if dry else ""))
|
||||
return
|
||||
_register_pair(m, root, orig_rel, orig_entry, f"raw/converted/{md_name}",
|
||||
md_text, conv_name, dry)
|
||||
finally:
|
||||
if tmpdir:
|
||||
shutil.rmtree(tmpdir, ignore_errors=True)
|
||||
|
||||
|
||||
def process_url(root, m, url, dry):
|
||||
@@ -164,6 +224,11 @@ def main(argv=None):
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("inputs", nargs="+", help="檔案路徑或 http(s) URL")
|
||||
ap.add_argument("--dry-run", action="store_true")
|
||||
ap.add_argument("--vision", action="store_true",
|
||||
help="掃描/圖片型 PDF（needs_ocr）改走本地 vision 模型轉錄，"
|
||||
"而非跳過（需 config/models.yaml 設定 tasks.vision）")
|
||||
ap.add_argument("--traceback", action="store_true",
|
||||
help="轉換失敗時額外印出完整堆疊，供定位除錯")
|
||||
ap.add_argument("--root", default=str(ROOT), help="專案根目錄（測試用）")
|
||||
a = ap.parse_args(argv)
|
||||
root = pathlib.Path(a.root).resolve()
|
||||
@@ -174,13 +239,22 @@ def main(argv=None):
|
||||
if item.startswith(("http://", "https://")):
|
||||
process_url(root, m, item, a.dry_run)
|
||||
else:
|
||||
process_local(root, m, item, a.dry_run)
|
||||
process_local(root, m, item, a.dry_run, vision=a.vision)
|
||||
except Exception as e: # 單檔失敗不中斷批次，最後彙總報錯
|
||||
failed.append((item, e))
|
||||
print(f"error: {item}: {e}", file=sys.stderr)
|
||||
# 帶例外型別：只印訊息字串常無法判斷成因（如 KeyError 只印 key）
|
||||
print(f"error: {item}: {type(e).__name__}: {e}", file=sys.stderr)
|
||||
if a.traceback:
|
||||
traceback.print_exc()
|
||||
if not a.dry_run:
|
||||
save_manifest(root, m)
|
||||
if failed:
|
||||
# 尾端彙總，避免失敗行淹沒在大批次輸出中而漏看
|
||||
print(f"\n{len(failed)} 個項目轉換失敗：", file=sys.stderr)
|
||||
for item, e in failed:
|
||||
print(f" - {item}: {type(e).__name__}: {e}", file=sys.stderr)
|
||||
if not a.traceback:
|
||||
print("（加 --traceback 可印完整堆疊定位）", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user