yellowadmin/pp-qa-km
1
0
Fork 0
Code Issues 5 Pull Requests Actions Packages Projects Releases Wiki Activity

轉換工具強化：失敗診斷訊息 + 舊版格式升版 + 掃描 PDF vision fallback

Browse Source 
- convert.py：轉換失敗改印例外型別、批次尾端彙總、--traceback 旗標；
  空白轉換結果示警（避免純圖片 docx 等假成功靜默入庫）
- from_legacy.py（B）：.doc/.xls/.ppt 經本地 LibreOffice headless 升版為
  現代格式後再走既有轉換器；raw/originals 仍保存真正的舊格式原始檔
- to_image.py + from_vision.py（A）：needs_ocr 的掃描/圖片型 PDF 逐頁
  render → 本地 vision 模型轉錄（--vision）；:cloud/未設定防護，
  模型設定走 config/models.yaml 的 tasks.vision（預設不啟用）
- kb.ollama_chat 加 images 參數（向後相容）；models.yaml 加 vision 範例
- selfcheck 補 legacy 路由、to_image 真測、from_vision 注入測、
  --vision 端到端，全數 ALL PASS

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
LittleYellow
2026-07-21 07:00:32 +08:00
parent 1d1454e1b9
commit cfd1876359
7 changed files with 428 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

106
tools/convert/selfcheck.py
View File

@@ -3,19 +3,25 @@
執行：.venv/Scripts/python tools/convert/selfcheck.py
邏輯壞掉時 assert 會失敗（AGENTS.md §13.4：一個可執行的檢查，不用框架）。
"""
import contextlib
import io
import json
import pathlib
import shutil
import sys
import tempfile
sys.path.insert(0, str(pathlib.Path(__file__).parent))
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1])) # tools/ 供 import kb
sys.path.insert(0, str(pathlib.Path(__file__).parent)) # tools/convert
import convert
import from_docx
import from_pdf
import from_pptx
import from_vision
import from_web
import from_xlsx
import kb
import to_image
def make_fixtures(d):
@@ -118,6 +124,104 @@ def main():
assert m2["files"] == {} and not list((root2 / "raw/originals").iterdir())
print("PASS: --dry-run 不落地")
# 失敗回報：壞輸入不中斷批次，錯誤帶例外型別 + 尾端彙總，好檔仍轉換
root3 = tmp / "root3"
for sub in ("raw/originals", "raw/converted"):
(root3 / sub).mkdir(parents=True)
(root3 / "raw/manifest.json").write_text('{"version": 1, "files": {}}', encoding="utf-8")
(fx / "notes.txt").write_text("不支援的格式", encoding="utf-8") # → ValueError
err = io.StringIO()
code = None
with contextlib.redirect_stderr(err):
try:
convert.main([str(fx / "report.docx"), str(fx / "notes.txt"),
str(fx / "missing.docx"), # 不存在 → FileNotFoundError
"--root", str(root3)])
except SystemExit as e:
code = e.code
log = err.getvalue()
assert code == 1, code
assert "ValueError" in log and "FileNotFoundError" in log, log # 錯誤帶例外型別
assert "2 個項目轉換失敗" in log, log # 尾端彙總
m3 = json.loads((root3 / "raw/manifest.json").read_text(encoding="utf-8"))
assert (root3 / "raw/originals/report.docx").is_file() # 好檔仍轉換並落地
assert any(v["kind"] == "converted" for v in m3["files"].values()), m3
print("PASS: 失敗回報（例外型別 + 尾端彙總，批次不中斷）")
# B: 舊格式路由 —— .doc 應走 legacy（LibreOffice），而非被當「不支援格式」拒收
root4 = tmp / "root4"
for sub in ("raw/originals", "raw/converted"):
(root4 / sub).mkdir(parents=True)
(root4 / "raw/manifest.json").write_text('{"version": 1, "files": {}}', encoding="utf-8")
(fx / "old.doc").write_bytes(b"\xd0\xcf\x11\xe0legacy-stub") # 非有效 .doc，僅測路由/錯誤
err = io.StringIO()
code = None
with contextlib.redirect_stderr(err):
try:
convert.main([str(fx / "old.doc"), "--root", str(root4)])
except SystemExit as e:
code = e.code
log = err.getvalue()
assert code == 1 and "old.doc" in log, log
assert "不支援的格式" not in log and "RuntimeError" in log, log # 已路由 legacy，乾淨報錯
print("PASS: 舊格式路由至 legacy（soffice 缺失/升版失敗皆乾淨報錯）")
# A: to_image render（deterministic，真跑）
imgs = to_image.render_pdf_pages(fx / "text.pdf", tmp / "imgs")
assert imgs and all(p.is_file() and p.stat().st_size > 0 for p in imgs), imgs
print(f"PASS: to_image render（{len(imgs)} 頁 PNG）")
# A: from_vision 轉錄邏輯（注入假 Ollama，不需真模型）+ 合規防護
vis_cfg = {"tasks": {"vision": {"model": "fake-vision", "temperature": 0.1,
"max_retries": 1}}}
orig_chat = kb.ollama_chat
calls = {}
def fake_chat(cfg, model, prompt, temperature, json_format=False, images=None):
calls["images"] = images # 記錄 base64 是否傳入
return "可疑交易監控：大額 12,000 元"
kb.ollama_chat = fake_chat
try:
texts, model = from_vision.transcribe(vis_cfg, imgs[:1])
finally:
kb.ollama_chat = orig_chat
assert model == "fake-vision" and texts and "可疑交易監控" in texts[0], texts
assert calls.get("images"), "vision 呼叫必須帶影像"
for bad in ({"tasks": {}}, {"tasks": {"vision": {"model": "foo:cloud"}}}):
try:
from_vision._require_vision(bad)
assert False, "應拒絕未設定 / :cloud"
except SystemExit:
pass
print("PASS: from_vision 轉錄 + :cloud/未設定防護")
# A: convert.py --vision 端到端（沙盒 config + 注入假 vision）
root_v = tmp / "root_v"
(root_v / "config").mkdir(parents=True)
for sub in ("raw/originals", "raw/converted"):
(root_v / sub).mkdir(parents=True)
(root_v / "config/models.yaml").write_text(
'ollama:\n base_url: "http://localhost:11434"\n timeout_seconds: 300\n'
'tasks:\n ingest: {model: "x", temperature: 0.2, max_retries: 1}\n'
' lint: {model: "x", temperature: 0.1, max_retries: 1}\n'
' fallback: {model: "x"}\n'
' vision: {model: "fake-vision", temperature: 0.1, max_retries: 1}\n'
'limits:\n mcp_response_max_tokens: 2000\n ingest_chunk_max_chars: 12000\n',
encoding="utf-8")
(root_v / "raw/manifest.json").write_text('{"version": 1, "files": {}}', encoding="utf-8")
kb.ollama_chat = fake_chat
try:
convert.main([str(fx / "scanned.pdf"), "--vision", "--root", str(root_v)])
finally:
kb.ollama_chat = orig_chat
mv = json.loads((root_v / "raw/manifest.json").read_text(encoding="utf-8"))
conv_rel = [k for k, v in mv["files"].items() if v["kind"] == "converted"]
assert len(conv_rel) == 1 and mv["files"][conv_rel[0]]["converter"] == "from_vision", mv
md_v = (root_v / conv_rel[0]).read_text(encoding="utf-8")
assert "可疑交易監控" in md_v and "(vision" in md_v and "![page 1]" in md_v, md_v
print("PASS: convert.py --vision 端到端（needs_ocr → 本地 vision 轉錄）")
print("ALL PASS")
finally:
shutil.rmtree(tmp, ignore_errors=True)