54 lines
1.6 KiB
Python
54 lines
1.6 KiB
Python
"""網頁 URL → 完整 HTML 快照 + Markdown 正文萃取（trafilatura，全程本地處理）。
|
||
|
||
可獨立執行：python tools/convert/from_web.py <url> [-o out.md] [--snapshot out.html] [--dry-run]
|
||
"""
|
||
import argparse
|
||
import pathlib
|
||
|
||
import trafilatura
|
||
|
||
|
||
def fetch(url):
|
||
"""抓取 URL，回傳完整 HTML 字串。失敗時拋 RuntimeError。"""
|
||
html = trafilatura.fetch_url(url)
|
||
if not html:
|
||
raise RuntimeError(f"抓取失敗：{url}")
|
||
return html
|
||
|
||
|
||
def extract(html):
|
||
"""HTML → Markdown 正文。回傳 (markdown, status)。萃取不到正文時 status='empty'。"""
|
||
md = trafilatura.extract(
|
||
html, output_format="markdown", include_tables=True, include_links=True
|
||
)
|
||
if not md or not md.strip():
|
||
return "", "empty"
|
||
return md.strip(), "ok"
|
||
|
||
|
||
def main(argv=None):
|
||
ap = argparse.ArgumentParser(description=__doc__)
|
||
ap.add_argument("url")
|
||
ap.add_argument("-o", "--output")
|
||
ap.add_argument("--snapshot", help="HTML 快照輸出路徑")
|
||
ap.add_argument("--dry-run", action="store_true")
|
||
a = ap.parse_args(argv)
|
||
html = fetch(a.url)
|
||
md, status = extract(html)
|
||
if a.dry_run:
|
||
print(f"[dry-run] {a.url}: status={status}, snapshot {len(html)} chars, "
|
||
f"markdown {len(md)} chars")
|
||
return
|
||
if status != "ok":
|
||
print(f"empty: {a.url} 萃取不到正文")
|
||
raise SystemExit(2)
|
||
if a.snapshot:
|
||
pathlib.Path(a.snapshot).write_text(html, encoding="utf-8")
|
||
out = pathlib.Path(a.output) if a.output else pathlib.Path("page.md")
|
||
out.write_text(md, encoding="utf-8")
|
||
print(out)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|