82 lines
2.6 KiB
Python
82 lines
2.6 KiB
Python
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
"""
|
|
从 Wolai 帮助中心导出语料中抽取 Markdown 大纲（仅标题行），用于快速定位“页面组成/页面操作”内容。
|
|
|
|
示例：
|
|
python scripts/wolai_help_center/extract_outlines.py --out artifacts/wolai-help-center-v4 --title-keyword 页面
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
from pathlib import Path
|
|
from typing import Any, Dict, List
|
|
|
|
|
|
def read_front_matter(md: str) -> Dict[str, Any]:
|
|
if not md.startswith("---"):
|
|
return {}
|
|
try:
|
|
meta_json = md.split("---", 2)[1].strip()
|
|
return json.loads(meta_json)
|
|
except Exception:
|
|
return {}
|
|
|
|
|
|
def extract_headings(md_body: str) -> List[str]:
|
|
lines: List[str] = []
|
|
for line in md_body.splitlines():
|
|
if line.startswith("#"):
|
|
lines.append(line.rstrip())
|
|
return lines
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--out", type=str, default="artifacts/wolai-help-center-v4", help="抓取输出目录")
|
|
parser.add_argument("--title-keyword", type=str, default="页面", help="仅输出标题包含该关键词的页面")
|
|
args = parser.parse_args()
|
|
|
|
out_dir = Path(args.out)
|
|
index_path = out_dir / "index.json"
|
|
if not index_path.exists():
|
|
raise SystemExit(f"未找到 {index_path}")
|
|
|
|
index = json.loads(index_path.read_text(encoding="utf-8"))
|
|
results: List[Dict[str, Any]] = index.get("results") or []
|
|
keyword = args.title_keyword.strip()
|
|
|
|
blocks: List[str] = []
|
|
for r in results:
|
|
md_rel = str(r.get("md") or "")
|
|
if not md_rel:
|
|
continue
|
|
md_path = out_dir / md_rel
|
|
if not md_path.exists():
|
|
continue
|
|
text = md_path.read_text(encoding="utf-8", errors="replace")
|
|
meta = read_front_matter(text)
|
|
title = str(meta.get("title") or "")
|
|
if keyword and keyword not in title:
|
|
continue
|
|
md_rel_norm = md_rel.replace("\\", "/")
|
|
body = text.split("---", 2)[2] if text.startswith("---") and len(text.split("---", 2)) == 3 else text
|
|
headings = extract_headings(body)
|
|
blocks.append(f"## {title}\n\n- pageId: {meta.get('pageId')}\n- md: {md_rel_norm}\n")
|
|
for h in headings:
|
|
blocks.append(f"{h}\n")
|
|
blocks.append("\n")
|
|
|
|
analysis_dir = out_dir / "analysis"
|
|
analysis_dir.mkdir(parents=True, exist_ok=True)
|
|
out_path = analysis_dir / f"outlines_{keyword or 'all'}.md"
|
|
out_path.write_text("".join(blocks), encoding="utf-8")
|
|
print(f"已写入：{out_path}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|