104 lines
2.8 KiB
Python
104 lines
2.8 KiB
Python
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
"""
|
|
在已导出的 Wolai 帮助中心语料中做关键词命中统计，方便定位“页面组成/页面操作”相关页面。
|
|
|
|
示例：
|
|
python scripts/wolai_help_center/build_keyword_hits.py --out artifacts/wolai-help-center-v4
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
from pathlib import Path
|
|
from typing import Any, Dict, List
|
|
|
|
|
|
DEFAULT_KEYWORDS = [
|
|
"页面",
|
|
"页面选项",
|
|
"自适应宽度",
|
|
"小字体",
|
|
"标题目录",
|
|
"标题自动编号",
|
|
"编辑保护",
|
|
"复制链接",
|
|
"导出",
|
|
"移动",
|
|
"嵌入",
|
|
"页面历史",
|
|
"字数",
|
|
"统计",
|
|
"权限",
|
|
"共享",
|
|
"公开",
|
|
"私有",
|
|
"删除",
|
|
"恢复",
|
|
"撤回",
|
|
"回收站",
|
|
"关系图",
|
|
"反向链接",
|
|
]
|
|
|
|
|
|
def read_front_matter_title(md: str) -> str:
|
|
if not md.startswith("---"):
|
|
return ""
|
|
try:
|
|
meta_json = md.split("---", 2)[1].strip()
|
|
meta = json.loads(meta_json)
|
|
return str(meta.get("title") or "")
|
|
except Exception:
|
|
return ""
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--out", type=str, default="artifacts/wolai-help-center-v4", help="抓取输出目录")
|
|
parser.add_argument("--keywords", type=str, default="", help="自定义关键词（用逗号分隔）")
|
|
args = parser.parse_args()
|
|
|
|
out_dir = Path(args.out)
|
|
index_path = out_dir / "index.json"
|
|
if not index_path.exists():
|
|
raise SystemExit(f"未找到 {index_path}")
|
|
|
|
index = json.loads(index_path.read_text(encoding="utf-8"))
|
|
results: List[Dict[str, Any]] = index.get("results") or []
|
|
keywords = (
|
|
[x.strip() for x in args.keywords.split(",") if x.strip()]
|
|
if args.keywords.strip()
|
|
else DEFAULT_KEYWORDS
|
|
)
|
|
|
|
hits: Dict[str, List[Dict[str, str]]] = {k: [] for k in keywords}
|
|
for r in results:
|
|
pid = str(r.get("pageId") or "")
|
|
md_rel = str(r.get("md") or "")
|
|
if not pid or not md_rel:
|
|
continue
|
|
md_path = out_dir / md_rel
|
|
if not md_path.exists():
|
|
continue
|
|
md = md_path.read_text(encoding="utf-8", errors="replace")
|
|
title = read_front_matter_title(md)
|
|
for k in keywords:
|
|
if k in md:
|
|
hits[k].append(
|
|
{"pageId": pid, "title": title, "md": md_rel.replace("\\", "/")}
|
|
)
|
|
|
|
analysis_dir = out_dir / "analysis"
|
|
analysis_dir.mkdir(parents=True, exist_ok=True)
|
|
out_path = analysis_dir / "keyword_hits.json"
|
|
out_path.write_text(json.dumps(hits, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
print(f"已写入：{out_path}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|
|
|