347 lines
12 KiB
Python
347 lines
12 KiB
Python
|
#!/usr/bin/env python3
|
|||
|
|
"""Export public Wolai pages listed in scraped_docs/网址.md to Markdown."""
|
|||
|
|
from __future__ import annotations
|
|||
|
|
|
|||
|
|
import json
|
|||
|
|
import os
|
|||
|
|
import re
|
|||
|
|
from collections import deque
|
|||
|
|
from dataclasses import dataclass
|
|||
|
|
from pathlib import Path
|
|||
|
|
from typing import Dict, Iterable, List, Sequence
|
|||
|
|
from urllib.parse import quote
|
|||
|
|
|
|||
|
|
import requests
|
|||
|
|
|
|||
|
|
|
|||
|
|
URL_LIST_PATH = Path("scraped_docs/网址.md")
|
|||
|
|
OUTPUT_DIR = Path("scraped_docs")
|
|||
|
|
COOKIE_ENV = "WOLAI_COOKIE"
|
|||
|
|
|
|||
|
|
|
|||
|
|
def load_page_mapping(path: Path) -> Dict[str, str]:
|
|||
|
|
content = path.read_text(encoding="utf-8").strip()
|
|||
|
|
entries = re.split(r"[；;\n]", content)
|
|||
|
|
mapping: Dict[str, str] = {}
|
|||
|
|
for entry in entries:
|
|||
|
|
entry = entry.strip()
|
|||
|
|
if not entry or "：" not in entry:
|
|||
|
|
continue
|
|||
|
|
name, url = entry.split("：", 1)
|
|||
|
|
name = name.strip()
|
|||
|
|
url = url.strip()
|
|||
|
|
if name and url.startswith("https://"):
|
|||
|
|
mapping[name] = url
|
|||
|
|
return mapping
|
|||
|
|
|
|||
|
|
|
|||
|
|
def sanitize_filename(name: str) -> str:
|
|||
|
|
return re.sub(r"[\\/:\*\?\"<>\|]", "_", name)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def split_batches(items: Iterable[str], size: int) -> Iterable[List[str]]:
|
|||
|
|
batch: List[str] = []
|
|||
|
|
for item in items:
|
|||
|
|
batch.append(item)
|
|||
|
|
if len(batch) >= size:
|
|||
|
|
yield batch
|
|||
|
|
batch = []
|
|||
|
|
if batch:
|
|||
|
|
yield batch
|
|||
|
|
|
|||
|
|
|
|||
|
|
class WolaiClient:
|
|||
|
|
API_URL = "https://api.wolai.com/v1/pages/getData"
|
|||
|
|
|
|||
|
|
def __init__(self, cookie: str) -> None:
|
|||
|
|
headers = {
|
|||
|
|
"wolai-client-platform": "web",
|
|||
|
|
"wolai-app-version": "1.2.3-15",
|
|||
|
|
"wolai-os-platform": "win",
|
|||
|
|
"Origin": "https://www.wolai.com",
|
|||
|
|
"Referer": "https://www.wolai.com/",
|
|||
|
|
"User-Agent": (
|
|||
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
|||
|
|
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
|||
|
|
"Chrome/120.0.0.0 Safari/537.36"
|
|||
|
|
),
|
|||
|
|
"Accept": "application/json, text/plain, */*",
|
|||
|
|
"Cookie": cookie,
|
|||
|
|
}
|
|||
|
|
self.session = requests.Session()
|
|||
|
|
self.session.headers.update(headers)
|
|||
|
|
|
|||
|
|
def fetch_blocks(self, page_id: str) -> Dict[str, dict]:
|
|||
|
|
blocks: Dict[str, dict] = {}
|
|||
|
|
queue: deque[str] = deque([page_id])
|
|||
|
|
seen: set[str] = set()
|
|||
|
|
while queue:
|
|||
|
|
chunk: List[str] = []
|
|||
|
|
while queue and len(chunk) < 50:
|
|||
|
|
bid = queue.popleft()
|
|||
|
|
if bid in seen:
|
|||
|
|
continue
|
|||
|
|
seen.add(bid)
|
|||
|
|
chunk.append(bid)
|
|||
|
|
if not chunk:
|
|||
|
|
continue
|
|||
|
|
payload = {"requests": [{"table": "wolai.block", "id": bid} for bid in chunk]}
|
|||
|
|
resp = self.session.post(self.API_URL, json=payload, timeout=30)
|
|||
|
|
resp.raise_for_status()
|
|||
|
|
for entry in resp.json().get("data", []):
|
|||
|
|
value = entry.get("value")
|
|||
|
|
if not value:
|
|||
|
|
continue
|
|||
|
|
blocks[value["id"]] = value
|
|||
|
|
for child in value.get("sub_nodes") or []:
|
|||
|
|
if child not in seen:
|
|||
|
|
queue.append(child)
|
|||
|
|
return blocks
|
|||
|
|
|
|||
|
|
|
|||
|
|
def get_title_fragments(block: dict) -> List[List]:
|
|||
|
|
attrs = block.get("attributes") or {}
|
|||
|
|
title = attrs.get("title")
|
|||
|
|
if isinstance(title, str):
|
|||
|
|
return [[title]]
|
|||
|
|
return title or []
|
|||
|
|
|
|||
|
|
|
|||
|
|
def rich_text(fragments: Sequence[Sequence]) -> str:
|
|||
|
|
result: List[str] = []
|
|||
|
|
for fragment in fragments:
|
|||
|
|
if not fragment:
|
|||
|
|
continue
|
|||
|
|
text = fragment[0]
|
|||
|
|
marks = fragment[1] if len(fragment) > 1 else None
|
|||
|
|
result.append(apply_marks(text, marks))
|
|||
|
|
return "".join(result).strip()
|
|||
|
|
|
|||
|
|
|
|||
|
|
def apply_marks(text: str, marks: Sequence[Sequence] | None) -> str:
|
|||
|
|
if not marks:
|
|||
|
|
return text
|
|||
|
|
decorated = text
|
|||
|
|
link_target: str | None = None
|
|||
|
|
for mark in marks:
|
|||
|
|
kind = mark[0]
|
|||
|
|
if kind == "B":
|
|||
|
|
decorated = f"**{decorated}**"
|
|||
|
|
elif kind == "I":
|
|||
|
|
decorated = f"*{decorated}*"
|
|||
|
|
elif kind == "S":
|
|||
|
|
decorated = f"~~{decorated}~~"
|
|||
|
|
elif kind == "<>":
|
|||
|
|
decorated = f"`{decorated}`"
|
|||
|
|
elif kind == "Link" and len(mark) > 1:
|
|||
|
|
link_target = mark[1]
|
|||
|
|
elif kind == "BiLink":
|
|||
|
|
# 链接到空间内其他块，保留文本即可
|
|||
|
|
continue
|
|||
|
|
elif kind == "h":
|
|||
|
|
continue
|
|||
|
|
else:
|
|||
|
|
continue
|
|||
|
|
if link_target:
|
|||
|
|
decorated = f"[{decorated}]({link_target})"
|
|||
|
|
return decorated
|
|||
|
|
|
|||
|
|
|
|||
|
|
def build_image_url(block: dict) -> str | None:
|
|||
|
|
attrs = block.get("attributes") or {}
|
|||
|
|
img = attrs.get("img") or []
|
|||
|
|
if not img or not img[0]:
|
|||
|
|
return None
|
|||
|
|
path = img[0][0]
|
|||
|
|
if path.startswith("http"):
|
|||
|
|
return path
|
|||
|
|
safe_path = quote(path, safe="/%")
|
|||
|
|
return f"https://secure2.wostatic.cn/{safe_path}"
|
|||
|
|
|
|||
|
|
|
|||
|
|
def indent_lines(lines: Sequence[str], prefix: str) -> List[str]:
|
|||
|
|
indented: List[str] = []
|
|||
|
|
for line in lines:
|
|||
|
|
if line:
|
|||
|
|
indented.append(f"{prefix}{line}")
|
|||
|
|
else:
|
|||
|
|
indented.append("")
|
|||
|
|
return indented
|
|||
|
|
|
|||
|
|
|
|||
|
|
@dataclass
|
|||
|
|
class MarkdownRenderer:
|
|||
|
|
blocks: Dict[str, dict]
|
|||
|
|
|
|||
|
|
def render_page(self, page_id: str) -> str:
|
|||
|
|
root = self.blocks.get(page_id)
|
|||
|
|
if not root:
|
|||
|
|
return ""
|
|||
|
|
lines = self.render_children(root.get("sub_nodes") or [])
|
|||
|
|
# 移除多余空行并确保结尾有换行符
|
|||
|
|
while lines and not lines[-1].strip():
|
|||
|
|
lines.pop()
|
|||
|
|
return "\n".join(lines) + "\n"
|
|||
|
|
|
|||
|
|
def render_children(self, child_ids: Sequence[str]) -> List[str]:
|
|||
|
|
lines: List[str] = []
|
|||
|
|
i = 0
|
|||
|
|
while i < len(child_ids):
|
|||
|
|
block = self.blocks.get(child_ids[i])
|
|||
|
|
i += 1
|
|||
|
|
if not block:
|
|||
|
|
continue
|
|||
|
|
btype = block["type"]
|
|||
|
|
if btype in {"enumList", "bullList", "todoList", "todoListPro"}:
|
|||
|
|
seq_lines, new_index = self.render_list_sequence(child_ids, i - 1)
|
|||
|
|
lines.extend(seq_lines)
|
|||
|
|
i = new_index
|
|||
|
|
continue
|
|||
|
|
block_lines = self.render_block(block)
|
|||
|
|
if not block_lines:
|
|||
|
|
continue
|
|||
|
|
lines.extend(block_lines)
|
|||
|
|
if block_lines[-1].strip():
|
|||
|
|
lines.append("")
|
|||
|
|
return lines
|
|||
|
|
|
|||
|
|
def render_list_sequence(self, ids: Sequence[str], start: int) -> tuple[List[str], int]:
|
|||
|
|
lines: List[str] = []
|
|||
|
|
i = start
|
|||
|
|
first_type = self.blocks[ids[start]]["type"]
|
|||
|
|
group = {
|
|||
|
|
"enumList": {"enumList"},
|
|||
|
|
"bullList": {"bullList"},
|
|||
|
|
"todoList": {"todoList", "todoListPro"},
|
|||
|
|
"todoListPro": {"todoList", "todoListPro"},
|
|||
|
|
}[first_type]
|
|||
|
|
counter = 1
|
|||
|
|
while i < len(ids):
|
|||
|
|
block = self.blocks.get(ids[i])
|
|||
|
|
if not block or block["type"] not in group:
|
|||
|
|
break
|
|||
|
|
text = rich_text(get_title_fragments(block))
|
|||
|
|
if block["type"] == "enumList":
|
|||
|
|
prefix = f"{counter}. "
|
|||
|
|
elif block["type"] in {"todoList", "todoListPro"}:
|
|||
|
|
checked = block.get("attributes", {}).get("checked") == "yes"
|
|||
|
|
prefix = f"- [{'x' if checked else ' '}] "
|
|||
|
|
else:
|
|||
|
|
prefix = "- "
|
|||
|
|
lines.append(f"{prefix}{text}")
|
|||
|
|
child_lines = self.render_children(block.get("sub_nodes") or [])
|
|||
|
|
if child_lines:
|
|||
|
|
lines.extend(indent_lines(child_lines, " "))
|
|||
|
|
counter += 1
|
|||
|
|
i += 1
|
|||
|
|
lines.append("")
|
|||
|
|
return lines, i
|
|||
|
|
|
|||
|
|
def render_block(self, block: dict) -> List[str]:
|
|||
|
|
btype = block["type"]
|
|||
|
|
text = rich_text(get_title_fragments(block))
|
|||
|
|
if btype == "text":
|
|||
|
|
return [text] if text else []
|
|||
|
|
if btype in {"midHeader", "subHeader", "tinyHeader"}:
|
|||
|
|
level = {"midHeader": "##", "subHeader": "###", "tinyHeader": "####"}[btype]
|
|||
|
|
return [f"{level} {text}"]
|
|||
|
|
if btype == "quote":
|
|||
|
|
quote_lines = [f"> {line}" if line else ">" for line in text.splitlines() or [""]]
|
|||
|
|
child_lines = self.render_children(block.get("sub_nodes") or [])
|
|||
|
|
if child_lines:
|
|||
|
|
quote_lines.extend(f"> {line}" if line else ">" for line in child_lines)
|
|||
|
|
return quote_lines
|
|||
|
|
if btype == "divider":
|
|||
|
|
return ["---"]
|
|||
|
|
if btype == "image":
|
|||
|
|
url = build_image_url(block)
|
|||
|
|
return [f"" if url else ""]
|
|||
|
|
if btype == "code":
|
|||
|
|
language = block.get("attributes", {}).get("language") or ""
|
|||
|
|
body = text
|
|||
|
|
fence = f"```{language.lower()}" if language else "```"
|
|||
|
|
return [fence, body, "```"]
|
|||
|
|
if btype in {"row", "column"}:
|
|||
|
|
return self.render_children(block.get("sub_nodes") or [])
|
|||
|
|
if btype == "enumList":
|
|||
|
|
return [f"1. {text}"]
|
|||
|
|
if btype == "bullList":
|
|||
|
|
return [f"- {text}"]
|
|||
|
|
if btype in {"todoList", "todoListPro"}:
|
|||
|
|
checked = block.get("attributes", {}).get("checked") == "yes"
|
|||
|
|
return [f"- [{'x' if checked else ' '}] {text}"]
|
|||
|
|
if btype == "toggleList":
|
|||
|
|
title = text or "折叠列表"
|
|||
|
|
child_lines = indent_lines(self.render_children(block.get("sub_nodes") or []), " ")
|
|||
|
|
return [f"- **{title}**", *child_lines]
|
|||
|
|
if btype in {"toggleSubHeader", "toggleTinyHeader"}:
|
|||
|
|
level = "####" if btype == "toggleSubHeader" else "#####"
|
|||
|
|
title = text + "（可折叠）"
|
|||
|
|
child_lines = self.render_children(block.get("sub_nodes") or [])
|
|||
|
|
return [f"{level} {title}", *child_lines]
|
|||
|
|
if btype == "simpleTable":
|
|||
|
|
return self.render_table(block)
|
|||
|
|
if btype == "progressBar":
|
|||
|
|
progress = block.get("attributes", {}).get("progress", 0)
|
|||
|
|
return [f"进度条：{progress}%"]
|
|||
|
|
if btype == "xiguaVideo":
|
|||
|
|
source = block.get("attributes", {}).get("source") or ""
|
|||
|
|
label = text or "西瓜视频"
|
|||
|
|
return [f"[{label}]({source})" if source else label]
|
|||
|
|
return [text] if text else []
|
|||
|
|
|
|||
|
|
def render_table(self, block: dict) -> List[str]:
|
|||
|
|
attrs = block.get("attributes") or {}
|
|||
|
|
raw = attrs.get("cells")
|
|||
|
|
if not raw:
|
|||
|
|
return []
|
|||
|
|
try:
|
|||
|
|
cells = json.loads(raw)
|
|||
|
|
except json.JSONDecodeError:
|
|||
|
|
return []
|
|||
|
|
rows: List[List[str]] = []
|
|||
|
|
for row in cells:
|
|||
|
|
cols = []
|
|||
|
|
for cell in row.get("column", []):
|
|||
|
|
cols.append(rich_text(cell.get("attributes", {}).get("title") or []))
|
|||
|
|
rows.append(cols)
|
|||
|
|
if not rows:
|
|||
|
|
return []
|
|||
|
|
widths = max(len(r) for r in rows)
|
|||
|
|
for row in rows:
|
|||
|
|
if len(row) < widths:
|
|||
|
|
row.extend([""] * (widths - len(row)))
|
|||
|
|
header = rows[0]
|
|||
|
|
separator = ["---" for _ in header]
|
|||
|
|
lines = ["| " + " | ".join(header) + " |", "| " + " | ".join(separator) + " |"]
|
|||
|
|
for row in rows[1:]:
|
|||
|
|
lines.append("| " + " | ".join(row) + " |")
|
|||
|
|
return lines
|
|||
|
|
|
|||
|
|
|
|||
|
|
def main() -> None:
|
|||
|
|
cookie = os.environ.get(COOKIE_ENV)
|
|||
|
|
if not cookie:
|
|||
|
|
raise SystemExit(
|
|||
|
|
f"请先在环境变量 {COOKIE_ENV} 中配置 wolai Cookie，例如：\n"
|
|||
|
|
f"$env:{COOKIE_ENV}='wolai_client_id=...; token=...'"
|
|||
|
|
)
|
|||
|
|
mapping = load_page_mapping(URL_LIST_PATH)
|
|||
|
|
client = WolaiClient(cookie)
|
|||
|
|
OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
|
|||
|
|
for name, url in mapping.items():
|
|||
|
|
if name == "页面选项":
|
|||
|
|
continue
|
|||
|
|
page_id = url.rstrip("/").split("/")[-1]
|
|||
|
|
print(f"正在导出 {name} ({page_id}) ...")
|
|||
|
|
blocks = client.fetch_blocks(page_id)
|
|||
|
|
renderer = MarkdownRenderer(blocks)
|
|||
|
|
content = renderer.render_page(page_id)
|
|||
|
|
out_path = OUTPUT_DIR / f"{sanitize_filename(name)}.md"
|
|||
|
|
out_path.write_text(content, encoding="utf-8")
|
|||
|
|
print("导出完成。")
|
|||
|
|
|
|||
|
|
|
|||
|
|
if __name__ == "__main__":
|
|||
|
|
main()
|