// -*- coding: utf-8 -*- /** * 使用 Playwright 访问 Wolai 页面,抓取其实际请求的「带 auth_key」图片 URL, * 并将图片下载到本地,随后把 pages/*.md 中的图片链接替换为本地相对路径。 * * 背景:Wostatic CDN 对未签名(缺少 auth_key)的 static 资源会返回 403。 * Wolai 前端会在渲染时生成/请求带 auth_key 的图片链接,因此需要借助浏览器抓包。 * * 用法: * node scripts/wolai_help_center/download_images_playwright.js --out artifacts/wolai-help-center-v4 */ const fs = require("fs/promises"); const path = require("path"); const crypto = require("crypto"); const { chromium } = require("playwright"); function parseArgs(argv) { const args = { out: "", maxPages: 0 }; for (let i = 2; i < argv.length; i++) { const a = argv[i]; if (a === "--out") args.out = argv[++i] || ""; else if (a === "--max-pages") args.maxPages = Number(argv[++i] || "0") || 0; } return args; } function sha256Hex(text) { return crypto.createHash("sha256").update(text, "utf8").digest("hex"); } function guessExt(contentType, urlPathname) { const ct = String(contentType || "").split(";")[0].trim().toLowerCase(); if (ct === "image/png") return ".png"; if (ct === "image/jpeg") return ".jpg"; if (ct === "image/webp") return ".webp"; if (ct === "image/gif") return ".gif"; if (ct === "image/svg+xml") return ".svg"; const lower = urlPathname.toLowerCase(); for (const ext of [".png", ".jpg", ".jpeg", ".webp", ".gif", ".svg"]) { if (lower.endsWith(ext)) return ext === ".jpeg" ? ".jpg" : ext; } return ".bin"; } function getBaseUrl(fullUrl) { const u = new URL(fullUrl); return `${u.origin}${u.pathname}`; } function getFileSizeHint(fullUrl) { try { const u = new URL(fullUrl); const raw = u.searchParams.get("file_size"); const n = raw ? Number(raw) : 0; return Number.isFinite(n) ? n : 0; } catch { return 0; } } async function fileExists(p) { try { await fs.access(p); return true; } catch { return false; } } async function replaceInMarkdown(pagesDir, baseToLocal) { const files = await fs.readdir(pagesDir); const mdFiles = files.filter((f) => f.endsWith(".md")); for (const f of mdFiles) { const full = path.join(pagesDir, f); const raw = await fs.readFile(full, "utf8"); let next = raw; for (const [baseUrl, localRel] of Object.entries(baseToLocal)) { if (next.includes(baseUrl)) { next = next.split(baseUrl).join(localRel); } } if (next !== raw) { await fs.writeFile(full, next, "utf8"); } } } async function main() { const args = parseArgs(process.argv); if (!args.out) { console.error("缺少参数:--out "); process.exit(2); } const outDir = path.resolve(args.out); const indexPath = path.join(outDir, "index.json"); const pagesDir = path.join(outDir, "pages"); const imagesDir = path.join(outDir, "images"); const mapPath = path.join(outDir, "image_map.json"); const index = JSON.parse(await fs.readFile(indexPath, "utf8")); const results = Array.isArray(index.results) ? index.results : []; const targets = args.maxPages > 0 ? results.slice(0, args.maxPages) : results; await fs.mkdir(imagesDir, { recursive: true }); // baseUrl -> { localRel, bestFileSize } const baseToMeta = new Map(); const browser = await chromium.launch(); const context = await browser.newContext(); for (const item of targets) { const url = item.source; if (!url) continue; const page = await context.newPage(); const pending = []; page.on("response", (resp) => { const u = resp.url(); if (!u.startsWith("https://secure2.wostatic.cn/") && !u.startsWith("https://api.wolai.com/v1/icon")) return; pending.push(resp); }); await page.goto(url, { waitUntil: "networkidle" }).catch(() => null); await page.waitForTimeout(1500); // 去重:同一个 response 可能重复进入队列 const seenResponseUrl = new Set(); for (const resp of pending) { const respUrl = resp.url(); if (seenResponseUrl.has(respUrl)) continue; seenResponseUrl.add(respUrl); const status = resp.status(); if (status !== 200) continue; const headers = resp.headers(); const contentType = headers["content-type"] || ""; if (!String(contentType).toLowerCase().startsWith("image/") && !respUrl.includes("image_process=")) { // 少数图片可能返回 octet-stream,但这里尽量保守 continue; } const baseUrl = getBaseUrl(respUrl); const fileSize = getFileSizeHint(respUrl); const u = new URL(respUrl); const ext = guessExt(contentType, u.pathname); const digest = sha256Hex(baseUrl).slice(0, 24); const filename = `${digest}${ext}`; const filePath = path.join(imagesDir, filename); const localRel = `images/${filename}`; const prev = baseToMeta.get(baseUrl); const shouldWrite = !prev || fileSize > (prev.bestFileSize || 0) || !(await fileExists(filePath)); if (!shouldWrite) { baseToMeta.set(baseUrl, { localRel, bestFileSize: prev.bestFileSize || 0 }); continue; } try { const body = await resp.body(); await fs.writeFile(filePath, body); baseToMeta.set(baseUrl, { localRel, bestFileSize: fileSize }); } catch { // 忽略单个图片失败 } } await page.close(); } await browser.close(); const baseToLocal = {}; for (const [k, v] of baseToMeta.entries()) { baseToLocal[k] = v.localRel; } await fs.writeFile(mapPath, JSON.stringify({ baseToLocal }, null, 2), "utf8"); await replaceInMarkdown(pagesDir, baseToLocal); console.log(`图片抓取完成:${Object.keys(baseToLocal).length} 个 baseUrl,输出:${imagesDir}`); console.log(`映射表:${mapPath}`); } main().catch((e) => { console.error(e); process.exit(1); });