mnote/scripts/wolai_help_center/download_images_playwright.js

// -*- coding: utf-8 -*-
/**
 * 使用 Playwright 访问 Wolai 页面，抓取其实际请求的「带 auth_key」图片 URL，
 * 并将图片下载到本地，随后把 pages/*.md 中的图片链接替换为本地相对路径。
 *
 * 背景：Wostatic CDN 对未签名（缺少 auth_key）的 static 资源会返回 403。
 * Wolai 前端会在渲染时生成/请求带 auth_key 的图片链接，因此需要借助浏览器抓包。
 *
 * 用法：
 *   node scripts/wolai_help_center/download_images_playwright.js --out artifacts/wolai-help-center-v4
 */

const fs = require("fs/promises");
const path = require("path");
const crypto = require("crypto");
const { chromium } = require("playwright");

function parseArgs(argv) {
  const args = { out: "", maxPages: 0 };
  for (let i = 2; i < argv.length; i++) {
    const a = argv[i];
    if (a === "--out") args.out = argv[++i] || "";
    else if (a === "--max-pages") args.maxPages = Number(argv[++i] || "0") || 0;
  }
  return args;
}

function sha256Hex(text) {
  return crypto.createHash("sha256").update(text, "utf8").digest("hex");
}

function guessExt(contentType, urlPathname) {
  const ct = String(contentType || "").split(";")[0].trim().toLowerCase();
  if (ct === "image/png") return ".png";
  if (ct === "image/jpeg") return ".jpg";
  if (ct === "image/webp") return ".webp";
  if (ct === "image/gif") return ".gif";
  if (ct === "image/svg+xml") return ".svg";
  const lower = urlPathname.toLowerCase();
  for (const ext of [".png", ".jpg", ".jpeg", ".webp", ".gif", ".svg"]) {
    if (lower.endsWith(ext)) return ext === ".jpeg" ? ".jpg" : ext;
  }
  return ".bin";
}

function getBaseUrl(fullUrl) {
  const u = new URL(fullUrl);
  return `${u.origin}${u.pathname}`;
}

function getFileSizeHint(fullUrl) {
  try {
    const u = new URL(fullUrl);
    const raw = u.searchParams.get("file_size");
    const n = raw ? Number(raw) : 0;
    return Number.isFinite(n) ? n : 0;
  } catch {
    return 0;
  }
}

async function fileExists(p) {
  try {
    await fs.access(p);
    return true;
  } catch {
    return false;
  }
}

async function replaceInMarkdown(pagesDir, baseToLocal) {
  const files = await fs.readdir(pagesDir);
  const mdFiles = files.filter((f) => f.endsWith(".md"));
  for (const f of mdFiles) {
    const full = path.join(pagesDir, f);
    const raw = await fs.readFile(full, "utf8");
    let next = raw;
    for (const [baseUrl, localRel] of Object.entries(baseToLocal)) {
      if (next.includes(baseUrl)) {
        next = next.split(baseUrl).join(localRel);
      }
    }
    if (next !== raw) {
      await fs.writeFile(full, next, "utf8");
    }
  }
}

async function main() {
  const args = parseArgs(process.argv);
  if (!args.out) {
    console.error("缺少参数：--out <artifacts目录>");
    process.exit(2);
  }

  const outDir = path.resolve(args.out);
  const indexPath = path.join(outDir, "index.json");
  const pagesDir = path.join(outDir, "pages");
  const imagesDir = path.join(outDir, "images");
  const mapPath = path.join(outDir, "image_map.json");

  const index = JSON.parse(await fs.readFile(indexPath, "utf8"));
  const results = Array.isArray(index.results) ? index.results : [];
  const targets = args.maxPages > 0 ? results.slice(0, args.maxPages) : results;

  await fs.mkdir(imagesDir, { recursive: true });

  // baseUrl -> { localRel, bestFileSize }
  const baseToMeta = new Map();

  const browser = await chromium.launch();
  const context = await browser.newContext();

  for (const item of targets) {
    const url = item.source;
    if (!url) continue;

    const page = await context.newPage();

    const pending = [];
    page.on("response", (resp) => {
      const u = resp.url();
      if (!u.startsWith("https://secure2.wostatic.cn/") && !u.startsWith("https://api.wolai.com/v1/icon")) return;
      pending.push(resp);
    });

    await page.goto(url, { waitUntil: "networkidle" }).catch(() => null);
    await page.waitForTimeout(1500);

    // 去重：同一个 response 可能重复进入队列
    const seenResponseUrl = new Set();
    for (const resp of pending) {
      const respUrl = resp.url();
      if (seenResponseUrl.has(respUrl)) continue;
      seenResponseUrl.add(respUrl);

      const status = resp.status();
      if (status !== 200) continue;

      const headers = resp.headers();
      const contentType = headers["content-type"] || "";
      if (!String(contentType).toLowerCase().startsWith("image/") && !respUrl.includes("image_process=")) {
        // 少数图片可能返回 octet-stream，但这里尽量保守
        continue;
      }

      const baseUrl = getBaseUrl(respUrl);
      const fileSize = getFileSizeHint(respUrl);
      const u = new URL(respUrl);
      const ext = guessExt(contentType, u.pathname);
      const digest = sha256Hex(baseUrl).slice(0, 24);
      const filename = `${digest}${ext}`;
      const filePath = path.join(imagesDir, filename);
      const localRel = `images/${filename}`;

      const prev = baseToMeta.get(baseUrl);
      const shouldWrite = !prev || fileSize > (prev.bestFileSize || 0) || !(await fileExists(filePath));
      if (!shouldWrite) {
        baseToMeta.set(baseUrl, { localRel, bestFileSize: prev.bestFileSize || 0 });
        continue;
      }

      try {
        const body = await resp.body();
        await fs.writeFile(filePath, body);
        baseToMeta.set(baseUrl, { localRel, bestFileSize: fileSize });
      } catch {
        // 忽略单个图片失败
      }
    }

    await page.close();
  }

  await browser.close();

  const baseToLocal = {};
  for (const [k, v] of baseToMeta.entries()) {
    baseToLocal[k] = v.localRel;
  }

  await fs.writeFile(mapPath, JSON.stringify({ baseToLocal }, null, 2), "utf8");
  await replaceInMarkdown(pagesDir, baseToLocal);

  console.log(`图片抓取完成：${Object.keys(baseToLocal).length} 个 baseUrl，输出：${imagesDir}`);
  console.log(`映射表：${mapPath}`);
}

main().catch((e) => {
  console.error(e);
  process.exit(1);
});