// scrape_wolai.js
const { chromium } = require('@playwright/test');
const fs = require('fs');
const path = require('path');

(async () => {
  const urls = [
    "https://www.wolai.com/wolai/4TsNd1GmGB3RYbahUZy1bz",
    "https://www.wolai.com/wolai/iG4uLuB67GuVgdH8b633Yk",
    "https://www.wolai.com/wolai/akwMqUeEu9JNqq6AzbL4Bk",
    "https://www.wolai.com/wolai/qN1Bh9YjLAXs8bxCJoAJ6C",
    "https://www.wolai.com/wolai/6zRnvRBPX1cXjzYdKq8f2B",
    "https://www.wolai.com/wolai/veSZ7eYZp48cCxLzH2Uwqt",
    "https://www.wolai.com/wolai/iLJdcXJp8nByXA8KyWDCmN",
    "https://www.wolai.com/wolai/j8en3Y2QvdHUkYAGjUcYmS",
    "https://www.wolai.com/wolai/i1eTuzCbCDV4ymqaDPRN5w",
    "https://www.wolai.com/wolai/doMqLaba4V76PByjJXacSc",
    "https://www.wolai.com/wolai/iokpaWtdKAZoHgMJ4SRNvD",
    "https://www.wolai.com/wolai/kssZs57pWUdiimVPL8c48E",
    "https://www.wolai.com/wolai/fZAdxjxWKMCN5mw4EcUDtC"
  ];
  const outputPath = path.resolve('scraped_docs', 'wolaipan.md');
  const writeStream = fs.createWriteStream(outputPath, { flags: 'w' });

  const browser = await chromium.launch();
  const context = await browser.newContext();
  const page = await context.newPage();

  for (const url of urls) {
    try {
      await page.goto(url, { waitUntil: 'networkidle' });
      const title = await page.title();
      const text = await page.evaluate(() => document.body.innerText);
      writeStream.write(`# ${title}\n\n`);
      writeStream.write(`${text}\n\n---\n\n`);
    } catch (e) {
      writeStream.write(`## Error loading ${url}\n${e}\n\n---\n\n`);
    }
  }

  await browser.close();
  writeStream.end();
  console.log('Scraping completed. Output written to', outputPath);
})();