45 lines
1.7 KiB
JavaScript
45 lines
1.7 KiB
JavaScript
// scrape_wolai.js
|
|
const { chromium } = require('@playwright/test');
|
|
const fs = require('fs');
|
|
const path = require('path');
|
|
|
|
(async () => {
|
|
const urls = [
|
|
"https://www.wolai.com/wolai/4TsNd1GmGB3RYbahUZy1bz",
|
|
"https://www.wolai.com/wolai/iG4uLuB67GuVgdH8b633Yk",
|
|
"https://www.wolai.com/wolai/akwMqUeEu9JNqq6AzbL4Bk",
|
|
"https://www.wolai.com/wolai/qN1Bh9YjLAXs8bxCJoAJ6C",
|
|
"https://www.wolai.com/wolai/6zRnvRBPX1cXjzYdKq8f2B",
|
|
"https://www.wolai.com/wolai/veSZ7eYZp48cCxLzH2Uwqt",
|
|
"https://www.wolai.com/wolai/iLJdcXJp8nByXA8KyWDCmN",
|
|
"https://www.wolai.com/wolai/j8en3Y2QvdHUkYAGjUcYmS",
|
|
"https://www.wolai.com/wolai/i1eTuzCbCDV4ymqaDPRN5w",
|
|
"https://www.wolai.com/wolai/doMqLaba4V76PByjJXacSc",
|
|
"https://www.wolai.com/wolai/iokpaWtdKAZoHgMJ4SRNvD",
|
|
"https://www.wolai.com/wolai/kssZs57pWUdiimVPL8c48E",
|
|
"https://www.wolai.com/wolai/fZAdxjxWKMCN5mw4EcUDtC"
|
|
];
|
|
const outputPath = path.resolve('scraped_docs', 'wolaipan.md');
|
|
const writeStream = fs.createWriteStream(outputPath, { flags: 'w' });
|
|
|
|
const browser = await chromium.launch();
|
|
const context = await browser.newContext();
|
|
const page = await context.newPage();
|
|
|
|
for (const url of urls) {
|
|
try {
|
|
await page.goto(url, { waitUntil: 'networkidle' });
|
|
const title = await page.title();
|
|
const text = await page.evaluate(() => document.body.innerText);
|
|
writeStream.write(`# ${title}\n\n`);
|
|
writeStream.write(`${text}\n\n---\n\n`);
|
|
} catch (e) {
|
|
writeStream.write(`## Error loading ${url}\n${e}\n\n---\n\n`);
|
|
}
|
|
}
|
|
|
|
await browser.close();
|
|
writeStream.end();
|
|
console.log('Scraping completed. Output written to', outputPath);
|
|
})();
|