lix/mnote
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

0.4.0 convex及界面修改

Browse Source
This commit is contained in:
liaibo
2026-02-01 08:47:40 +08:00
parent d1f055f51a
commit af92c4b149
636 changed files with 7522 additions and 1815 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/desktop-hot.js
+70 -8
View File
@@ -30,6 +30,7 @@ const skipCelery =
const celeryCmdFromEnv = process.env.CELERY_CMD;
const redisUrl = process.env.REDIS_URL || "redis://localhost:6379/0";
const frontendPortFromEnv = Number(process.env.FRONTEND_PORT || 3000);
const backendPortFromEnv = Number(process.env.BACKEND_PORT || 8000);
const tasks = [
{
@@ -106,6 +107,26 @@ function getListeningPidsByPort(port) {
}
}
function getProcessNameByPid(pid) {
try {
// 输出为 CSV，示例：
// "Image Name","PID","Session Name","Session#","Mem Usage"
// "python.exe","12345","Console","1","12,345 K"
const out = execSync(`tasklist /FI "PID eq ${pid}" /FO CSV /NH`, {
encoding: "utf8",
}).trim();
if (!out || /No tasks are running/i.test(out)) return "";
const firstLine = out.split(/\r?\n/)[0].trim();
if (!firstLine) return "";
const cells = firstLine
.split('","')
.map((s) => s.replace(/^"/, "").replace(/"$/, ""));
return (cells[0] || "").trim();
} catch {
return "";
}
}
async function ensurePortFree(port, nameForLog) {
const free = await isPortFree("127.0.0.1", port);
if (free) return true;
@@ -116,8 +137,24 @@ async function ensurePortFree(port, nameForLog) {
return false;
}
logPrefix(nameForLog, `检测到端口 ${port} 被占用，准备重启（结束旧进程）：${pids.join(", ")}`);
for (const pid of pids) {
let killPids = pids;
// 安全策略：后端默认只结束 python 进程（避免误杀其他服务）。
if (nameForLog === "backend") {
killPids = pids.filter((pid) => {
const procName = getProcessNameByPid(pid).toLowerCase();
return procName.includes("python");
});
if (killPids.length === 0) {
logPrefix(
nameForLog,
`检测到端口 ${port} 被占用，但没有可安全结束的 python 进程（PIDs=${pids.join(", ")}）。请手动释放端口后重试。`,
);
return false;
}
}
logPrefix(nameForLog, `检测到端口 ${port} 被占用，准备重启（结束旧进程）：${killPids.join(", ")}`);
for (const pid of killPids) {
try {
execSync(`taskkill /PID ${pid} /T /F`, { stdio: "ignore" });
} catch {
@@ -154,14 +191,16 @@ function loadEnvFile(filePath) {
}, {});
}
const envFromRoot = loadEnvFile(path.join(rootDir, ".env.local"));
const envFromFrontend = loadEnvFile(path.join(frontendDir, ".env.local"));
const envFromBackend = loadEnvFile(path.join(backendDir, ".env.local"));
// 约定：全局仅使用仓库根目录的 .env.all 作为配置来源（生产优先）。
const envAllPath = path.join(rootDir, ".env.all");
if (!fs.existsSync(envAllPath)) {
console.error("[system] 缺少 .env.all：请在仓库根目录创建全局唯一 env 文件后重试。");
process.exit(1);
}
const envFromAll = loadEnvFile(envAllPath);
const mergedEnv = {
...process.env,
...envFromRoot,
...envFromFrontend,
...envFromBackend,
...envFromAll,
};
function logPrefix(name, message) {
@@ -203,6 +242,19 @@ function shutdown(code) {
logPrefix("system", "收到终止信号，正在关闭所有子进程…");
for (const child of children) {
if (!child.pid) continue;
// Windows 下，shell 子进程常常无法可靠传播 SIGINT/SIGTERM 到孙进程（例如 uvicorn --reload）。
// 这里优先用 taskkill /T /F 确保整个进程树被结束，避免残留占用端口导致下次启动失败。
if (process.platform === "win32") {
try {
execSync(`taskkill /PID ${child.pid} /T /F`, { stdio: "ignore" });
continue;
} catch {
// fallback to signals
}
}
if (!child.killed) {
child.kill("SIGINT");
setTimeout(() => {
@@ -279,6 +331,16 @@ async function main() {
logPrefix("frontend", `前端目录：${frontendDir}`);
logPrefix("frontend", `前端地址：${frontendUrl}`);
const desiredBackendPort = backendPortFromEnv;
if (!process.env.BACKEND_CMD) {
const backendPortOk = await ensurePortFree(desiredBackendPort, "backend");
if (!backendPortOk) {
console.error(`后端端口 ${desiredBackendPort} 无法释放，已中止启动。`);
process.exit(1);
}
tasks[1].command = `${pythonBin} -m uvicorn app.main:app --reload --port ${desiredBackendPort}`;
}
if (!skipCelery) {
const celeryTask = {
name: "celery",
scripts/desktop-local.js
+8 -4
View File
@@ -85,11 +85,16 @@ async function waitForPort(host, port, timeoutMs = 20_000) {
}
async function main() {
const envAllPath = path.join(rootDir, ".env.all");
if (!fs.existsSync(envAllPath)) {
console.error("[desktop-local] 缺少 .env.all：请在仓库根目录创建全局唯一 env 文件后重试。");
process.exit(1);
}
const envMerged = {
...process.env,
...loadEnvFile(path.join(rootDir, ".env.local")),
...loadEnvFile(path.join(rootDir, ".env")),
...loadEnvFile(path.join(backendDir, ".env")),
// 约定：全局仅使用仓库根目录的 .env.all 作为配置来源（生产优先）。
...loadEnvFile(envAllPath),
PYTHONUTF8: "1",
MNOTE_DATA_DIR: dataDir,
};
@@ -156,4 +161,3 @@ main().catch((err) => {
console.error(err instanceof Error ? err.stack : String(err));
process.exit(1);
});
scripts/desktop-prod.js
+9 -48
View File
@@ -3,8 +3,7 @@
* 一键启动“生产模式”的前后端与相关服务（适用于 Cloudflare Tunnel / 公网访问）。
*
* 目标：
* - 前端使用 Next production（读取 wolai-frontend/.env.production.local）
* - 后端与内部服务使用本机/内网地址（读取各自 .env 或仓库根目录 .env.local）
* - 前后端与内部服务统一读取仓库根目录 `.env.all`（生产优先，全局唯一 env 文件）
*
* 可选环境变量：
* - SKIP_FRONTEND_BUILD=1：跳过前端 build（仅 start）
@@ -185,65 +184,28 @@ function ensureStandaloneStatic(frontendDir) {
}
async function main() {
const frontendProdEnv = path.join(frontendDir, ".env.production.local");
// 约定：全局仅使用仓库根目录的 .env.all 作为配置来源（生产优先）。
const envAllPath = path.join(rootDir, ".env.all");
requireFile(envAllPath, "请先在仓库根目录创建 .env.all（全局唯一 env 文件）");
const envAll = loadEnvFile(envAllPath);
// 说明：Next.js 在 production 也会加载 wolai-frontend/.env.local。
// 为避免被本地开发环境（可能指向远端 Supabase）的配置污染，这里把“服务端专用”的关键 env 显式注入。
// 规则：优先使用 wolai-frontend/.env.production.local，其次使用仓库根目录 .env.local/.env。
const envRoot = {
...loadEnvFile(path.join(rootDir, ".env.local")),
...loadEnvFile(path.join(rootDir, ".env")),
};
requireFile(
frontendProdEnv,
"请先创建 wolai-frontend/.env.production.local（可参考 wolai-frontend/.env.production.example）",
);
// 生产模式：前端必须使用 production env，避免把 .env.local 的 127.0.0.1 泄露到公网用户
// 生产模式：前端必须使用 production env，避免本地开发配置污染公网用户
const envFrontend = {
...process.env,
...loadEnvFile(frontendProdEnv),
...envAll,
NODE_ENV: "production",
NEXT_TELEMETRY_DISABLED: "1",
};
// Next 的 API Route 也会用到这些“内部服务”环境变量（Search / MinerU / LightRAG 等）。
// 规则：从仓库根目录 .env.local/.env 读取并注入，避免 production server 缺少配置。
const passthroughKeys = [
"MINERU_ENDPOINT",
"LIGHTRAG_URL",
"LIGHTRAG_API_KEY",
"SEARXNG_BASE_URL",
"SEARXNG_API_TOKEN",
];
for (const key of passthroughKeys) {
if (envRoot[key] && !envFrontend[key]) {
envFrontend[key] = envRoot[key];
}
}
// Next 服务端（API Route）需要 service role 才能生成签名 URL、清理资源等。
if (envRoot.SUPABASE_SERVICE_ROLE_KEY) {
envFrontend.SUPABASE_SERVICE_ROLE_KEY = envRoot.SUPABASE_SERVICE_ROLE_KEY;
}
// 兼容旧代码：部分模块读取 SUPABASE_URL（而非 SUPABASE_INTERNAL_URL）。
// 强制覆盖：避免从系统环境变量或 wolai-frontend/.env.local 继承到“远端 Supabase”。
envFrontend.SUPABASE_URL = envFrontend.SUPABASE_INTERNAL_URL || envRoot.SUPABASE_URL || envFrontend.SUPABASE_URL || "";
// 后端/内部服务：尽量使用本机/内网配置（不会暴露给浏览器）
// wolai-backend 使用自身目录下 .env
const envBackend = {
...process.env,
...loadEnvFile(path.join(backendDir, ".env")),
...envAll,
PYTHONUTF8: "1",
};
// ingest_service / rag_gateway 主要依赖仓库根目录 .env.local/.env
const envInternalServices = {
...process.env,
...loadEnvFile(path.join(rootDir, ".env.local")),
...loadEnvFile(path.join(rootDir, ".env")),
...envAll,
PYTHONUTF8: "1",
};
@@ -275,7 +237,6 @@ async function main() {
{ name: "backend", port: backendPort },
{ name: "rag_gateway", port: ragGatewayPort },
{ name: "ingest_service", port: ingestPort },
{ name: "supabase(kong)", port: 18000 },
];
for (const item of portChecks) {
// 仅检查本机 127.0.0.1 是否已有服务在监听，方便你快速判断“是否重复启动”
scripts/env_tools.ps1
+602
View File
@@ -0,0 +1,602 @@
# -*- coding: utf-8 -*-
<#
说明：
- 本脚本用于在本仓库内整理与汇总各处的 env 文件。
- 默认不会在控制台打印任何 env 的内容，避免泄露敏感信息。
用法示例：
pwsh -NoProfile -ExecutionPolicy Bypass -File scripts/env_tools.ps1 -CleanupSupabase -Merge -OutFile .env.all
#>
[CmdletBinding()]
param(
[switch]$CleanupSupabase,
# 兼容旧参数：-Merge 等价于 -MergeRaw
[switch]$Merge,
[switch]$MergeRaw,
[switch]$MergeUnique,
[switch]$Report,
[switch]$ListApplied,
# 合并优先级：默认生产优先；如需本地开发优先，传 -DevFirst
[switch]$DevFirst,
[switch]$AppliedOnly,
[switch]$IncludeLegacy,
[string]$RawOutFile = ".env.all.raw",
[string]$OutFile = ".env.all",
[string]$ConflictReportFile = "env.all.conflicts.md",
[string]$AppliedReportFile = "env.applied.md"
)
$ErrorActionPreference = "Stop"
function New-Utf8NoBomEncoding {
return New-Object System.Text.UTF8Encoding($false)
}
function Read-TextUtf8 {
param([Parameter(Mandatory = $true)][string]$Path)
return [System.IO.File]::ReadAllText($Path, [System.Text.Encoding]::UTF8)
}
function Write-TextUtf8NoBom {
param(
[Parameter(Mandatory = $true)][string]$Path,
[Parameter(Mandatory = $true)][string]$Text
)
$encoding = New-Utf8NoBomEncoding
[System.IO.File]::WriteAllText($Path, $Text, $encoding)
}
function Get-RepoRoot {
return (Resolve-Path -LiteralPath (Join-Path $PSScriptRoot "..")).Path
}
function Get-EnvFiles {
param([Parameter(Mandatory = $true)][string]$Root)
$excludePattern = "\\(node_modules|\.git|dist|build|\.next|out|coverage|pnpm-store|artifacts)\\"
$envFiles =
Get-ChildItem -Path $Root -Recurse -Force -File |
Where-Object {
$n = $_.Name
($n -eq ".env" -or $n -like ".env.*" -or $n -like "*.env" -or $n -like "*.env.*" -or $n -like "*.env.example" -or $n -eq ".env.example" -or $n -like ".env.*.example") -and
($_.FullName -notmatch $excludePattern)
} |
Sort-Object FullName
return $envFiles
}
function Get-EnvFileKind {
param(
[Parameter(Mandatory = $true)][string]$Root,
[Parameter(Mandatory = $true)][string]$FullName
)
$rel = $FullName.Substring($Root.Length).TrimStart("\","/")
$name = [System.IO.Path]::GetFileName($FullName)
$lowerName = $name.ToLowerInvariant()
if ($rel -match "^(?i)cankao[/\\\\]") { return "legacy" }
if ($rel -match "(?i)[/\\\\]test[/\\\\]") { return "legacy" }
if ($lowerName.EndsWith(".example")) { return "template" }
if ($lowerName.EndsWith(".sample")) { return "template" }
if ($lowerName.EndsWith(".mock")) { return "template" }
if ($lowerName.EndsWith(".library")) { return "template" }
if ($lowerName -eq "test.env") { return "template" }
return "runtime"
}
function Get-AppliedEnvFiles {
param([Parameter(Mandatory = $true)][string]$Root)
# 说明：这里的[已应用]指默认启动链路/配置文件明确加载的 env 文件。
$candidates = @(
# 统一入口：仓库根目录的 .env.all（生产优先，且要求 Key 唯一）
(Join-Path $Root ".env.all")
)
$applied = New-Object System.Collections.Generic.List[string]
foreach ($p in ($candidates | Where-Object { Test-Path -LiteralPath $_ })) {
$applied.Add((Resolve-Path -LiteralPath $p).Path) | Out-Null
}
# Docker（运行中）推断：从容器的 compose working_dir 与 compose 文件解析 env_file
$dockerApplied = Get-AppliedEnvFilesFromDocker -Root $Root
foreach ($p in $dockerApplied) {
if (Test-Path -LiteralPath $p) {
$applied.Add((Resolve-Path -LiteralPath $p).Path) | Out-Null
}
}
# Vite（开发模式）默认读取：仅对仓库内已存在的 webui 目录做显式识别
$viteDev = Join-Path $Root "LightRAG\lightrag_webui\.env.development"
if (Test-Path -LiteralPath $viteDev) {
$applied.Add((Resolve-Path -LiteralPath $viteDev).Path) | Out-Null
}
return $applied | Select-Object -Unique
}
function Get-AppliedEnvFilesFromDocker {
param([Parameter(Mandatory = $true)][string]$Root)
$result = New-Object System.Collections.Generic.List[string]
$dockerOk = $false
try {
$null = & docker ps --format "{{.ID}}" 2>$null
$dockerOk = $true
} catch {
$dockerOk = $false
}
if (-not $dockerOk) {
return @()
}
$ids = @()
try {
$ids = & docker ps --format "{{.ID}}" 2>$null
} catch {
return @()
}
$composeFiles = New-Object System.Collections.Generic.HashSet[string]
$workingDirs = New-Object System.Collections.Generic.HashSet[string]
foreach ($id in $ids) {
$workdir = ""
$cfg = ""
try { $workdir = (& docker inspect -f '{{ index .Config.Labels "com.docker.compose.project.working_dir" }}' $id 2>$null) } catch {}
try { $cfg = (& docker inspect -f '{{ index .Config.Labels "com.docker.compose.project.config_files" }}' $id 2>$null) } catch {}
if ($workdir -and $workdir -ne "<no value>" -and ($workdir -like "$Root*")) {
$workingDirs.Add($workdir) | Out-Null
}
if ($cfg -and $cfg -ne "<no value>") {
foreach ($p in ($cfg -split "[;,]" | ForEach-Object { $_.Trim() } | Where-Object { $_ })) {
if ($p -like "$Root*") {
$composeFiles.Add($p) | Out-Null
}
}
}
}
foreach ($wd in $workingDirs) {
$p = Join-Path $wd ".env"
if (Test-Path -LiteralPath $p) {
$result.Add($p) | Out-Null
}
}
foreach ($cf in $composeFiles) {
foreach ($p in (Get-EnvFilesFromCompose -ComposeFile $cf)) {
if ($p -like "$Root*") {
$result.Add($p) | Out-Null
}
}
}
return $result | Select-Object -Unique
}
function Get-EnvFilesFromCompose {
param([Parameter(Mandatory = $true)][string]$ComposeFile)
if (-not (Test-Path -LiteralPath $ComposeFile)) {
return @()
}
$dir = Split-Path -Parent $ComposeFile
$lines = Get-Content -LiteralPath $ComposeFile
$out = New-Object System.Collections.Generic.List[string]
$inEnvFile = $false
$envIndent = 0
foreach ($line in $lines) {
if ($line -match "^(\\s*)env_file\\s*:\\s*$") {
$inEnvFile = $true
$envIndent = $matches[1].Length
continue
}
if ($inEnvFile) {
# 退出条件：遇到更浅的缩进且不是空行/注释
if ($line -match "^(\\s*)(\\S.*)$") {
$indent = $matches[1].Length
$payload = $matches[2]
if ($indent -le $envIndent -and $payload -notmatch "^-") {
$inEnvFile = $false
}
}
}
if (-not $inEnvFile) { continue }
if ($line -match "^\\s*-\\s*(.+?)\\s*$") {
$raw = $matches[1].Trim()
$raw = $raw.Trim('"').Trim("'")
if (-not $raw) { continue }
$resolved = if ([System.IO.Path]::IsPathRooted($raw)) { $raw } else { Join-Path $dir $raw }
try {
# 归一化 .. 等相对段，避免 Test-Path / -LiteralPath 因路径包含 .. 而误判
$resolved = [System.IO.Path]::GetFullPath($resolved)
} catch {}
$out.Add($resolved) | Out-Null
}
}
return $out | Select-Object -Unique
}
function Remove-LinesContaining {
param(
[Parameter(Mandatory = $true)][string]$Path,
[Parameter(Mandatory = $true)][string]$Pattern
)
$text = Read-TextUtf8 -Path $Path
$lines = $text -split "(\r?\n)", 0
# 上面的 split 会保留分隔符（换行）作为独立元素；便于最大限度保留原始换行风格。
$out = New-Object System.Text.StringBuilder
for ($i = 0; $i -lt $lines.Count; $i++) {
$chunk = $lines[$i]
if ($chunk -match "^\r?\n$") {
[void]$out.Append($chunk)
continue
}
if ($chunk -match $Pattern) {
continue
}
[void]$out.Append($chunk)
}
Write-TextUtf8NoBom -Path $Path -Text $out.ToString()
}
function Get-MergeOrderScore {
param(
[Parameter(Mandatory = $true)][string]$Root,
[Parameter(Mandatory = $true)][string]$FullName
)
$rel = $FullName.Substring($Root.Length).TrimStart("\","/")
$name = [System.IO.Path]::GetFileName($FullName)
# 目录组优先级：越大越[后覆盖]（优先级更高）
$group = 0
if ($rel -match "^(?i)wolai-backend[/\\\\]") { $group = 60 }
elseif ($rel -match "^(?i)wolai-frontend[/\\\\]") { $group = 50 }
elseif ($rel -match "^(?i)infra[/\\\\]convex[/\\\\]") { $group = 40 }
elseif ($rel -match "^(?i)services[/\\\\]") { $group = 30 }
elseif ($rel -match "^(?i)LightRAG[/\\\\]") { $group = 20 }
elseif ($rel -match "^(?i)cankao[/\\\\]") { $group = -50 }
elseif ($rel -match "(?i)[/\\\\]test[/\\\\]") { $group = -40 }
else { $group = 0 }
# 文件名优先级：越大越[后覆盖]
$fileScore = 0
if ($DevFirst) {
# 开发优先：.env.local 覆盖 .env；production 文件仍保持最高
if ($name -match "(?i)\\.production\\.local$") { $fileScore = 50 }
elseif ($name -match "(?i)\\.production$") { $fileScore = 40 }
elseif ($name -match "(?i)\\.local$") { $fileScore = 30 }
elseif ($name -eq ".env") { $fileScore = 20 }
else { $fileScore = 0 }
} else {
# 生产优先（默认）：.env 覆盖 .env.local；production 文件最高
if ($name -match "(?i)\\.production\\.local$") { $fileScore = 50 }
elseif ($name -match "(?i)\\.production$") { $fileScore = 40 }
elseif ($name -eq ".env") { $fileScore = 30 }
elseif ($name -match "(?i)\\.local$") { $fileScore = 20 }
else { $fileScore = 0 }
}
# template/legacy 默认更弱（更靠前），避免覆盖运行时真实值
$kind = Get-EnvFileKind -Root $Root -FullName $FullName
$kindScore = 0
if ($kind -eq "template") { $kindScore = -1000 }
elseif ($kind -eq "legacy") { $kindScore = -2000 }
return ($kindScore + ($group * 100) + $fileScore)
}
function Merge-EnvFilesRaw {
param(
[Parameter(Mandatory = $true)][string]$Root,
[Parameter(Mandatory = $true)][string]$OutPath,
[Parameter(Mandatory = $true)][System.IO.FileInfo[]]$Files
)
$sb = New-Object System.Text.StringBuilder
$now = Get-Date -Format "yyyy-MM-dd HH:mm:ss"
[void]$sb.AppendLine("# 自动生成（UTF-8）。此文件为[原样拼接版]，可能包含重复 Key。")
[void]$sb.AppendLine("# 生成时间：$now")
[void]$sb.AppendLine("# 重新生成：pwsh -NoProfile -ExecutionPolicy Bypass -File scripts/env_tools.ps1 -MergeRaw -RawOutFile `"$($OutPath | Split-Path -Leaf)`"")
foreach ($f in $files) {
$rel = $f.FullName.Substring($Root.Length).TrimStart("\","/")
[void]$sb.AppendLine("")
[void]$sb.AppendLine("# ===== $rel =====")
try {
$content = Read-TextUtf8 -Path $f.FullName
} catch {
# 若遇到非 UTF-8 文件，仍尽量以系统默认读取（但写回统一为 UTF-8）。
$content = [System.IO.File]::ReadAllText($f.FullName)
}
$content = $content.TrimEnd("`r", "`n")
if ($content.Length -gt 0) {
[void]$sb.AppendLine($content)
}
[void]$sb.AppendLine("")
}
Write-TextUtf8NoBom -Path $OutPath -Text $sb.ToString()
}
function Parse-EnvAssignments {
param(
[Parameter(Mandatory = $true)][string]$Root,
[Parameter(Mandatory = $true)][string]$FullName
)
try {
$content = Read-TextUtf8 -Path $FullName
} catch {
$content = [System.IO.File]::ReadAllText($FullName)
}
$rel = $FullName.Substring($Root.Length).TrimStart("\","/")
$lines = $content -split "\r?\n"
$out = @()
for ($i = 0; $i -lt $lines.Count; $i++) {
$line = $lines[$i]
if ($null -eq $line) { continue }
if ($line -match "^\s*#") { continue }
if ($line -match "^\s*$") { continue }
if ($line -match "^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)\s*$") {
$key = $matches[1]
$rhs = $matches[2]
$out += [pscustomobject]@{
Key = $key
Rhs = $rhs
Source = $rel
Line = ($i + 1)
}
}
}
return $out
}
function Merge-EnvUnique {
param(
[Parameter(Mandatory = $true)][string]$Root,
[Parameter(Mandatory = $true)][string]$OutPath,
[Parameter(Mandatory = $true)][System.IO.FileInfo[]]$Files,
[Parameter(Mandatory = $true)][string]$ConflictReportPath
)
# 排序：分数越大，越靠后覆盖
$ordered = $Files | Sort-Object { Get-MergeOrderScore -Root $Root -FullName $_.FullName }, FullName
$final = @{}
$conflicts = @{}
foreach ($f in $ordered) {
$assignments = Parse-EnvAssignments -Root $Root -FullName $f.FullName
foreach ($a in $assignments) {
if ($final.ContainsKey($a.Key)) {
if (-not $conflicts.ContainsKey($a.Key)) {
$conflicts[$a.Key] = @($final[$a.Key].Source)
}
$conflicts[$a.Key] += $a.Source
}
$final[$a.Key] = $a
}
}
$now = Get-Date -Format "yyyy-MM-dd HH:mm:ss"
$sb = New-Object System.Text.StringBuilder
[void]$sb.AppendLine("# 自动生成（UTF-8）。此文件为[去重版]：每个 Key 只保留一个定义。")
[void]$sb.AppendLine("# 生成时间：$now")
[void]$sb.AppendLine("# 合并规则：按目录/文件优先级排序，越靠后越覆盖（template/legacy 默认不会覆盖 runtime）。")
[void]$sb.AppendLine("# 冲突报告：$([System.IO.Path]::GetFileName($ConflictReportPath))（不包含任何值）")
[void]$sb.AppendLine("")
foreach ($k in ($final.Keys | Sort-Object)) {
$a = $final[$k]
[void]$sb.AppendLine("# from: $($a.Source)")
[void]$sb.AppendLine("$($a.Key)=$($a.Rhs)")
[void]$sb.AppendLine("")
}
Write-TextUtf8NoBom -Path $OutPath -Text $sb.ToString()
# 写冲突报告（不输出值）
$rep = New-Object System.Text.StringBuilder
[void]$rep.AppendLine("# env key 冲突报告（不含任何值）")
[void]$rep.AppendLine("# 生成时间：$now")
[void]$rep.AppendLine("")
if ($conflicts.Keys.Count -eq 0) {
[void]$rep.AppendLine("未发现重复 Key。")
} else {
foreach ($k in ($conflicts.Keys | Sort-Object)) {
$chosen = $final[$k].Source
$allSources = ($conflicts[$k] + @($chosen)) | Select-Object -Unique
$others = $allSources | Where-Object { $_ -ne $chosen }
[void]$rep.AppendLine("- $k")
[void]$rep.AppendLine(" - 采用：$chosen")
if ($others.Count -gt 0) {
[void]$rep.AppendLine(" - 其它：$($others -join ', ')")
}
}
}
Write-TextUtf8NoBom -Path $ConflictReportPath -Text $rep.ToString()
}
$root = Get-RepoRoot
if ($Merge -and -not $MergeRaw) {
$MergeRaw = $true
}
$appliedPaths = Get-AppliedEnvFiles -Root $root
if ($ListApplied) {
foreach ($p in ($appliedPaths | Sort-Object)) {
$rel = $p.Substring($root.Length).TrimStart("\","/")
Write-Output $rel
}
return
}
$appliedOnlyDefault = $true
if (-not $PSBoundParameters.ContainsKey("AppliedOnly")) {
$AppliedOnly = $appliedOnlyDefault
}
$allEnvFiles = Get-EnvFiles -Root $root
$appliedSet = @{}
foreach ($p in $appliedPaths) { $appliedSet[$p.ToLowerInvariant()] = $true }
$selected =
if ($AppliedOnly) {
$allEnvFiles | Where-Object { $appliedSet.ContainsKey($_.FullName.ToLowerInvariant()) }
} else {
$allEnvFiles
}
if (-not $IncludeLegacy) {
$selected =
$selected | Where-Object {
(Get-EnvFileKind -Root $root -FullName $_.FullName) -ne "legacy"
}
}
$rawOutFull = Join-Path $root $RawOutFile
$outFull = Join-Path $root $OutFile
$conflictFull = Join-Path $root $ConflictReportFile
$appliedReportFull = Join-Path $root $AppliedReportFile
$root = Get-RepoRoot
if ($CleanupSupabase) {
$filesWithSupabase =
$allEnvFiles |
Where-Object {
try {
(Read-TextUtf8 -Path $_.FullName) -match "(?i)supabase"
} catch {
([System.IO.File]::ReadAllText($_.FullName)) -match "(?i)supabase"
}
}
foreach ($f in $filesWithSupabase) {
Remove-LinesContaining -Path $f.FullName -Pattern "(?i)supabase"
}
}
if ($MergeRaw) {
$files = $selected | Where-Object { $_.FullName -ne $rawOutFull -and $_.FullName -ne $outFull }
Merge-EnvFilesRaw -Root $root -OutPath $rawOutFull -Files $files
}
if ($MergeUnique) {
$files = $selected | Where-Object { $_.FullName -ne $rawOutFull -and $_.FullName -ne $outFull }
Merge-EnvUnique -Root $root -OutPath $outFull -Files $files -ConflictReportPath $conflictFull
}
if ($Report -or $MergeRaw -or $MergeUnique) {
# 写[已应用/未应用]报告（不包含值）
$now = Get-Date -Format "yyyy-MM-dd HH:mm:ss"
$sb2 = New-Object System.Text.StringBuilder
[void]$sb2.AppendLine("# env 应用情况报告（不含任何值）")
[void]$sb2.AppendLine("# 生成时间：$now")
[void]$sb2.AppendLine("")
[void]$sb2.AppendLine("## 默认链路明确加载的 env 文件（Applied）")
if ($appliedPaths.Count -eq 0) {
[void]$sb2.AppendLine("- （未发现）")
} else {
foreach ($p in $appliedPaths) {
$rel = $p.Substring($root.Length).TrimStart("\","/")
[void]$sb2.AppendLine("- $rel")
}
}
[void]$sb2.AppendLine("")
[void]$sb2.AppendLine("## 扫描到的 env 文件（Found）")
$generatedEnvFull = @(
$outFull.ToLowerInvariant(),
$rawOutFull.ToLowerInvariant()
)
foreach ($f in $allEnvFiles) {
$rel = $f.FullName.Substring($root.Length).TrimStart("\","/")
$kind = Get-EnvFileKind -Root $root -FullName $f.FullName
$tag = ""
if ($generatedEnvFull -contains $f.FullName.ToLowerInvariant()) {
$tag = "generated"
}
if ($tag -ne "") {
[void]$sb2.AppendLine("- $rel ($kind, $tag)")
} else {
[void]$sb2.AppendLine("- $rel ($kind)")
}
}
[void]$sb2.AppendLine("")
[void]$sb2.AppendLine("## 默认链路未使用的文件（按类型分组）")
$notApplied = $allEnvFiles | Where-Object {
(-not $appliedSet.ContainsKey($_.FullName.ToLowerInvariant())) -and
(-not ($generatedEnvFull -contains $_.FullName.ToLowerInvariant()))
}
$notAppliedRuntime = $notApplied | Where-Object { (Get-EnvFileKind -Root $root -FullName $_.FullName) -eq "runtime" }
$notAppliedTemplate = $notApplied | Where-Object { (Get-EnvFileKind -Root $root -FullName $_.FullName) -eq "template" }
$notAppliedLegacy = $notApplied | Where-Object { (Get-EnvFileKind -Root $root -FullName $_.FullName) -eq "legacy" }
[void]$sb2.AppendLine("")
[void]$sb2.AppendLine("### runtime（可能是历史遗留/需确认是否仍在用）")
if ($notAppliedRuntime.Count -eq 0) {
[void]$sb2.AppendLine("- （无）")
} else {
foreach ($f in $notAppliedRuntime) {
$rel = $f.FullName.Substring($root.Length).TrimStart("\","/")
[void]$sb2.AppendLine("- $rel")
}
}
[void]$sb2.AppendLine("")
[void]$sb2.AppendLine("### template（示例/样例文件）")
if ($notAppliedTemplate.Count -eq 0) {
[void]$sb2.AppendLine("- （无）")
} else {
foreach ($f in $notAppliedTemplate) {
$rel = $f.FullName.Substring($root.Length).TrimStart("\","/")
[void]$sb2.AppendLine("- $rel")
}
}
[void]$sb2.AppendLine("")
[void]$sb2.AppendLine("### legacy（参考资料/测试夹带）")
if ($notAppliedLegacy.Count -eq 0) {
[void]$sb2.AppendLine("- （无）")
} else {
foreach ($f in $notAppliedLegacy) {
$rel = $f.FullName.Substring($root.Length).TrimStart("\","/")
[void]$sb2.AppendLine("- $rel")
}
}
Write-TextUtf8NoBom -Path $appliedReportFull -Text $sb2.ToString()
}
scripts/prepare-desktop-next.js
+5 -5
View File
@@ -72,14 +72,14 @@ function copyDir(from, to) {
function main() {
console.log("[prepare-desktop-next] 开始构建 wolai-frontend（standalone）…");
// 说明：桌面端默认不走 Cloudflare（避免 NEXT_PUBLIC_* 写死导致“像网页版一样卡”）。
// 优先读取 wolai-frontend/.env.local，并允许用 wolai-frontend/.env.desktop.local 覆盖。
// 约定：全局仅使用仓库根目录的 .env.all 作为配置来源（生产优先）。
// 桌面端如需覆盖，可通过系统环境变量注入（例如在启动命令前临时设置）。
const envAllPath = path.join(rootDir, ".env.all");
assertExists(envAllPath, "请先在仓库根目录创建 .env.all（全局唯一 env 文件）。");
const desktopEnv = {
...parseDotenv(path.join(frontendDir, ".env.local")),
...parseDotenv(path.join(frontendDir, ".env.desktop.local")),
...parseDotenv(envAllPath),
};
console.log("[prepare-desktop-next] 桌面端 env 摘要：", {
NEXT_PUBLIC_SUPABASE_URL: desktopEnv.NEXT_PUBLIC_SUPABASE_URL,
NEXT_PUBLIC_BACKEND_URL: desktopEnv.NEXT_PUBLIC_BACKEND_URL,
NEXT_PUBLIC_ONLYOFFICE_BASE_URL: desktopEnv.NEXT_PUBLIC_ONLYOFFICE_BASE_URL,
});
scripts/wolai_help_center/build_keyword_hits.py
+103
View File
@@ -0,0 +1,103 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
在已导出的 Wolai 帮助中心语料中做关键词命中统计，方便定位“页面组成/页面操作”相关页面。
示例：
python scripts/wolai_help_center/build_keyword_hits.py --out artifacts/wolai-help-center-v4
"""
from __future__ import annotations
import argparse
import json
from pathlib import Path
from typing import Any, Dict, List
DEFAULT_KEYWORDS = [
"页面",
"页面选项",
"自适应宽度",
"小字体",
"标题目录",
"标题自动编号",
"编辑保护",
"复制链接",
"导出",
"移动",
"嵌入",
"页面历史",
"字数",
"统计",
"权限",
"共享",
"公开",
"私有",
"删除",
"恢复",
"撤回",
"回收站",
"关系图",
"反向链接",
]
def read_front_matter_title(md: str) -> str:
if not md.startswith("---"):
return ""
try:
meta_json = md.split("---", 2)[1].strip()
meta = json.loads(meta_json)
return str(meta.get("title") or "")
except Exception:
return ""
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--out", type=str, default="artifacts/wolai-help-center-v4", help="抓取输出目录")
parser.add_argument("--keywords", type=str, default="", help="自定义关键词（用逗号分隔）")
args = parser.parse_args()
out_dir = Path(args.out)
index_path = out_dir / "index.json"
if not index_path.exists():
raise SystemExit(f"未找到 {index_path}")
index = json.loads(index_path.read_text(encoding="utf-8"))
results: List[Dict[str, Any]] = index.get("results") or []
keywords = (
[x.strip() for x in args.keywords.split(",") if x.strip()]
if args.keywords.strip()
else DEFAULT_KEYWORDS
)
hits: Dict[str, List[Dict[str, str]]] = {k: [] for k in keywords}
for r in results:
pid = str(r.get("pageId") or "")
md_rel = str(r.get("md") or "")
if not pid or not md_rel:
continue
md_path = out_dir / md_rel
if not md_path.exists():
continue
md = md_path.read_text(encoding="utf-8", errors="replace")
title = read_front_matter_title(md)
for k in keywords:
if k in md:
hits[k].append(
{"pageId": pid, "title": title, "md": md_rel.replace("\\", "/")}
)
analysis_dir = out_dir / "analysis"
analysis_dir.mkdir(parents=True, exist_ok=True)
out_path = analysis_dir / "keyword_hits.json"
out_path.write_text(json.dumps(hits, ensure_ascii=False, indent=2), encoding="utf-8")
print(f"已写入：{out_path}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
scripts/wolai_help_center/download_images_playwright.js
+193
View File
@@ -0,0 +1,193 @@
// -*- coding: utf-8 -*-
/**
* 使用 Playwright 访问 Wolai 页面，抓取其实际请求的「带 auth_key」图片 URL，
* 并将图片下载到本地，随后把 pages/*.md 中的图片链接替换为本地相对路径。
*
* 背景：Wostatic CDN 对未签名（缺少 auth_key）的 static 资源会返回 403。
* Wolai 前端会在渲染时生成/请求带 auth_key 的图片链接，因此需要借助浏览器抓包。
*
* 用法：
* node scripts/wolai_help_center/download_images_playwright.js --out artifacts/wolai-help-center-v4
*/
const fs = require("fs/promises");
const path = require("path");
const crypto = require("crypto");
const { chromium } = require("playwright");
function parseArgs(argv) {
const args = { out: "", maxPages: 0 };
for (let i = 2; i < argv.length; i++) {
const a = argv[i];
if (a === "--out") args.out = argv[++i] || "";
else if (a === "--max-pages") args.maxPages = Number(argv[++i] || "0") || 0;
}
return args;
}
function sha256Hex(text) {
return crypto.createHash("sha256").update(text, "utf8").digest("hex");
}
function guessExt(contentType, urlPathname) {
const ct = String(contentType || "").split(";")[0].trim().toLowerCase();
if (ct === "image/png") return ".png";
if (ct === "image/jpeg") return ".jpg";
if (ct === "image/webp") return ".webp";
if (ct === "image/gif") return ".gif";
if (ct === "image/svg+xml") return ".svg";
const lower = urlPathname.toLowerCase();
for (const ext of [".png", ".jpg", ".jpeg", ".webp", ".gif", ".svg"]) {
if (lower.endsWith(ext)) return ext === ".jpeg" ? ".jpg" : ext;
}
return ".bin";
}
function getBaseUrl(fullUrl) {
const u = new URL(fullUrl);
return `${u.origin}${u.pathname}`;
}
function getFileSizeHint(fullUrl) {
try {
const u = new URL(fullUrl);
const raw = u.searchParams.get("file_size");
const n = raw ? Number(raw) : 0;
return Number.isFinite(n) ? n : 0;
} catch {
return 0;
}
}
async function fileExists(p) {
try {
await fs.access(p);
return true;
} catch {
return false;
}
}
async function replaceInMarkdown(pagesDir, baseToLocal) {
const files = await fs.readdir(pagesDir);
const mdFiles = files.filter((f) => f.endsWith(".md"));
for (const f of mdFiles) {
const full = path.join(pagesDir, f);
const raw = await fs.readFile(full, "utf8");
let next = raw;
for (const [baseUrl, localRel] of Object.entries(baseToLocal)) {
if (next.includes(baseUrl)) {
next = next.split(baseUrl).join(localRel);
}
}
if (next !== raw) {
await fs.writeFile(full, next, "utf8");
}
}
}
async function main() {
const args = parseArgs(process.argv);
if (!args.out) {
console.error("缺少参数：--out <artifacts目录>");
process.exit(2);
}
const outDir = path.resolve(args.out);
const indexPath = path.join(outDir, "index.json");
const pagesDir = path.join(outDir, "pages");
const imagesDir = path.join(outDir, "images");
const mapPath = path.join(outDir, "image_map.json");
const index = JSON.parse(await fs.readFile(indexPath, "utf8"));
const results = Array.isArray(index.results) ? index.results : [];
const targets = args.maxPages > 0 ? results.slice(0, args.maxPages) : results;
await fs.mkdir(imagesDir, { recursive: true });
// baseUrl -> { localRel, bestFileSize }
const baseToMeta = new Map();
const browser = await chromium.launch();
const context = await browser.newContext();
for (const item of targets) {
const url = item.source;
if (!url) continue;
const page = await context.newPage();
const pending = [];
page.on("response", (resp) => {
const u = resp.url();
if (!u.startsWith("https://secure2.wostatic.cn/") && !u.startsWith("https://api.wolai.com/v1/icon")) return;
pending.push(resp);
});
await page.goto(url, { waitUntil: "networkidle" }).catch(() => null);
await page.waitForTimeout(1500);
// 去重：同一个 response 可能重复进入队列
const seenResponseUrl = new Set();
for (const resp of pending) {
const respUrl = resp.url();
if (seenResponseUrl.has(respUrl)) continue;
seenResponseUrl.add(respUrl);
const status = resp.status();
if (status !== 200) continue;
const headers = resp.headers();
const contentType = headers["content-type"] || "";
if (!String(contentType).toLowerCase().startsWith("image/") && !respUrl.includes("image_process=")) {
// 少数图片可能返回 octet-stream，但这里尽量保守
continue;
}
const baseUrl = getBaseUrl(respUrl);
const fileSize = getFileSizeHint(respUrl);
const u = new URL(respUrl);
const ext = guessExt(contentType, u.pathname);
const digest = sha256Hex(baseUrl).slice(0, 24);
const filename = `${digest}${ext}`;
const filePath = path.join(imagesDir, filename);
const localRel = `images/${filename}`;
const prev = baseToMeta.get(baseUrl);
const shouldWrite = !prev || fileSize > (prev.bestFileSize || 0) || !(await fileExists(filePath));
if (!shouldWrite) {
baseToMeta.set(baseUrl, { localRel, bestFileSize: prev.bestFileSize || 0 });
continue;
}
try {
const body = await resp.body();
await fs.writeFile(filePath, body);
baseToMeta.set(baseUrl, { localRel, bestFileSize: fileSize });
} catch {
// 忽略单个图片失败
}
}
await page.close();
}
await browser.close();
const baseToLocal = {};
for (const [k, v] of baseToMeta.entries()) {
baseToLocal[k] = v.localRel;
}
await fs.writeFile(mapPath, JSON.stringify({ baseToLocal }, null, 2), "utf8");
await replaceInMarkdown(pagesDir, baseToLocal);
console.log(`图片抓取完成：${Object.keys(baseToLocal).length} 个 baseUrl，输出：${imagesDir}`);
console.log(`映射表：${mapPath}`);
}
main().catch((e) => {
console.error(e);
process.exit(1);
});
scripts/wolai_help_center/export_wolai_help_center.py
+647
View File
@@ -0,0 +1,647 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
抓取 Wolai 帮助中心（公开页面）并落盘为可检索语料：
- 通过 Wolai API 拉取 blocks（比 Playwright 更稳定、噪声更少）
- 递归抓取子块，并从内容里的链接继续爬取其他页面
- 下载图片到本地，并在 Markdown 中引用本地路径
用法示例（PowerShell）：
python scripts/wolai_help_center/export_wolai_help_center.py `
--seed-file scripts/wolai_help_center/seeds.txt `
--out artifacts/wolai-help-center `
--max-pages 200 `
--download-images
"""
from __future__ import annotations
import argparse
import hashlib
import json
import re
import sys
import time
from collections import deque
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Dict, Iterable, List, Optional, Sequence, Set, Tuple
from urllib.parse import quote, urlparse
import requests
API_URL = "https://api.wolai.com/v1/pages/getData"
# Wolai 页面 URL： https://www.wolai.com/wolai/<pageId>
WOLAI_PAGE_ID_RE = re.compile(r"(?:https?://www\.wolai\.com)?/wolai/([A-Za-z0-9]+)")
def _read_text(path: Path) -> str:
return path.read_text(encoding="utf-8")
def _write_text(path: Path, content: str) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(content, encoding="utf-8")
def _write_json(path: Path, payload: Any) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
def sanitize_filename(name: str) -> str:
return re.sub(r"[\\/:\*\?\"<>\|]", "_", name).strip() or "untitled"
def parse_page_id(url_or_id: str) -> Optional[str]:
url_or_id = url_or_id.strip()
if not url_or_id:
return None
if "/" not in url_or_id:
return url_or_id
m = WOLAI_PAGE_ID_RE.search(url_or_id)
if not m:
return None
return m.group(1)
def build_image_url(block: dict) -> Optional[str]:
attrs = block.get("attributes") or {}
# 新版/部分图片块使用 source 字段（例如动态图标、截图等）
source = attrs.get("source")
if isinstance(source, list) and source and isinstance(source[0], str) and source[0].strip():
return source[0].strip()
img = attrs.get("img") or []
if not img or not img[0]:
return None
path = img[0][0]
if isinstance(path, str) and path.startswith("http"):
return path
if not isinstance(path, str):
return None
safe_path = quote(path, safe="/%")
return f"https://secure2.wostatic.cn/{safe_path}"
def guess_extension(content_type: str, url: str) -> str:
content_type = (content_type or "").split(";")[0].strip().lower()
if content_type in {"image/png"}:
return ".png"
if content_type in {"image/jpeg", "image/jpg"}:
return ".jpg"
if content_type in {"image/webp"}:
return ".webp"
if content_type in {"image/gif"}:
return ".gif"
if content_type in {"image/svg+xml"}:
return ".svg"
# 兜底：从 URL 后缀猜
path = urlparse(url).path.lower()
for ext in [".png", ".jpg", ".jpeg", ".webp", ".gif", ".svg"]:
if path.endswith(ext):
return ".jpg" if ext == ".jpeg" else ext
return ".bin"
def sha256_hex(text: str) -> str:
return hashlib.sha256(text.encode("utf-8")).hexdigest()
def iter_strings(obj: Any) -> Iterable[str]:
"""深度遍历任意 JSON 结构，返回其中所有字符串。"""
if isinstance(obj, str):
yield obj
elif isinstance(obj, list):
for item in obj:
yield from iter_strings(item)
elif isinstance(obj, dict):
for v in obj.values():
yield from iter_strings(v)
def extract_mark_links_from_title_fragments(fragments: Sequence[Sequence]) -> Tuple[List[str], List[str]]:
"""从富文本片段中提取 Link(url) 与 BiLink(id)。"""
link_urls: List[str] = []
bilink_ids: List[str] = []
for fragment in fragments:
if not fragment:
continue
marks = fragment[1] if len(fragment) > 1 else None
if not marks:
continue
for mark in marks:
if not mark:
continue
kind = mark[0]
if kind == "Link" and len(mark) > 1 and isinstance(mark[1], str):
link_urls.append(mark[1])
elif kind == "BiLink":
# Wolai 的 BiLink 结构常见为：
# ["BiLink", "<blockId>", "<pageId>", ...]
# 这里优先取第三段（pageId），避免把 blockId 当成页面导致爆炸式爬取。
if len(mark) > 2 and isinstance(mark[2], str):
bilink_ids.append(mark[2])
elif len(mark) > 1 and isinstance(mark[1], str):
# 兜底：少数情况下可能只有一个 id
bilink_ids.append(mark[1])
return link_urls, bilink_ids
def extract_page_ids_from_blocks(blocks: Dict[str, dict]) -> Set[str]:
"""从 blocks 中提取可能的 Wolai 页面 ID（用于爬取下一页）。"""
ids: Set[str] = set()
for block in blocks.values():
attrs = block.get("attributes") or {}
title = attrs.get("title") or []
if isinstance(title, str):
title = [[title]]
if isinstance(title, list):
link_urls, bilink_ids = extract_mark_links_from_title_fragments(title)
for url in link_urls:
pid = parse_page_id(url)
if pid:
ids.add(pid)
for bid in bilink_ids:
# 只接受“看起来像 Wolai id”的 BiLink 目标，避免误把其它资源标识加入队列
if re.fullmatch(r"[A-Za-z0-9]{16,32}", bid):
ids.add(bid)
# 更激进的兜底：扫整个 block 里的字符串，找 /wolai/<id>
for s in iter_strings(block):
for m in WOLAI_PAGE_ID_RE.finditer(s):
ids.add(m.group(1))
return ids
class WolaiApiClient:
def __init__(self) -> None:
self.session = requests.Session()
self.session.headers.update(
{
"wolai-client-platform": "web",
"wolai-app-version": "1.2.3-15",
"wolai-os-platform": "win",
"Origin": "https://www.wolai.com",
"Referer": "https://www.wolai.com/",
"User-Agent": (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/120.0.0.0 Safari/537.36"
),
"Accept": "application/json, text/plain, */*",
}
)
def fetch_blocks_recursive(self, root_id: str) -> Dict[str, dict]:
"""从 root_id 开始，批量拉取并递归展开 sub_nodes。"""
blocks: Dict[str, dict] = {}
queue: deque[str] = deque([root_id])
seen: Set[str] = set()
while queue:
chunk: List[str] = []
while queue and len(chunk) < 50:
bid = queue.popleft()
if bid in seen:
continue
seen.add(bid)
chunk.append(bid)
if not chunk:
continue
payload = {"requests": [{"table": "wolai.block", "id": bid} for bid in chunk]}
resp = self.session.post(API_URL, json=payload, timeout=60)
resp.raise_for_status()
data = resp.json().get("data") or []
for entry in data:
value = entry.get("value")
if not value or not isinstance(value, dict):
continue
bid = value.get("id")
if not isinstance(bid, str):
continue
blocks[bid] = value
for child in value.get("sub_nodes") or []:
if isinstance(child, str) and child not in seen:
queue.append(child)
return blocks
def get_title_fragments(block: dict) -> List[List]:
attrs = block.get("attributes") or {}
title = attrs.get("title")
if isinstance(title, str):
return [[title]]
return title or []
def apply_marks(text: str, marks: Sequence[Sequence] | None) -> str:
if not marks:
return text
decorated = text
link_target: Optional[str] = None
for mark in marks:
if not mark:
continue
kind = mark[0]
if kind == "B":
decorated = f"**{decorated}**"
elif kind == "I":
decorated = f"*{decorated}*"
elif kind == "S":
decorated = f"~~{decorated}~~"
elif kind == "<>":
decorated = f"`{decorated}`"
elif kind == "Link" and len(mark) > 1 and isinstance(mark[1], str):
link_target = mark[1]
elif kind == "BiLink":
# 链接到空间内其他块/页面：这里只影响显示，不强行转成链接，避免生成无效 URL
continue
else:
continue
if link_target:
decorated = f"[{decorated}]({link_target})"
return decorated
def rich_text(fragments: Sequence[Sequence]) -> str:
result: List[str] = []
for fragment in fragments:
if not fragment:
continue
text = fragment[0]
if not isinstance(text, str):
continue
marks = fragment[1] if len(fragment) > 1 else None
result.append(apply_marks(text, marks))
return "".join(result).strip()
def indent_lines(lines: Sequence[str], prefix: str) -> List[str]:
out: List[str] = []
for line in lines:
out.append(f"{prefix}{line}" if line else "")
return out
@dataclass
class MarkdownRenderer:
blocks: Dict[str, dict]
image_url_to_local: Dict[str, str]
def render_page(self, page_id: str) -> str:
root = self.blocks.get(page_id)
if not root:
return ""
lines = self.render_children(root.get("sub_nodes") or [])
while lines and not lines[-1].strip():
lines.pop()
return "\n".join(lines) + "\n"
def render_children(self, child_ids: Sequence[str]) -> List[str]:
lines: List[str] = []
i = 0
while i < len(child_ids):
block = self.blocks.get(child_ids[i])
i += 1
if not block:
continue
btype = block.get("type")
if btype in {"enumList", "bullList", "todoList", "todoListPro"}:
seq_lines, new_index = self.render_list_sequence(child_ids, i - 1)
lines.extend(seq_lines)
i = new_index
continue
block_lines = self.render_block(block)
if not block_lines:
continue
lines.extend(block_lines)
if block_lines[-1].strip():
lines.append("")
return lines
def render_list_sequence(self, ids: Sequence[str], start: int) -> Tuple[List[str], int]:
lines: List[str] = []
i = start
first = self.blocks.get(ids[start]) or {}
first_type = first.get("type")
group = {
"enumList": {"enumList"},
"bullList": {"bullList"},
"todoList": {"todoList", "todoListPro"},
"todoListPro": {"todoList", "todoListPro"},
}.get(first_type, {first_type})
counter = 1
while i < len(ids):
block = self.blocks.get(ids[i])
if not block or block.get("type") not in group:
break
text = rich_text(get_title_fragments(block))
btype = block.get("type")
if btype == "enumList":
prefix = f"{counter}. "
elif btype in {"todoList", "todoListPro"}:
checked = (block.get("attributes") or {}).get("checked") == "yes"
prefix = f"- [{'x' if checked else ' '}] "
else:
prefix = "- "
lines.append(f"{prefix}{text}".rstrip())
child_lines = self.render_children(block.get("sub_nodes") or [])
if child_lines:
lines.extend(indent_lines(child_lines, " "))
counter += 1
i += 1
lines.append("")
return lines, i
def render_block(self, block: dict) -> List[str]:
btype = block.get("type")
text = rich_text(get_title_fragments(block))
if btype == "text":
return [text] if text else []
if btype in {"midHeader", "subHeader", "tinyHeader"}:
level = {"midHeader": "##", "subHeader": "###", "tinyHeader": "####"}[btype]
return [f"{level} {text}".rstrip()]
if btype == "quote":
quote_lines = [f"> {line}" if line else ">" for line in (text.splitlines() or [""])]
child_lines = self.render_children(block.get("sub_nodes") or [])
if child_lines:
quote_lines.extend(f"> {line}" if line else ">" for line in child_lines)
return quote_lines
if btype == "divider":
return ["---"]
if btype == "image":
url = build_image_url(block)
if not url:
return ["![](图片资源)"]
local = self.image_url_to_local.get(url)
return [f"![]({local or url})"]
if btype == "code":
language = (block.get("attributes") or {}).get("language") or ""
body = text
fence = f"```{str(language).lower()}" if language else "```"
return [fence, body, "```"]
if btype in {"row", "column"}:
return self.render_children(block.get("sub_nodes") or [])
if btype == "toggleList":
title = text or "折叠列表"
child_lines = indent_lines(self.render_children(block.get("sub_nodes") or []), " ")
return [f"- **{title}**", *child_lines]
if btype in {"toggleSubHeader", "toggleTinyHeader"}:
level = "####" if btype == "toggleSubHeader" else "#####"
title = (text + "（可折叠）").strip()
child_lines = self.render_children(block.get("sub_nodes") or [])
return [f"{level} {title}", *child_lines]
if btype == "simpleTable":
return self.render_table(block)
if btype == "progressBar":
progress = (block.get("attributes") or {}).get("progress", 0)
return [f"进度条：{progress}%"]
# 兜底
return [text] if text else []
def render_table(self, block: dict) -> List[str]:
attrs = block.get("attributes") or {}
raw = attrs.get("cells")
if not raw:
return []
try:
cells = json.loads(raw)
except json.JSONDecodeError:
return []
rows: List[List[str]] = []
for row in cells:
cols = []
for cell in row.get("column", []):
cols.append(rich_text((cell.get("attributes") or {}).get("title") or []))
rows.append(cols)
if not rows:
return []
width = max(len(r) for r in rows)
for r in rows:
r.extend([""] * (width - len(r)))
header = rows[0]
sep = ["---"] * len(header)
lines = ["| " + " | ".join(header) + " |", "| " + " | ".join(sep) + " |"]
for r in rows[1:]:
lines.append("| " + " | ".join(r) + " |")
return lines
def load_seed_ids(seed_urls: List[str], seed_file: Optional[Path]) -> List[str]:
ids: List[str] = []
for u in seed_urls:
pid = parse_page_id(u)
if pid:
ids.append(pid)
if seed_file and seed_file.exists():
for line in _read_text(seed_file).splitlines():
line = line.strip()
if not line or line.startswith("#"):
continue
pid = parse_page_id(line)
if pid:
ids.append(pid)
# 去重但保持顺序
seen: Set[str] = set()
out: List[str] = []
for pid in ids:
if pid in seen:
continue
seen.add(pid)
out.append(pid)
return out
def download_images(
session: requests.Session,
image_urls: List[str],
out_images_dir: Path,
) -> Dict[str, str]:
"""下载图片并返回 url -> 相对路径 的映射。"""
mapping: Dict[str, str] = {}
out_images_dir.mkdir(parents=True, exist_ok=True)
for url in image_urls:
if url in mapping:
continue
digest = sha256_hex(url)[:24]
# 若已下载过（不同页面复用），直接复用
existing = list(out_images_dir.glob(f"{digest}.*"))
if existing:
mapping[url] = str(Path("images") / existing[0].name).replace("\\", "/")
continue
try:
resp = session.get(url, timeout=60)
resp.raise_for_status()
ext = guess_extension(resp.headers.get("content-type", ""), url)
filename = f"{digest}{ext}"
out_path = out_images_dir / filename
out_path.write_bytes(resp.content)
mapping[url] = str(Path("images") / filename).replace("\\", "/")
except Exception:
# 下载失败也要留痕，避免重复尝试拖慢整体
mapping[url] = url
return mapping
def main(argv: Optional[List[str]] = None) -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--seed-url", action="append", default=[], help="种子页面 URL 或 ID（可重复指定）")
parser.add_argument("--seed-file", type=str, default="", help="种子列表文件（每行一个 URL 或 ID）")
parser.add_argument("--out", type=str, default="artifacts/wolai-help-center", help="输出目录")
parser.add_argument("--max-pages", type=int, default=200, help="最大抓取页面数（防止无限扩张）")
parser.add_argument("--download-images", action="store_true", help="下载图片到本地并在 Markdown 中引用")
parser.add_argument("--sleep-ms", type=int, default=0, help="每页抓取后的延迟（毫秒）")
parser.add_argument("--resume", action="store_true", help="从 state.json 恢复")
args = parser.parse_args(argv)
out_dir = Path(args.out)
pages_dir = out_dir / "pages"
images_dir = out_dir / "images"
state_path = out_dir / "state.json"
index_path = out_dir / "index.json"
seed_file = Path(args.seed_file) if args.seed_file else None
seeds = load_seed_ids(args.seed_url, seed_file)
if not seeds:
print("未提供 seed（--seed-url 或 --seed-file）。", file=sys.stderr)
return 2
api = WolaiApiClient()
allowed_page_id: Optional[str] = None
queue: deque[str] = deque(seeds)
seen: Set[str] = set()
results: List[dict] = []
failures: List[dict] = []
if args.resume and state_path.exists():
try:
state = json.loads(_read_text(state_path))
allowed_page_id = state.get("allowed_page_id") or None
queue = deque(state.get("queue") or [])
seen = set(state.get("seen") or [])
results = state.get("results") or []
failures = state.get("failures") or []
except Exception:
pass
exported = 0
while queue and exported < args.max_pages:
page_id = queue.popleft()
if page_id in seen:
continue
seen.add(page_id)
try:
blocks = api.fetch_blocks_recursive(page_id)
root = blocks.get(page_id)
if not root:
raise RuntimeError("root block 缺失")
if root.get("type") != "page":
# 队列里可能混入 blockId；仅导出“页面”类型，避免爆炸式爬取
continue
if allowed_page_id is None:
allowed_page_id = root.get("page_id") if isinstance(root.get("page_id"), str) else None
if allowed_page_id and root.get("page_id") != allowed_page_id:
# 只抓同一个帮助中心空间，避免爬到用户分享页/其它空间
continue
title = rich_text(get_title_fragments(root)) or page_id
safe_title = sanitize_filename(title)
md_name = f"{page_id}__{safe_title}.md"
json_name = f"{page_id}__{safe_title}.json"
page_json_path = pages_dir / json_name
page_md_path = pages_dir / md_name
image_urls: List[str] = []
if args.download_images:
for b in blocks.values():
if b.get("type") == "image":
url = build_image_url(b)
if url:
image_urls.append(url)
image_map: Dict[str, str] = {}
if args.download_images and image_urls:
image_map = download_images(api.session, image_urls, images_dir)
renderer = MarkdownRenderer(blocks=blocks, image_url_to_local=image_map)
body = renderer.render_page(page_id)
header = {
"source": f"https://www.wolai.com/wolai/{page_id}",
"title": title,
"pageId": page_id,
"exportedAt": int(time.time() * 1000),
"imageCount": len(image_urls),
}
md = "---\n" + json.dumps(header, ensure_ascii=False, indent=2) + "\n---\n\n" + body
_write_text(page_md_path, md)
# blocks 数据落盘（便于后续结构化分析）
_write_json(
page_json_path,
{
"meta": header,
"blocks": blocks,
"images": [{"url": u, "local": image_map.get(u, u)} for u in image_urls],
},
)
results.append(
{
"pageId": page_id,
"title": title,
"source": header["source"],
"md": str(page_md_path.relative_to(out_dir)).replace("\\", "/"),
"json": str(page_json_path.relative_to(out_dir)).replace("\\", "/"),
"imageCount": len(image_urls),
}
)
# 继续爬取下一层链接
next_ids = extract_page_ids_from_blocks(blocks)
for nid in sorted(next_ids):
if nid not in seen:
queue.append(nid)
exported += 1
if args.sleep_ms > 0:
time.sleep(args.sleep_ms / 1000)
except Exception as e:
failures.append({"pageId": page_id, "error": str(e)})
# 持久化状态，避免中断重来
_write_json(
state_path,
{
"allowed_page_id": allowed_page_id,
"queue": list(queue),
"seen": sorted(seen),
"results": results,
"failures": failures,
},
)
_write_json(index_path, {"results": results, "failures": failures, "allowed_page_id": allowed_page_id})
print(f"已导出页面：{exported}，输出目录：{out_dir}")
print(f"索引：{index_path}")
if failures:
print(f"失败：{len(failures)}（见 {state_path} / {index_path}）")
return 0
if __name__ == "__main__":
raise SystemExit(main())
scripts/wolai_help_center/extract_outlines.py
+81
View File
@@ -0,0 +1,81 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
从 Wolai 帮助中心导出语料中抽取 Markdown 大纲（仅标题行），用于快速定位“页面组成/页面操作”内容。
示例：
python scripts/wolai_help_center/extract_outlines.py --out artifacts/wolai-help-center-v4 --title-keyword 页面
"""
from __future__ import annotations
import argparse
import json
from pathlib import Path
from typing import Any, Dict, List
def read_front_matter(md: str) -> Dict[str, Any]:
if not md.startswith("---"):
return {}
try:
meta_json = md.split("---", 2)[1].strip()
return json.loads(meta_json)
except Exception:
return {}
def extract_headings(md_body: str) -> List[str]:
lines: List[str] = []
for line in md_body.splitlines():
if line.startswith("#"):
lines.append(line.rstrip())
return lines
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--out", type=str, default="artifacts/wolai-help-center-v4", help="抓取输出目录")
parser.add_argument("--title-keyword", type=str, default="页面", help="仅输出标题包含该关键词的页面")
args = parser.parse_args()
out_dir = Path(args.out)
index_path = out_dir / "index.json"
if not index_path.exists():
raise SystemExit(f"未找到 {index_path}")
index = json.loads(index_path.read_text(encoding="utf-8"))
results: List[Dict[str, Any]] = index.get("results") or []
keyword = args.title_keyword.strip()
blocks: List[str] = []
for r in results:
md_rel = str(r.get("md") or "")
if not md_rel:
continue
md_path = out_dir / md_rel
if not md_path.exists():
continue
text = md_path.read_text(encoding="utf-8", errors="replace")
meta = read_front_matter(text)
title = str(meta.get("title") or "")
if keyword and keyword not in title:
continue
md_rel_norm = md_rel.replace("\\", "/")
body = text.split("---", 2)[2] if text.startswith("---") and len(text.split("---", 2)) == 3 else text
headings = extract_headings(body)
blocks.append(f"## {title}\n\n- pageId: {meta.get('pageId')}\n- md: {md_rel_norm}\n")
for h in headings:
blocks.append(f"{h}\n")
blocks.append("\n")
analysis_dir = out_dir / "analysis"
analysis_dir.mkdir(parents=True, exist_ok=True)
out_path = analysis_dir / f"outlines_{keyword or 'all'}.md"
out_path.write_text("".join(blocks), encoding="utf-8")
print(f"已写入：{out_path}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
scripts/wolai_help_center/seeds.txt
+16
View File
@@ -0,0 +1,16 @@
# Wolai 帮助中心（公开）若干入口页
# 每行一个 URL 或 pageId
https://www.wolai.com/wolai/4TsNd1GmGB3RYbahUZy1bz
https://www.wolai.com/wolai/iG4uLuB67GuVgdH8b633Yk
https://www.wolai.com/wolai/akwMqUeEu9JNqq6AzbL4Bk
https://www.wolai.com/wolai/qN1Bh9YjLAXs8bxCJoAJ6C
https://www.wolai.com/wolai/6zRnvRBPX1cXjzYdKq8f2B
https://www.wolai.com/wolai/veSZ7eYZp48cCxLzH2Uwqt
https://www.wolai.com/wolai/iLJdcXJp8nByXA8KyWDCmN
https://www.wolai.com/wolai/j8en3Y2QvdHUkYAGjUcYmS
https://www.wolai.com/wolai/i1eTuzCbCDV4ymqaDPRN5w
https://www.wolai.com/wolai/doMqLaba4V76PByjJXacSc
https://www.wolai.com/wolai/iokpaWtdKAZoHgMJ4SRNvD
https://www.wolai.com/wolai/kssZs57pWUdiimVPL8c48E
https://www.wolai.com/wolai/fZAdxjxWKMCN5mw4EcUDtC