369 lines
16 KiB
JavaScript
369 lines
16 KiB
JavaScript
#!/usr/bin/env node
|
||
"use strict";
|
||
|
||
const assert = require("node:assert");
|
||
const fs = require("node:fs");
|
||
const os = require("node:os");
|
||
const path = require("node:path");
|
||
const { chromium } = require("playwright");
|
||
|
||
const BASE = process.env.MNOTE_UI_BASE_URL || "http://127.0.0.1:3000";
|
||
const STAMP = Date.now();
|
||
const OUT = process.env.MNOTE_PI_REAL_LIGHTRAG_OUT || path.join(os.tmpdir(), `mnote-pi-real-lightrag-${STAMP}`);
|
||
const TIMEOUT = Number.parseInt(process.env.UI_TIMEOUT_MS || "420000", 10);
|
||
const ACTOR_ID = process.env.MNOTE_E2E_ACTOR_ID || "mnote-e2e";
|
||
const WORKSPACE_ID = process.env.MNOTE_E2E_WORKSPACE_ID || "local-ws:mnote-e2e:my-space";
|
||
const ROOT_PATH = process.env.MNOTE_E2E_ROOT_PATH || "/mnt/Data1T/Mnote_data/users/mnote-e2e/workspaces/my-space";
|
||
const ROOT_URI = process.env.MNOTE_E2E_ROOT_URI || `file://${ROOT_PATH}`;
|
||
const MODEL_PROVIDER = process.env.MNOTE_PI_REAL_MODEL_PROVIDER || "omniroute";
|
||
const MODEL_ID = process.env.MNOTE_PI_REAL_MODEL_ID || "gpt-5.4-mini";
|
||
const MARKER = `PI_REAL_LIGHTRAG_CARBOXY_OK_${STAMP}`;
|
||
const CHROMIUM_EXECUTABLE = process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE
|
||
|| (fs.existsSync("/usr/bin/chromium-browser") ? "/usr/bin/chromium-browser" : "")
|
||
|| (fs.existsSync("/usr/bin/chromium") ? "/usr/bin/chromium" : "")
|
||
|| (fs.existsSync("/usr/bin/google-chrome") ? "/usr/bin/google-chrome" : "");
|
||
|
||
function mkdirp(dir) {
|
||
fs.mkdirSync(dir, { recursive: true });
|
||
}
|
||
|
||
async function quickLogin(page) {
|
||
await page.goto(`${BASE}/auth`, { waitUntil: "commit", timeout: TIMEOUT });
|
||
const quickLoginButton = page.getByRole("button", { name: "测试账号快速登录" });
|
||
await quickLoginButton.waitFor({ state: "visible", timeout: TIMEOUT });
|
||
await Promise.all([
|
||
page.waitForURL((url) => !url.toString().includes("/auth"), { timeout: TIMEOUT }).catch(() => null),
|
||
quickLoginButton.click(),
|
||
]);
|
||
}
|
||
|
||
async function requestJson(page, url, options = {}) {
|
||
const response = await page.request.fetch(`${BASE}${url}`, {
|
||
...options,
|
||
headers: {
|
||
accept: "application/json",
|
||
"content-type": "application/json",
|
||
...(options.headers || {}),
|
||
},
|
||
timeout: options.timeout || TIMEOUT,
|
||
});
|
||
const text = await response.text();
|
||
let body = {};
|
||
try {
|
||
body = text ? JSON.parse(text) : {};
|
||
} catch {
|
||
body = { raw: text };
|
||
}
|
||
assert(response.ok(), `${options.method || "GET"} ${url} failed: ${response.status()} ${text.slice(0, 800)}`);
|
||
return body;
|
||
}
|
||
|
||
function policyForLightRag() {
|
||
return {
|
||
defaultModel: `${MODEL_PROVIDER}/${MODEL_ID}`,
|
||
allowedModels: [`${MODEL_PROVIDER}/${MODEL_ID}`],
|
||
tools: {
|
||
"mnote.current_page.read": "allow",
|
||
"mnote.selection.read": "allow",
|
||
"mnote.allowed_roots.describe": "allow",
|
||
"mnote.knowledge_rag.status": "allow",
|
||
"mnote.knowledge_rag.query": "allow",
|
||
"mnote.knowledge_rag.section_context": "allow",
|
||
"mnote.knowledge_rag.open_reference": "allow",
|
||
"mnote.reference.open": "allow",
|
||
"mnote.tool_receipt.write": "allow",
|
||
},
|
||
skills: {},
|
||
mcpServers: {},
|
||
};
|
||
}
|
||
|
||
async function ensureDirectoryGrant(page) {
|
||
const response = await page.request.fetch(`${BASE}/api/admin/access-policy/grants`, {
|
||
method: "POST",
|
||
headers: {
|
||
accept: "application/json",
|
||
"content-type": "application/json",
|
||
},
|
||
data: {
|
||
userId: ACTOR_ID,
|
||
rootUri: ROOT_URI,
|
||
rootPath: ROOT_PATH,
|
||
permission: "write",
|
||
recursive: true,
|
||
capabilities: ["ai"],
|
||
},
|
||
timeout: TIMEOUT,
|
||
});
|
||
const text = await response.text();
|
||
let body = {};
|
||
try {
|
||
body = text ? JSON.parse(text) : {};
|
||
} catch {
|
||
body = { raw: text };
|
||
}
|
||
if (response.ok()) return body;
|
||
if (body && body.code === "local_access_policy_grant_duplicate") return body;
|
||
throw new Error(`POST /api/admin/access-policy/grants failed: ${response.status()} ${text.slice(0, 800)}`);
|
||
}
|
||
|
||
async function seedRuntimePolicy(page) {
|
||
mkdirp(ROOT_PATH);
|
||
await ensureDirectoryGrant(page);
|
||
await requestJson(page, "/api/ai-admin/settings", {
|
||
method: "PUT",
|
||
data: {
|
||
...policyForLightRag(),
|
||
quota: { daily: 200 },
|
||
},
|
||
});
|
||
}
|
||
|
||
async function ensureLightRagReady(page) {
|
||
const status = await requestJson(
|
||
page,
|
||
`/api/knowledge-rag/status?workspaceId=${encodeURIComponent(WORKSPACE_ID)}&rootUri=${encodeURIComponent(ROOT_URI)}`,
|
||
);
|
||
const legacyHealthOk = status && status.legacyHealth && status.legacyHealth.ok === true;
|
||
const documentsOk = status && status.documents && status.documents.ok === true;
|
||
assert(
|
||
legacyHealthOk && documentsOk,
|
||
`LightRAG provider is not ready: ${JSON.stringify({
|
||
legacyHealth: status && status.legacyHealth,
|
||
documents: status && status.documents,
|
||
}).slice(0, 1000)}`,
|
||
);
|
||
return status;
|
||
}
|
||
|
||
async function abortExistingSession(page) {
|
||
const status = await requestJson(page, "/api/page-ai/pi/status").catch(() => null);
|
||
const sessionId = status && status.session && status.session.sessionId;
|
||
if (!sessionId) return null;
|
||
await requestJson(page, "/api/page-ai/pi/abort", { method: "POST", data: { sessionId } }).catch(() => null);
|
||
return sessionId;
|
||
}
|
||
|
||
async function startRealPi(page) {
|
||
const sessionId = `pi-real-lightrag-${STAMP}`;
|
||
const pagePath = `__pi_real_lightrag_${STAMP}.md`;
|
||
fs.writeFileSync(path.join(ROOT_PATH, pagePath), "# Pi real LightRAG smoke\n", "utf8");
|
||
const start = await requestJson(page, "/api/page-ai/pi/start", {
|
||
method: "POST",
|
||
data: {
|
||
sessionId,
|
||
rootUri: ROOT_URI,
|
||
workspaceId: WORKSPACE_ID,
|
||
pagePath,
|
||
pageTitle: "Pi real LightRAG smoke",
|
||
modelProvider: MODEL_PROVIDER,
|
||
modelId: MODEL_ID,
|
||
thinkingLevel: "off",
|
||
permissionMode: "full_access",
|
||
},
|
||
});
|
||
assert.equal(start.ok, true, "Pi start ok should be true");
|
||
assert.equal(start.session.runtimeMode, "rpc", `Pi 必须以 rpc 模式启动，实际=${start.session.runtimeMode}`);
|
||
assert(start.session.runtimePid, "真实 Pi RPC 启动后应有 runtimePid");
|
||
return start.session;
|
||
}
|
||
|
||
async function openPiUi(page) {
|
||
await page.goto(`${BASE}/page-ai/pi`, { waitUntil: "commit", timeout: TIMEOUT });
|
||
await page.locator('[data-page-ai-pi-lab="drawer"]').waitFor({ state: "visible", timeout: TIMEOUT });
|
||
await page.waitForFunction(() => {
|
||
const text = document.querySelector("[data-page-ai-pi-lab-status-text]")?.textContent || "";
|
||
return /ready|running|streaming/.test(text);
|
||
}, null, { timeout: TIMEOUT }).catch(() => null);
|
||
}
|
||
|
||
async function expandToolTimelines(page) {
|
||
const timelines = await page.locator("[data-page-ai-pi-lab-tool-timeline], [data-page-ai-pi-tool-timeline]").all();
|
||
for (const timeline of timelines) {
|
||
await timeline.evaluate((node) => { node.open = true; }).catch(() => {});
|
||
}
|
||
}
|
||
|
||
function readSessionJsonl(sessionDir) {
|
||
const files = [];
|
||
const walk = (dir) => {
|
||
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
||
const full = path.join(dir, entry.name);
|
||
if (entry.isDirectory()) walk(full);
|
||
else if (entry.isFile() && entry.name.endsWith(".jsonl")) files.push(full);
|
||
}
|
||
};
|
||
walk(sessionDir);
|
||
files.sort();
|
||
const sessionFile = files[files.length - 1];
|
||
assert(sessionFile, `session jsonl not found in ${sessionDir}`);
|
||
return { sessionFile, raw: fs.readFileSync(sessionFile, "utf8") };
|
||
}
|
||
|
||
function sessionHasKnowledgeRagEvidence(raw) {
|
||
return String(raw || "")
|
||
.split("\n")
|
||
.filter(Boolean)
|
||
.some((line) => {
|
||
try {
|
||
const entry = JSON.parse(line);
|
||
const message = entry && entry.message;
|
||
if (
|
||
!message
|
||
|| message.role !== "toolResult"
|
||
|| !/mnote_knowledge_rag_query|mnote\.knowledge_rag\.query/.test(String(message.toolName || ""))
|
||
) {
|
||
return false;
|
||
}
|
||
const result = message.details && message.details.result;
|
||
return Boolean(
|
||
result
|
||
&& ((Array.isArray(result.references) && result.references.length)
|
||
|| (Array.isArray(result.citations) && result.citations.length)
|
||
|| (Array.isArray(result.uiCitations) && result.uiCitations.length)),
|
||
);
|
||
} catch {
|
||
return false;
|
||
}
|
||
});
|
||
}
|
||
|
||
async function waitForSessionKnowledgeRagEvidence(sessionDir, timeoutMs = 10000) {
|
||
const started = Date.now();
|
||
let snapshot = readSessionJsonl(sessionDir);
|
||
while (Date.now() - started < timeoutMs) {
|
||
if (sessionHasKnowledgeRagEvidence(snapshot.raw)) return snapshot;
|
||
await new Promise((resolve) => setTimeout(resolve, 200));
|
||
snapshot = readSessionJsonl(sessionDir);
|
||
}
|
||
return snapshot;
|
||
}
|
||
|
||
function extractAssistantText(text) {
|
||
return text
|
||
.replace(/\s+/g, " ")
|
||
.trim();
|
||
}
|
||
|
||
async function main() {
|
||
mkdirp(OUT);
|
||
const browser = await chromium.launch({
|
||
headless: process.env.MNOTE_PI_REAL_LIGHTRAG_HEADED === "1" ? false : true,
|
||
executablePath: CHROMIUM_EXECUTABLE || undefined,
|
||
});
|
||
const context = await browser.newContext({ viewport: { width: 1440, height: 980 } });
|
||
const page = await context.newPage();
|
||
const consoleMessages = [];
|
||
page.on("console", (message) => {
|
||
if (["error", "warning"].includes(message.type())) consoleMessages.push(`${message.type()}: ${message.text()}`);
|
||
});
|
||
page.on("pageerror", (error) => consoleMessages.push(`pageerror: ${error.message}`));
|
||
|
||
const result = {
|
||
base: BASE,
|
||
outputDir: OUT,
|
||
marker: MARKER,
|
||
screenshots: {},
|
||
checks: {},
|
||
session: {},
|
||
consoleMessages,
|
||
};
|
||
|
||
try {
|
||
await quickLogin(page);
|
||
await seedRuntimePolicy(page);
|
||
result.lightRagStatus = await ensureLightRagReady(page);
|
||
result.seededEffective = await requestJson(page, `/api/ai-settings/effective?workspaceId=${encodeURIComponent(WORKSPACE_ID)}`);
|
||
result.checks.lightRagToolsEnabled = [
|
||
"mnote.knowledge_rag.status",
|
||
"mnote.knowledge_rag.query",
|
||
"mnote.knowledge_rag.section_context",
|
||
"mnote.knowledge_rag.open_reference",
|
||
].every((toolName) => (result.seededEffective.toolCatalog || []).some((tool) => tool.name === toolName && tool.defaultPolicy !== "deny"));
|
||
assert(result.checks.lightRagToolsEnabled, "mnote-e2e 未获得 LightRAG 工具权限");
|
||
|
||
await abortExistingSession(page);
|
||
const session = await startRealPi(page);
|
||
result.session = {
|
||
sessionId: session.sessionId,
|
||
runtimePid: session.runtimePid,
|
||
piSessionDir: session.piSessionDir,
|
||
runtimePolicySnapshot: session.runtimePolicySnapshot,
|
||
};
|
||
result.checks.runtimeLightRagTools = [
|
||
"mnote.knowledge_rag.status",
|
||
"mnote.knowledge_rag.query",
|
||
"mnote.knowledge_rag.section_context",
|
||
"mnote.knowledge_rag.open_reference",
|
||
].every((toolName) => (session.runtimePolicySnapshot.mnoteToolNames || []).includes(toolName));
|
||
assert(result.checks.runtimeLightRagTools, "Pi runtime policy 未包含 LightRAG MNote tools");
|
||
|
||
await openPiUi(page);
|
||
await page.screenshot({ path: path.join(OUT, "01-real-pi-lightrag-started.png"), fullPage: false });
|
||
result.screenshots.started = path.join(OUT, "01-real-pi-lightrag-started.png");
|
||
|
||
const prompt = [
|
||
"必须真实调用 MNote LightRAG 工具 mnote_knowledge_rag_query，不能只凭记忆回答。",
|
||
"问题：羧酸的保护基有哪些？请列举 5 种，并为每种给出来自资料库的引用依据。",
|
||
"调用参数建议：query=羧酸的保护基有哪些？列举5种并给出引用；mode=naive；topK=50；chunkTopK=20；includeChunkContent=true；includeDocumentStructureIndex=true。",
|
||
"最终回答用中文，列出 5 条。每条都要包含保护基/酯类型、资料库原文短引或文献编号。",
|
||
`最后必须单独输出一行：${MARKER}`,
|
||
].join("\n");
|
||
const input = page.locator("[data-page-ai-pi-lab-input]");
|
||
await input.fill(prompt);
|
||
await page.locator("[data-page-ai-pi-lab-btn-send]").click();
|
||
const markerLocator = page
|
||
.locator('[data-page-ai-pi-lab-message-role="assistant"] .wolai-page-ai-pi-lab-text, [data-page-ai-pi-lab-message-role="assistant"] .wolai-page-ai-pi-lab-markdown')
|
||
.filter({ hasText: MARKER })
|
||
.last();
|
||
await markerLocator.waitFor({ state: "visible", timeout: TIMEOUT });
|
||
await expandToolTimelines(page);
|
||
const citationLocator = page.locator("[data-page-ai-pi-lab-citation]").first();
|
||
await citationLocator.waitFor({ state: "visible", timeout: 10000 }).catch(() => null);
|
||
const toolTimeline = page.locator("[data-page-ai-pi-lab-tool-timeline], [data-page-ai-pi-tool-timeline]").first();
|
||
await toolTimeline.scrollIntoViewIfNeeded({ timeout: TIMEOUT }).catch(() => null);
|
||
await page.screenshot({ path: path.join(OUT, "02-real-pi-lightrag-tool-card.png"), fullPage: false });
|
||
result.screenshots.toolCard = path.join(OUT, "02-real-pi-lightrag-tool-card.png");
|
||
await markerLocator.scrollIntoViewIfNeeded({ timeout: TIMEOUT }).catch(() => null);
|
||
await page.screenshot({ path: path.join(OUT, "02-real-pi-lightrag-answer.png"), fullPage: false });
|
||
result.screenshots.answer = path.join(OUT, "02-real-pi-lightrag-answer.png");
|
||
|
||
const assistantText = extractAssistantText((await markerLocator.textContent({ timeout: TIMEOUT })) || "");
|
||
result.answerText = assistantText;
|
||
const toolText = (await page.locator("[data-page-ai-pi-lab-tool-timeline], [data-page-ai-pi-tool-timeline]").allTextContents()).join("\n");
|
||
const sessionJsonl = await waitForSessionKnowledgeRagEvidence(session.piSessionDir);
|
||
result.session.sessionFile = sessionJsonl.sessionFile;
|
||
result.checks.toolCardVisible = /LightRAG|knowledge_rag|mnote_knowledge_rag_query|mnote\.knowledge_rag\.query/i.test(toolText);
|
||
result.checks.queryToolCalled = /mnote_knowledge_rag_query|mnote\.knowledge_rag\.query/.test(sessionJsonl.raw);
|
||
result.checks.referencesReturned = sessionHasKnowledgeRagEvidence(sessionJsonl.raw);
|
||
result.checks.citationChipsVisible = await page.locator("[data-page-ai-pi-lab-citation]").count() > 0;
|
||
result.checks.answerHasFiveItems = (assistantText.match(/(^|\s)([1-5]([.、．]|️⃣)|[-*]\s)/g) || []).length >= 5 || /5\s*种|五种|5\s*種|五種/.test(assistantText);
|
||
result.checks.answerMentionsEvidence = /(\[[0-9]{3,4}\]|苯甲酰溴甲酯|碳酸二|硫酸二甲酯|磷酸三甲酯|引用|原文)/.test(assistantText);
|
||
result.checks.noReasoningLeakInToolTimeline = !/reasoning_content|\"thinking\"/i.test(toolText);
|
||
assert(result.checks.toolCardVisible, "UI 未显示 LightRAG/MNote RAG 工具卡");
|
||
assert(result.checks.queryToolCalled, "Pi session JSONL 未记录 LightRAG 查询工具调用");
|
||
assert(result.checks.referencesReturned, "Pi session JSONL 未包含 LightRAG references/citations");
|
||
assert(result.checks.citationChipsVisible, "Pi 最终回答未显示可点击的 LightRAG 引用");
|
||
assert(result.checks.answerHasFiveItems, "Pi 最终回答未明显列出 5 项");
|
||
assert(result.checks.answerMentionsEvidence, "Pi 最终回答未明显包含引用依据");
|
||
assert(result.checks.noReasoningLeakInToolTimeline, "工具时间线不应泄漏 thinking/reasoning 字段");
|
||
|
||
await requestJson(page, "/api/page-ai/pi/abort", { method: "POST", data: { sessionId: session.sessionId } }).catch(() => ({}));
|
||
result.ok = true;
|
||
} catch (error) {
|
||
result.ok = false;
|
||
result.error = error && error.stack ? error.stack : String(error);
|
||
try {
|
||
await page.screenshot({ path: path.join(OUT, "99-failure.png"), fullPage: true });
|
||
result.screenshots.failure = path.join(OUT, "99-failure.png");
|
||
} catch {}
|
||
process.exitCode = 1;
|
||
} finally {
|
||
fs.writeFileSync(path.join(OUT, "result.json"), JSON.stringify(result, null, 2), "utf8");
|
||
await browser.close();
|
||
console.log(JSON.stringify({ ok: result.ok, outputDir: OUT, result: path.join(OUT, "result.json") }, null, 2));
|
||
}
|
||
}
|
||
|
||
main();
|