#!/usr/bin/env node "use strict"; const assert = require("node:assert"); const fs = require("node:fs"); const os = require("node:os"); const path = require("node:path"); const { chromium } = require("playwright"); const BASE = process.env.MNOTE_UI_BASE_URL || "http://127.0.0.1:3000"; const STAMP = Date.now(); const OUT = process.env.MNOTE_PI_EXTENSION_MCP_MATRIX_OUT || path.join(os.tmpdir(), `mnote-pi-extension-mcp-matrix-${STAMP}`); const TIMEOUT = Number.parseInt(process.env.UI_TIMEOUT_MS || "420000", 10); const ACTOR_ID = process.env.MNOTE_E2E_ACTOR_ID || "mnote-e2e"; const WORKSPACE_ID = process.env.MNOTE_PI_EXTENSION_MCP_MATRIX_WORKSPACE_ID || `local-ws:${ACTOR_ID}:my-space`; const ROOT_PATH = process.env.MNOTE_PI_EXTENSION_MCP_MATRIX_ROOT_PATH || `/mnt/Data1T/Mnote_data/users/${ACTOR_ID}/workspaces/my-space`; const ROOT_URI = process.env.MNOTE_PI_EXTENSION_MCP_MATRIX_ROOT_URI || `file://${ROOT_PATH}`; const MODEL_PROVIDER = process.env.MNOTE_PI_EXTENSION_MCP_MATRIX_MODEL_PROVIDER || "omniroute"; const MODEL_ID = process.env.MNOTE_PI_EXTENSION_MCP_MATRIX_MODEL_ID || "gpt-5.4-mini"; const CHROMIUM_EXECUTABLE = process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE || (fs.existsSync("/usr/bin/chromium-browser") ? "/usr/bin/chromium-browser" : "") || (fs.existsSync("/usr/bin/chromium") ? "/usr/bin/chromium" : "") || (fs.existsSync("/usr/bin/google-chrome") ? "/usr/bin/google-chrome" : ""); const CASES = [ { id: "codegraph", kind: "mcp", toolName: "mcp", serverName: "codegraph", expectedWireTool: "codegraph_search", marker: `PI_MATRIX_CODEGRAPH_OK_${STAMP}`, prompt: [ "必须真实调用 MCP 工具,不能只描述。", "先调用 mcp 的 list 模式查看 codegraph 服务工具,再调用 codegraph_search。", "搜索词使用 PiLabSession,项目根使用 /mnt/Data1T/mnote,最多返回 3 条。", "简要写出一个实际命中的文件路径。", ], }, { id: "searxng", kind: "mcp", toolName: "mcp", serverName: "searxng", expectedWireTool: "searxng_search", marker: `PI_MATRIX_SEARXNG_OK_${STAMP}`, prompt: [ "必须真实调用 MCP 工具,不能只描述。", "先调用 mcp 的 list 模式查看 searxng 服务工具,再调用 searxng_search。", "搜索 Rust programming language official website,最多返回 3 条。", "简要写出一个实际返回的标题或网址域名。", ], }, { id: "chrome-bridge", kind: "mcp", toolName: "mcp", serverName: "chrome-bridge", expectedWireTool: "chrome_bridge_session_summary", marker: `PI_MATRIX_CHROME_BRIDGE_OK_${STAMP}`, prompt: [ "必须真实调用 MCP 工具,不能只描述。", "先调用 mcp 的 list 模式查看 chrome-bridge 服务工具,再调用 chrome_bridge_session_summary,参数为空对象。", "简要报告工具实际返回的 bridge 或 extension 状态。", ], }, { id: "todo", kind: "extension", toolName: "todo", minimumResults: 4, marker: `PI_MATRIX_TODO_OK_${STAMP}`, prompt: [ "必须真实调用 Pi 官方 todo 工具,不能只描述。", `先 add 一条文本为 TODO_MATRIX_${STAMP} 的任务,再 list,再 toggle id=1,最后再次 list。`, "简要确认任务已完成。", ], }, { id: "subagent", kind: "extension", toolName: "subagent", minimumResults: 1, marker: `PI_MATRIX_SUBAGENT_OK_${STAMP}`, prompt: [ "必须真实调用 Pi 官方 subagent 工具,不能只描述。", `使用 single 模式:agent=worker,agentScope=user,cwd=${ROOT_PATH}。`, `委派任务为:不要调用任何工具,只回复 CHILD_SUBAGENT_OK_${STAMP}。`, `最终回答必须包含 CHILD_SUBAGENT_OK_${STAMP}。`, ], }, ]; const CASE_FILTER = new Set( String(process.env.MNOTE_PI_EXTENSION_MCP_MATRIX_CASES || "") .split(",") .map((value) => value.trim()) .filter(Boolean), ); const ACTIVE_CASES = CASE_FILTER.size > 0 ? CASES.filter((testCase) => CASE_FILTER.has(testCase.id)) : CASES; function mkdirp(dir) { fs.mkdirSync(dir, { recursive: true }); } async function quickLogin(page) { await page.goto(`${BASE}/auth`, { waitUntil: "commit", timeout: TIMEOUT }); const quickLoginButton = page.getByRole("button", { name: "测试账号快速登录" }); await quickLoginButton.waitFor({ state: "visible", timeout: TIMEOUT }); await Promise.all([ page.waitForURL((url) => !url.toString().includes("/auth"), { timeout: TIMEOUT }).catch(() => null), quickLoginButton.click(), ]); } async function requestJson(page, url, options = {}) { const response = await page.request.fetch(`${BASE}${url}`, { ...options, headers: { accept: "application/json", "content-type": "application/json", ...(options.headers || {}), }, timeout: options.timeout || TIMEOUT, }); const text = await response.text(); let body = {}; try { body = text ? JSON.parse(text) : {}; } catch { body = { raw: text }; } assert(response.ok(), `${options.method || "GET"} ${url} failed: ${response.status()} ${text.slice(0, 1000)}`); return body; } function skillConfig(name, description, source, riskLevel, requiredScopes = []) { return { name, description, source, riskLevel, requiredScopes, enabled: true }; } function mcpConfig(name, description, transport, command, url, networkPolicy, secretRefs, riskLevel, requiredScopes = []) { return { name, description, transport, command, url, networkPolicy, secretRefs, riskLevel, requiredScopes, enabled: true, facadeOnly: true, sandbox: true, }; } function piExtensionConfig(name, description, source, toolNames, riskLevel, requiredScopes = []) { return { name, description, source, toolNames, riskLevel, requiredScopes, enabled: true }; } function policyForMatrix() { return { defaultModel: `${MODEL_PROVIDER}/${MODEL_ID}`, allowedModels: [`${MODEL_PROVIDER}/${MODEL_ID}`], tools: { "mnote.current_page.read": "allow", "mnote.allowed_roots.describe": "allow", "mnote.tool_receipt.write": "allow", }, skills: { codegraph: skillConfig( "CodeGraph", "读取项目代码图、符号和调用关系。", "mcp://codegraph", "medium", ["workspace:code-read"], ), searxng: skillConfig( "SearXNG Search", "通过本地 SearXNG MCP 做网页检索。", "mcp://searxng", "medium", ["network:search"], ), "chrome-bridge": skillConfig( "Chrome Bridge", "通过受控浏览器桥接执行页面验证。", "mcp://chrome-bridge", "high", ["browser:automation", "qa:browser"], ), }, mcpServers: { codegraph: mcpConfig( "CodeGraph", "代码图 MCP。", "stdio", "codegraph serve --mcp", "", "deny-all", [], "medium", ["workspace:code-read"], ), searxng: mcpConfig( "SearXNG", "本地 SearXNG 检索 MCP。", "stdio", "node /home/lix/.agent-infra/searxng/codex-searxng-mcp.cjs", "", "allow-local", [], "medium", ["network:search"], ), "chrome-bridge": mcpConfig( "Chrome Bridge", "本机 Chromium/Chrome 桥接。", "stdio", "node /home/lix/apps/codex-chrome-bridge/mcp/chrome-bridge-mcp.mjs", "", "allow-local", [], "high", ["browser:automation", "qa:browser"], ), }, piExtensions: { "pi-rust-official-todo": piExtensionConfig( "Pi Rust Official Todo", "Pi Rust 官方 todo 扩展。", "pi-rust-official:todo", ["todo"], "medium", ["workflow:todo"], ), "pi-rust-official-subagent": piExtensionConfig( "Pi Rust Official Subagent", "Pi Rust 官方 subagent 扩展。", "pi-rust-official:subagent", ["subagent"], "high", ["agent:delegate"], ), "pi-rust-official-permission-gate": piExtensionConfig( "Pi Rust Official Permission Gate", "Pi Rust 官方 permission-gate 扩展。", "pi-rust-official:permission-gate", [], "high", ["tool:policy"], ), }, }; } async function seedWorkspace(page) { mkdirp(ROOT_PATH); fs.writeFileSync( path.join(ROOT_PATH, "pi-extension-mcp-matrix.md"), "# Pi extension and MCP matrix smoke\n", "utf8", ); const grantResponse = await page.request.fetch(`${BASE}/api/admin/access-policy/grants`, { method: "POST", headers: { accept: "application/json", "content-type": "application/json", }, data: { userId: ACTOR_ID, rootUri: ROOT_URI, rootPath: ROOT_PATH, permission: "write", recursive: true, capabilities: ["ai"], }, timeout: TIMEOUT, }); const grantText = await grantResponse.text(); let grantBody = {}; try { grantBody = grantText ? JSON.parse(grantText) : {}; } catch { grantBody = { raw: grantText }; } if (!grantResponse.ok() && grantBody.code !== "local_access_policy_grant_duplicate") { throw new Error( `POST /api/admin/access-policy/grants failed: ${grantResponse.status()} ${grantText.slice(0, 1000)}`, ); } await requestJson(page, "/api/ai-admin/settings", { method: "PUT", data: { ...policyForMatrix(), quota: { daily: 200 }, }, }); return requestJson(page, `/api/ai-settings/effective?workspaceId=${encodeURIComponent(WORKSPACE_ID)}`); } async function abortExistingSession(page) { const status = await requestJson(page, "/api/page-ai/pi/status").catch(() => null); const sessionId = status?.session?.sessionId; if (!sessionId) return null; await requestJson(page, "/api/page-ai/pi/abort", { method: "POST", data: { sessionId }, }).catch(() => null); return sessionId; } async function startRealPi(page, testCase) { const sessionId = `pi-extension-mcp-matrix-${testCase.id}-${STAMP}`; const pagePath = `pi-extension-mcp-matrix-${testCase.id}-${STAMP}.md`; fs.writeFileSync(path.join(ROOT_PATH, pagePath), `# ${testCase.id} smoke\n`, "utf8"); const start = await requestJson(page, "/api/page-ai/pi/start", { method: "POST", data: { sessionId, rootUri: ROOT_URI, workspaceId: WORKSPACE_ID, pagePath, pageTitle: `${testCase.id} smoke`, modelProvider: MODEL_PROVIDER, modelId: MODEL_ID, thinkingLevel: "off", permissionMode: "full_access", }, }); assert.equal(start.ok, true, `${testCase.id}: Pi start ok should be true`); assert.equal(start.permissionMode, "full_access", `${testCase.id}: start should expose full_access`); assert.equal( start.session.runtimePolicySnapshot.permissionMode, "full_access", `${testCase.id}: runtime policy should persist full_access`, ); assert.equal(start.session.runtimeMode, "rpc", `${testCase.id}: Pi 必须以 rpc 模式启动`); assert(start.session.runtimePid, `${testCase.id}: 真实 Pi RPC 启动后应有 runtimePid`); return start.session; } async function openPiUi(page) { await page.goto(`${BASE}/page-ai/pi`, { waitUntil: "commit", timeout: TIMEOUT }); await page.locator('[data-page-ai-pi-lab="drawer"]').waitFor({ state: "visible", timeout: TIMEOUT }); await page.waitForFunction(() => { const text = document.querySelector("[data-page-ai-pi-lab-status-text]")?.textContent || ""; return /ready|running|streaming/.test(text); }, null, { timeout: TIMEOUT }).catch(() => null); } async function expandToolTimelines(page) { const timelines = await page.locator( "[data-page-ai-pi-lab-tool-timeline], [data-page-ai-pi-tool-timeline]", ).all(); for (const timeline of timelines) { await timeline.evaluate((node) => { node.open = true; }).catch(() => {}); } } async function sendPrompt(page, testCase) { const fullPrompt = [ ...testCase.prompt, `最后必须单独输出一行:${testCase.marker}`, ].join("\n"); await page.locator("[data-page-ai-pi-lab-input]").fill(fullPrompt); await page.locator("[data-page-ai-pi-lab-btn-send]").click(); const markerLocator = page .locator( '[data-page-ai-pi-lab-message-role="assistant"] .wolai-page-ai-pi-lab-text, ' + '[data-page-ai-pi-lab-message-role="assistant"] .wolai-page-ai-pi-lab-markdown', ) .filter({ hasText: testCase.marker }) .last(); await markerLocator.waitFor({ state: "visible", timeout: TIMEOUT }); await page.waitForFunction(() => { const status = document.querySelector("[data-page-ai-pi-lab-status-text]")?.textContent || ""; return status === "ready"; }, null, { timeout: 30000 }).catch(() => null); await expandToolTimelines(page); return ((await markerLocator.textContent({ timeout: TIMEOUT })) || "").trim(); } function readSessionJsonl(sessionDir) { const files = []; const walk = (dir) => { for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { const target = path.join(dir, entry.name); if (entry.isDirectory()) walk(target); else if (entry.isFile() && entry.name.endsWith(".jsonl")) files.push(target); } }; walk(sessionDir); files.sort(); const sessionFile = files[files.length - 1]; assert(sessionFile, `session jsonl not found in ${sessionDir}`); return { sessionFile, raw: fs.readFileSync(sessionFile, "utf8") }; } function parseSessionMessages(raw) { return String(raw || "") .split("\n") .filter(Boolean) .map((line) => { try { return JSON.parse(line); } catch { return undefined; } }) .filter(Boolean); } function readToolResults(raw, toolName) { return parseSessionMessages(raw) .filter((entry) => entry?.type === "message" && entry?.message?.role === "toolResult" && entry?.message?.toolName === toolName) .map((entry) => entry.message); } function readTargetToolResults(testCase, raw) { const entries = parseSessionMessages(raw); const targetCallIds = new Set(); for (const entry of entries) { if (entry?.type !== "message" || entry?.message?.role !== "assistant") continue; for (const item of entry.message.content || []) { if (item?.type !== "toolCall" || item?.name !== testCase.toolName) continue; if (testCase.kind === "mcp") { if (item.arguments?.server !== testCase.serverName) continue; if (item.arguments?.mode !== "call" || item.arguments?.tool !== testCase.expectedWireTool) continue; } targetCallIds.add(item.id); } } return entries .filter((entry) => entry?.type === "message" && entry?.message?.role === "toolResult" && entry?.message?.toolName === testCase.toolName && targetCallIds.has(entry.message.toolCallId)) .map((entry) => entry.message); } function containsBridgeFailure(text) { return /page_ai_pi_lab_session_not_found|page_ai_pi_lab_bridge_token_invalid|mnote_pi_rust_service_bridge_unavailable|mnote_pi_bridge_session_id_missing/i.test(String(text || "")); } function sessionEvidenceReady(testCase, raw) { const results = readTargetToolResults(testCase, raw); const minimumResults = testCase.minimumResults || 1; if (results.length < minimumResults || !raw.includes(testCase.marker)) return false; if (testCase.kind === "mcp") { return raw.includes(testCase.serverName) && raw.includes(testCase.expectedWireTool); } return raw.includes(`"name":"${testCase.toolName}"`); } async function waitForSessionEvidence(testCase, sessionDir) { const deadline = Date.now() + Math.min(TIMEOUT, 30000); let snapshot = readSessionJsonl(sessionDir); while (Date.now() < deadline) { if (sessionEvidenceReady(testCase, snapshot.raw)) return snapshot; await new Promise((resolve) => setTimeout(resolve, 200)); snapshot = readSessionJsonl(sessionDir); } return snapshot; } async function runCase(page, result, testCase) { await abortExistingSession(page); const session = await startRealPi(page, testCase); result.sessions[testCase.id] = { sessionId: session.sessionId, runtimePid: session.runtimePid, piSessionDir: session.piSessionDir, runtimePolicySnapshot: session.runtimePolicySnapshot, }; try { await openPiUi(page); const answer = await sendPrompt(page, testCase); const sessionJsonl = await waitForSessionEvidence(testCase, session.piSessionDir); const allToolResults = readToolResults(sessionJsonl.raw, testCase.toolName); const toolResults = readTargetToolResults(testCase, sessionJsonl.raw); const toolText = (await page.locator( "[data-page-ai-pi-lab-tool-timeline], [data-page-ai-pi-tool-timeline]", ).allTextContents()).join("\n"); const screenshot = path.join(OUT, `${String(Object.keys(result.sessions).length).padStart(2, "0")}-${testCase.id}.png`); await page.screenshot({ path: screenshot, fullPage: false }); const check = { answer, screenshot, sessionFile: sessionJsonl.sessionFile, toolResultCount: toolResults.length, allToolResultCount: allToolResults.length, failedToolResults: toolResults.filter((message) => message.isError === true).length, allFailedToolResults: allToolResults.filter((message) => message.isError === true).length, toolCardVisible: new RegExp( testCase.kind === "mcp" ? `mcp:${testCase.serverName}|${testCase.serverName}|${testCase.expectedWireTool}` : testCase.toolName, "i", ).test(toolText), toolCallRecorded: sessionJsonl.raw.includes(`"name":"${testCase.toolName}"`), expectedMcpToolRecorded: testCase.kind !== "mcp" || (sessionJsonl.raw.includes(testCase.serverName) && sessionJsonl.raw.includes(testCase.expectedWireTool)), noReasoningLeak: !/reasoning_content|"thinking"/i.test(toolText), noEmptyReplyError: !/Pi runtime 返回了空回复|empty response/i.test(`${answer}\n${toolText}`), noBridgeSessionFailure: !containsBridgeFailure(`${answer}\n${toolText}\n${sessionJsonl.raw}`), }; result.checks[testCase.id] = check; result.screenshots[testCase.id] = screenshot; result.sessions[testCase.id].sessionFile = sessionJsonl.sessionFile; assert(answer.includes(testCase.marker), `${testCase.id}: 最终回复缺少 marker`); assert(check.toolCardVisible, `${testCase.id}: UI 未显示对应工具卡`); assert(check.toolCallRecorded, `${testCase.id}: session JSONL 未记录工具调用`); assert(check.expectedMcpToolRecorded, `${testCase.id}: session JSONL 未记录目标 MCP 工具`); assert( toolResults.length >= (testCase.minimumResults || 1), `${testCase.id}: toolResult 数量不足,实际=${toolResults.length}`, ); assert.equal(check.failedToolResults, 0, `${testCase.id}: 存在 isError=true 的 toolResult`); assert(check.noReasoningLeak, `${testCase.id}: 工具时间线泄漏 thinking/reasoning`); assert(check.noEmptyReplyError, `${testCase.id}: 仍出现 Pi runtime 空回复错误`); assert(check.noBridgeSessionFailure, `${testCase.id}: 工具结果仍包含 Pi bridge/session 错误`); if (testCase.id === "subagent") { assert( answer.includes(`CHILD_SUBAGENT_OK_${STAMP}`), "subagent: 最终回复未包含子 agent 的真实返回", ); } } finally { await requestJson(page, "/api/page-ai/pi/abort", { method: "POST", data: { sessionId: session.sessionId }, }).catch(() => ({})); } } async function main() { mkdirp(OUT); const browser = await chromium.launch({ headless: process.env.MNOTE_PI_EXTENSION_MCP_MATRIX_HEADED === "1" ? false : true, executablePath: CHROMIUM_EXECUTABLE || undefined, }); const context = await browser.newContext({ viewport: { width: 1440, height: 980 } }); const page = await context.newPage(); const consoleMessages = []; page.on("console", (message) => { if (["error", "warning"].includes(message.type())) { consoleMessages.push(`${message.type()}: ${message.text()}`); } }); page.on("pageerror", (error) => consoleMessages.push(`pageerror: ${error.message}`)); const result = { base: BASE, outputDir: OUT, workspaceId: WORKSPACE_ID, rootUri: ROOT_URI, model: `${MODEL_PROVIDER}/${MODEL_ID}`, sessions: {}, checks: {}, screenshots: {}, consoleMessages, }; try { await quickLogin(page); result.seededEffective = await seedWorkspace(page); result.checks.policy = { mcpServers: ["CodeGraph", "SearXNG", "Chrome Bridge"].every( (name) => (result.seededEffective.mcpServers || []).some( (server) => server.name === name && server.enabled !== false, ), ), piExtensions: ["Pi Rust Official Todo", "Pi Rust Official Subagent"].every( (name) => (result.seededEffective.piExtensions || []).some( (extension) => extension.name === name && extension.enabled !== false, ), ), }; assert(result.checks.policy.mcpServers, "有效策略未启用全部目标 MCP"); assert(result.checks.policy.piExtensions, "有效策略未启用 Todo/Subagent 扩展"); assert(ACTIVE_CASES.length > 0, "未匹配到需要执行的 matrix case"); for (const testCase of ACTIVE_CASES) { await runCase(page, result, testCase); } result.ok = true; } catch (error) { result.ok = false; result.error = error && error.stack ? error.stack : String(error); await abortExistingSession(page).catch(() => null); try { const screenshot = path.join(OUT, "99-failure.png"); await page.screenshot({ path: screenshot, fullPage: true }); result.screenshots.failure = screenshot; } catch {} process.exitCode = 1; } finally { fs.writeFileSync(path.join(OUT, "result.json"), JSON.stringify(result, null, 2), "utf8"); await browser.close(); console.log(JSON.stringify({ ok: result.ok, outputDir: OUT, result: path.join(OUT, "result.json"), }, null, 2)); } } main();