643 lines
22 KiB
JavaScript
643 lines
22 KiB
JavaScript
#!/usr/bin/env node
|
||
"use strict";
|
||
|
||
const { loginViaAuthForm, withAdminBrowserSession } = require('./lib/browser-auth-login');
|
||
const assert = require("node:assert");
|
||
const fs = require("node:fs");
|
||
const os = require("node:os");
|
||
const path = require("node:path");
|
||
const { chromium } = require("playwright");
|
||
|
||
const BASE = process.env.MNOTE_UI_BASE_URL || "http://127.0.0.1:3000";
|
||
const STAMP = Date.now();
|
||
const OUT = process.env.MNOTE_PI_EXTENSION_MCP_MATRIX_OUT
|
||
|| path.join(os.tmpdir(), `mnote-pi-extension-mcp-matrix-${STAMP}`);
|
||
const TIMEOUT = Number.parseInt(process.env.UI_TIMEOUT_MS || "420000", 10);
|
||
const ACTOR_ID = process.env.MNOTE_E2E_ACTOR_ID || "mnote-e2e";
|
||
const WORKSPACE_ID = process.env.MNOTE_PI_EXTENSION_MCP_MATRIX_WORKSPACE_ID
|
||
|| `local-ws:${ACTOR_ID}:my-space`;
|
||
const ROOT_PATH = process.env.MNOTE_PI_EXTENSION_MCP_MATRIX_ROOT_PATH
|
||
|| `/mnt/Data1T/Mnote_data/users/${ACTOR_ID}/workspaces/my-space`;
|
||
const ROOT_URI = process.env.MNOTE_PI_EXTENSION_MCP_MATRIX_ROOT_URI || `file://${ROOT_PATH}`;
|
||
const MODEL_PROVIDER = process.env.MNOTE_PI_EXTENSION_MCP_MATRIX_MODEL_PROVIDER || "omniroute";
|
||
const MODEL_ID = process.env.MNOTE_PI_EXTENSION_MCP_MATRIX_MODEL_ID || "gpt-5.4-mini";
|
||
const CHROMIUM_EXECUTABLE = process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE
|
||
|| (fs.existsSync("/usr/bin/chromium-browser") ? "/usr/bin/chromium-browser" : "")
|
||
|| (fs.existsSync("/usr/bin/chromium") ? "/usr/bin/chromium" : "")
|
||
|| (fs.existsSync("/usr/bin/google-chrome") ? "/usr/bin/google-chrome" : "");
|
||
|
||
const CASES = [
|
||
{
|
||
id: "codegraph",
|
||
kind: "mcp",
|
||
toolName: "mcp",
|
||
serverName: "codegraph",
|
||
expectedWireTool: "codegraph_search",
|
||
marker: `PI_MATRIX_CODEGRAPH_OK_${STAMP}`,
|
||
prompt: [
|
||
"必须真实调用 MCP 工具，不能只描述。",
|
||
"先调用 mcp 的 list 模式查看 codegraph 服务工具，再调用 codegraph_search。",
|
||
"搜索词使用 PiLabSession，项目根使用 /mnt/Data1T/mnote，最多返回 3 条。",
|
||
"简要写出一个实际命中的文件路径。",
|
||
],
|
||
},
|
||
{
|
||
id: "searxng",
|
||
kind: "mcp",
|
||
toolName: "mcp",
|
||
serverName: "searxng",
|
||
expectedWireTool: "searxng_search",
|
||
marker: `PI_MATRIX_SEARXNG_OK_${STAMP}`,
|
||
prompt: [
|
||
"必须真实调用 MCP 工具，不能只描述。",
|
||
"先调用 mcp 的 list 模式查看 searxng 服务工具，再调用 searxng_search。",
|
||
"搜索 Rust programming language official website，最多返回 3 条。",
|
||
"简要写出一个实际返回的标题或网址域名。",
|
||
],
|
||
},
|
||
{
|
||
id: "chrome-bridge",
|
||
kind: "mcp",
|
||
toolName: "mcp",
|
||
serverName: "chrome-bridge",
|
||
expectedWireTool: "chrome_bridge_session_summary",
|
||
marker: `PI_MATRIX_CHROME_BRIDGE_OK_${STAMP}`,
|
||
prompt: [
|
||
"必须真实调用 MCP 工具，不能只描述。",
|
||
"先调用 mcp 的 list 模式查看 chrome-bridge 服务工具，再调用 chrome_bridge_session_summary，参数为空对象。",
|
||
"简要报告工具实际返回的 bridge 或 extension 状态。",
|
||
],
|
||
},
|
||
{
|
||
id: "todo",
|
||
kind: "extension",
|
||
toolName: "todo",
|
||
minimumResults: 4,
|
||
marker: `PI_MATRIX_TODO_OK_${STAMP}`,
|
||
prompt: [
|
||
"必须真实调用 Pi 官方 todo 工具，不能只描述。",
|
||
`先 add 一条文本为 TODO_MATRIX_${STAMP} 的任务，再 list，再 toggle id=1，最后再次 list。`,
|
||
"简要确认任务已完成。",
|
||
],
|
||
},
|
||
{
|
||
id: "subagent",
|
||
kind: "extension",
|
||
toolName: "subagent",
|
||
minimumResults: 1,
|
||
marker: `PI_MATRIX_SUBAGENT_OK_${STAMP}`,
|
||
prompt: [
|
||
"必须真实调用 Pi 官方 subagent 工具，不能只描述。",
|
||
`使用 single 模式：agent=worker，agentScope=user，cwd=${ROOT_PATH}。`,
|
||
`委派任务为：不要调用任何工具，只回复 CHILD_SUBAGENT_OK_${STAMP}。`,
|
||
`最终回答必须包含 CHILD_SUBAGENT_OK_${STAMP}。`,
|
||
],
|
||
},
|
||
];
|
||
const CASE_FILTER = new Set(
|
||
String(process.env.MNOTE_PI_EXTENSION_MCP_MATRIX_CASES || "")
|
||
.split(",")
|
||
.map((value) => value.trim())
|
||
.filter(Boolean),
|
||
);
|
||
const ACTIVE_CASES = CASE_FILTER.size > 0
|
||
? CASES.filter((testCase) => CASE_FILTER.has(testCase.id))
|
||
: CASES;
|
||
|
||
function mkdirp(dir) {
|
||
fs.mkdirSync(dir, { recursive: true });
|
||
}
|
||
|
||
async function quickLogin(page) {
|
||
// 7-76 P0: 标准表单登录（无测试快速登录按钮）
|
||
const base =
|
||
(typeof BASE_URL !== "undefined" && BASE_URL) ||
|
||
(typeof baseUrl !== "undefined" && baseUrl) ||
|
||
process.env.MNOTE_UI_BASE_URL ||
|
||
"http://127.0.0.1:3000";
|
||
const timeout =
|
||
(typeof UI_TIMEOUT_MS !== "undefined" && UI_TIMEOUT_MS) ||
|
||
(typeof TIMEOUT !== "undefined" && TIMEOUT) ||
|
||
30_000;
|
||
if (!String(page.url() || "").includes("/auth")) {
|
||
await page.goto(String(base).replace(/\/+$/, "") + "/auth", {
|
||
waitUntil: "commit",
|
||
timeout,
|
||
});
|
||
}
|
||
await loginViaAuthForm(page, {
|
||
baseUrl: base,
|
||
timeoutMs: timeout,
|
||
gotoAuth: false,
|
||
});
|
||
await page
|
||
.waitForURL((url) => !String(url).includes("/auth"), { timeout })
|
||
.catch(() => {});
|
||
}
|
||
|
||
async function requestJson(page, url, options = {}) {
|
||
const response = await page.request.fetch(`${BASE}${url}`, {
|
||
...options,
|
||
headers: {
|
||
accept: "application/json",
|
||
"content-type": "application/json",
|
||
...(options.headers || {}),
|
||
},
|
||
timeout: options.timeout || TIMEOUT,
|
||
});
|
||
const text = await response.text();
|
||
let body = {};
|
||
try {
|
||
body = text ? JSON.parse(text) : {};
|
||
} catch {
|
||
body = { raw: text };
|
||
}
|
||
assert(response.ok(), `${options.method || "GET"} ${url} failed: ${response.status()} ${text.slice(0, 1000)}`);
|
||
return body;
|
||
}
|
||
|
||
function skillConfig(name, description, source, riskLevel, requiredScopes = []) {
|
||
return { name, description, source, riskLevel, requiredScopes, enabled: true };
|
||
}
|
||
|
||
function mcpConfig(name, description, transport, command, url, networkPolicy, secretRefs, riskLevel, requiredScopes = []) {
|
||
return {
|
||
name,
|
||
description,
|
||
transport,
|
||
command,
|
||
url,
|
||
networkPolicy,
|
||
secretRefs,
|
||
riskLevel,
|
||
requiredScopes,
|
||
enabled: true,
|
||
facadeOnly: true,
|
||
sandbox: true,
|
||
};
|
||
}
|
||
|
||
function piExtensionConfig(name, description, source, toolNames, riskLevel, requiredScopes = []) {
|
||
return { name, description, source, toolNames, riskLevel, requiredScopes, enabled: true };
|
||
}
|
||
|
||
function policyForMatrix() {
|
||
return {
|
||
defaultModel: `${MODEL_PROVIDER}/${MODEL_ID}`,
|
||
allowedModels: [`${MODEL_PROVIDER}/${MODEL_ID}`],
|
||
tools: {
|
||
"mnote.current_page.read": "allow",
|
||
"mnote.allowed_roots.describe": "allow",
|
||
"mnote.tool_receipt.write": "allow",
|
||
},
|
||
skills: {
|
||
codegraph: skillConfig(
|
||
"CodeGraph",
|
||
"读取项目代码图、符号和调用关系。",
|
||
"mcp://codegraph",
|
||
"medium",
|
||
["workspace:code-read"],
|
||
),
|
||
searxng: skillConfig(
|
||
"SearXNG Search",
|
||
"通过本地 SearXNG MCP 做网页检索。",
|
||
"mcp://searxng",
|
||
"medium",
|
||
["network:search"],
|
||
),
|
||
"chrome-bridge": skillConfig(
|
||
"Chrome Bridge",
|
||
"通过受控浏览器桥接执行页面验证。",
|
||
"mcp://chrome-bridge",
|
||
"high",
|
||
["browser:automation", "qa:browser"],
|
||
),
|
||
},
|
||
mcpServers: {
|
||
codegraph: mcpConfig(
|
||
"CodeGraph",
|
||
"代码图 MCP。",
|
||
"stdio",
|
||
"codegraph serve --mcp",
|
||
"",
|
||
"deny-all",
|
||
[],
|
||
"medium",
|
||
["workspace:code-read"],
|
||
),
|
||
searxng: mcpConfig(
|
||
"SearXNG",
|
||
"本地 SearXNG 检索 MCP。",
|
||
"stdio",
|
||
"node /home/lix/.agent-infra/searxng/codex-searxng-mcp.cjs",
|
||
"",
|
||
"allow-local",
|
||
[],
|
||
"medium",
|
||
["network:search"],
|
||
),
|
||
"chrome-bridge": mcpConfig(
|
||
"Chrome Bridge",
|
||
"本机 Chromium/Chrome 桥接。",
|
||
"stdio",
|
||
"node /home/lix/apps/codex-chrome-bridge/mcp/chrome-bridge-mcp.mjs",
|
||
"",
|
||
"allow-local",
|
||
[],
|
||
"high",
|
||
["browser:automation", "qa:browser"],
|
||
),
|
||
},
|
||
piExtensions: {
|
||
"pi-rust-official-todo": piExtensionConfig(
|
||
"Pi Rust Official Todo",
|
||
"Pi Rust 官方 todo 扩展。",
|
||
"pi-rust-official:todo",
|
||
["todo"],
|
||
"medium",
|
||
["workflow:todo"],
|
||
),
|
||
"pi-rust-official-subagent": piExtensionConfig(
|
||
"Pi Rust Official Subagent",
|
||
"Pi Rust 官方 subagent 扩展。",
|
||
"pi-rust-official:subagent",
|
||
["subagent"],
|
||
"high",
|
||
["agent:delegate"],
|
||
),
|
||
"pi-rust-official-permission-gate": piExtensionConfig(
|
||
"Pi Rust Official Permission Gate",
|
||
"Pi Rust 官方 permission-gate 扩展。",
|
||
"pi-rust-official:permission-gate",
|
||
[],
|
||
"high",
|
||
["tool:policy"],
|
||
),
|
||
},
|
||
};
|
||
}
|
||
|
||
async function seedWorkspace(browser, page) {
|
||
mkdirp(ROOT_PATH);
|
||
fs.writeFileSync(
|
||
path.join(ROOT_PATH, "pi-extension-mcp-matrix.md"),
|
||
"# Pi extension and MCP matrix smoke\n",
|
||
"utf8",
|
||
);
|
||
// 方案 A：admin 能力仅 mnote-admin
|
||
await withAdminBrowserSession(browser, async (adminPage) => {
|
||
const grantResponse = await adminPage.request.fetch(`${BASE}/api/admin/access-policy/grants`, {
|
||
method: "POST",
|
||
headers: {
|
||
accept: "application/json",
|
||
"content-type": "application/json",
|
||
},
|
||
data: {
|
||
userId: ACTOR_ID,
|
||
rootUri: ROOT_URI,
|
||
rootPath: ROOT_PATH,
|
||
permission: "write",
|
||
recursive: true,
|
||
capabilities: ["ai"],
|
||
},
|
||
timeout: TIMEOUT,
|
||
});
|
||
const grantText = await grantResponse.text();
|
||
let grantBody = {};
|
||
try {
|
||
grantBody = grantText ? JSON.parse(grantText) : {};
|
||
} catch {
|
||
grantBody = { raw: grantText };
|
||
}
|
||
if (!grantResponse.ok() && grantBody.code !== "local_access_policy_grant_duplicate") {
|
||
throw new Error(
|
||
`POST /api/admin/access-policy/grants failed: ${grantResponse.status()} ${grantText.slice(0, 1000)}`,
|
||
);
|
||
}
|
||
await requestJson(adminPage, "/api/ai-admin/settings", {
|
||
method: "PUT",
|
||
data: {
|
||
...policyForMatrix(),
|
||
quota: { daily: 200 },
|
||
},
|
||
});
|
||
}, { baseUrl: BASE, timeoutMs: TIMEOUT });
|
||
return requestJson(page, `/api/ai-settings/effective?workspaceId=${encodeURIComponent(WORKSPACE_ID)}`);
|
||
}
|
||
|
||
async function abortExistingSession(page) {
|
||
const status = await requestJson(page, "/api/page-ai/pi/status").catch(() => null);
|
||
const sessionId = status?.session?.sessionId;
|
||
if (!sessionId) return null;
|
||
await requestJson(page, "/api/page-ai/pi/abort", {
|
||
method: "POST",
|
||
data: { sessionId },
|
||
}).catch(() => null);
|
||
return sessionId;
|
||
}
|
||
|
||
async function startRealPi(page, testCase) {
|
||
const sessionId = `pi-extension-mcp-matrix-${testCase.id}-${STAMP}`;
|
||
const pagePath = `pi-extension-mcp-matrix-${testCase.id}-${STAMP}.md`;
|
||
fs.writeFileSync(path.join(ROOT_PATH, pagePath), `# ${testCase.id} smoke\n`, "utf8");
|
||
const start = await requestJson(page, "/api/page-ai/pi/start", {
|
||
method: "POST",
|
||
data: {
|
||
sessionId,
|
||
rootUri: ROOT_URI,
|
||
workspaceId: WORKSPACE_ID,
|
||
pagePath,
|
||
pageTitle: `${testCase.id} smoke`,
|
||
modelProvider: MODEL_PROVIDER,
|
||
modelId: MODEL_ID,
|
||
thinkingLevel: "off",
|
||
permissionMode: "full_access",
|
||
},
|
||
});
|
||
assert.equal(start.ok, true, `${testCase.id}: Pi start ok should be true`);
|
||
assert.equal(start.permissionMode, "full_access", `${testCase.id}: start should expose full_access`);
|
||
assert.equal(
|
||
start.session.runtimePolicySnapshot.permissionMode,
|
||
"full_access",
|
||
`${testCase.id}: runtime policy should persist full_access`,
|
||
);
|
||
assert.equal(start.session.runtimeMode, "rpc", `${testCase.id}: Pi 必须以 rpc 模式启动`);
|
||
assert(start.session.runtimePid, `${testCase.id}: 真实 Pi RPC 启动后应有 runtimePid`);
|
||
return start.session;
|
||
}
|
||
|
||
async function openPiUi(page) {
|
||
await page.goto(`${BASE}/page-ai/pi`, { waitUntil: "commit", timeout: TIMEOUT });
|
||
await page.locator('[data-page-ai-pi-lab="drawer"]').waitFor({ state: "visible", timeout: TIMEOUT });
|
||
await page.waitForFunction(() => {
|
||
const text = document.querySelector("[data-page-ai-pi-lab-status-text]")?.textContent || "";
|
||
return /ready|running|streaming/.test(text);
|
||
}, null, { timeout: TIMEOUT }).catch(() => null);
|
||
}
|
||
|
||
async function expandToolTimelines(page) {
|
||
const timelines = await page.locator(
|
||
"[data-page-ai-pi-lab-tool-timeline], [data-page-ai-pi-tool-timeline]",
|
||
).all();
|
||
for (const timeline of timelines) {
|
||
await timeline.evaluate((node) => {
|
||
node.open = true;
|
||
}).catch(() => {});
|
||
}
|
||
}
|
||
|
||
async function sendPrompt(page, testCase) {
|
||
const fullPrompt = [
|
||
...testCase.prompt,
|
||
`最后必须单独输出一行：${testCase.marker}`,
|
||
].join("\n");
|
||
await page.locator("[data-page-ai-pi-lab-input]").fill(fullPrompt);
|
||
await page.locator("[data-page-ai-pi-lab-btn-send]").click();
|
||
const markerLocator = page
|
||
.locator(
|
||
'[data-page-ai-pi-lab-message-role="assistant"] .wolai-page-ai-pi-lab-text, '
|
||
+ '[data-page-ai-pi-lab-message-role="assistant"] .wolai-page-ai-pi-lab-markdown',
|
||
)
|
||
.filter({ hasText: testCase.marker })
|
||
.last();
|
||
await markerLocator.waitFor({ state: "visible", timeout: TIMEOUT });
|
||
await page.waitForFunction(() => {
|
||
const status = document.querySelector("[data-page-ai-pi-lab-status-text]")?.textContent || "";
|
||
return status === "ready";
|
||
}, null, { timeout: 30000 }).catch(() => null);
|
||
await expandToolTimelines(page);
|
||
return ((await markerLocator.textContent({ timeout: TIMEOUT })) || "").trim();
|
||
}
|
||
|
||
function readSessionJsonl(sessionDir) {
|
||
const files = [];
|
||
const walk = (dir) => {
|
||
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
||
const target = path.join(dir, entry.name);
|
||
if (entry.isDirectory()) walk(target);
|
||
else if (entry.isFile() && entry.name.endsWith(".jsonl")) files.push(target);
|
||
}
|
||
};
|
||
walk(sessionDir);
|
||
files.sort();
|
||
const sessionFile = files[files.length - 1];
|
||
assert(sessionFile, `session jsonl not found in ${sessionDir}`);
|
||
return { sessionFile, raw: fs.readFileSync(sessionFile, "utf8") };
|
||
}
|
||
|
||
function parseSessionMessages(raw) {
|
||
return String(raw || "")
|
||
.split("\n")
|
||
.filter(Boolean)
|
||
.map((line) => {
|
||
try {
|
||
return JSON.parse(line);
|
||
} catch {
|
||
return undefined;
|
||
}
|
||
})
|
||
.filter(Boolean);
|
||
}
|
||
|
||
function readToolResults(raw, toolName) {
|
||
return parseSessionMessages(raw)
|
||
.filter((entry) => entry?.type === "message"
|
||
&& entry?.message?.role === "toolResult"
|
||
&& entry?.message?.toolName === toolName)
|
||
.map((entry) => entry.message);
|
||
}
|
||
|
||
function readTargetToolResults(testCase, raw) {
|
||
const entries = parseSessionMessages(raw);
|
||
const targetCallIds = new Set();
|
||
for (const entry of entries) {
|
||
if (entry?.type !== "message" || entry?.message?.role !== "assistant") continue;
|
||
for (const item of entry.message.content || []) {
|
||
if (item?.type !== "toolCall" || item?.name !== testCase.toolName) continue;
|
||
if (testCase.kind === "mcp") {
|
||
if (item.arguments?.server !== testCase.serverName) continue;
|
||
if (item.arguments?.mode !== "call" || item.arguments?.tool !== testCase.expectedWireTool) continue;
|
||
}
|
||
targetCallIds.add(item.id);
|
||
}
|
||
}
|
||
return entries
|
||
.filter((entry) => entry?.type === "message"
|
||
&& entry?.message?.role === "toolResult"
|
||
&& entry?.message?.toolName === testCase.toolName
|
||
&& targetCallIds.has(entry.message.toolCallId))
|
||
.map((entry) => entry.message);
|
||
}
|
||
|
||
function containsBridgeFailure(text) {
|
||
return /page_ai_pi_lab_session_not_found|page_ai_pi_lab_bridge_token_invalid|mnote_pi_rust_service_bridge_unavailable|mnote_pi_bridge_session_id_missing/i.test(String(text || ""));
|
||
}
|
||
|
||
function sessionEvidenceReady(testCase, raw) {
|
||
const results = readTargetToolResults(testCase, raw);
|
||
const minimumResults = testCase.minimumResults || 1;
|
||
if (results.length < minimumResults || !raw.includes(testCase.marker)) return false;
|
||
if (testCase.kind === "mcp") {
|
||
return raw.includes(testCase.serverName) && raw.includes(testCase.expectedWireTool);
|
||
}
|
||
return raw.includes(`"name":"${testCase.toolName}"`);
|
||
}
|
||
|
||
async function waitForSessionEvidence(testCase, sessionDir) {
|
||
const deadline = Date.now() + Math.min(TIMEOUT, 30000);
|
||
let snapshot = readSessionJsonl(sessionDir);
|
||
while (Date.now() < deadline) {
|
||
if (sessionEvidenceReady(testCase, snapshot.raw)) return snapshot;
|
||
await new Promise((resolve) => setTimeout(resolve, 200));
|
||
snapshot = readSessionJsonl(sessionDir);
|
||
}
|
||
return snapshot;
|
||
}
|
||
|
||
async function runCase(page, result, testCase) {
|
||
await abortExistingSession(page);
|
||
const session = await startRealPi(page, testCase);
|
||
result.sessions[testCase.id] = {
|
||
sessionId: session.sessionId,
|
||
runtimePid: session.runtimePid,
|
||
piSessionDir: session.piSessionDir,
|
||
runtimePolicySnapshot: session.runtimePolicySnapshot,
|
||
};
|
||
try {
|
||
await openPiUi(page);
|
||
const answer = await sendPrompt(page, testCase);
|
||
const sessionJsonl = await waitForSessionEvidence(testCase, session.piSessionDir);
|
||
const allToolResults = readToolResults(sessionJsonl.raw, testCase.toolName);
|
||
const toolResults = readTargetToolResults(testCase, sessionJsonl.raw);
|
||
const toolText = (await page.locator(
|
||
"[data-page-ai-pi-lab-tool-timeline], [data-page-ai-pi-tool-timeline]",
|
||
).allTextContents()).join("\n");
|
||
const screenshot = path.join(OUT, `${String(Object.keys(result.sessions).length).padStart(2, "0")}-${testCase.id}.png`);
|
||
await page.screenshot({ path: screenshot, fullPage: false });
|
||
|
||
const check = {
|
||
answer,
|
||
screenshot,
|
||
sessionFile: sessionJsonl.sessionFile,
|
||
toolResultCount: toolResults.length,
|
||
allToolResultCount: allToolResults.length,
|
||
failedToolResults: toolResults.filter((message) => message.isError === true).length,
|
||
allFailedToolResults: allToolResults.filter((message) => message.isError === true).length,
|
||
toolCardVisible: new RegExp(
|
||
testCase.kind === "mcp"
|
||
? `mcp:${testCase.serverName}|${testCase.serverName}|${testCase.expectedWireTool}`
|
||
: testCase.toolName,
|
||
"i",
|
||
).test(toolText),
|
||
toolCallRecorded: sessionJsonl.raw.includes(`"name":"${testCase.toolName}"`),
|
||
expectedMcpToolRecorded: testCase.kind !== "mcp"
|
||
|| (sessionJsonl.raw.includes(testCase.serverName)
|
||
&& sessionJsonl.raw.includes(testCase.expectedWireTool)),
|
||
noReasoningLeak: !/reasoning_content|"thinking"/i.test(toolText),
|
||
noEmptyReplyError: !/Pi runtime 返回了空回复|empty response/i.test(`${answer}\n${toolText}`),
|
||
noBridgeSessionFailure: !containsBridgeFailure(`${answer}\n${toolText}\n${sessionJsonl.raw}`),
|
||
};
|
||
result.checks[testCase.id] = check;
|
||
result.screenshots[testCase.id] = screenshot;
|
||
result.sessions[testCase.id].sessionFile = sessionJsonl.sessionFile;
|
||
|
||
assert(answer.includes(testCase.marker), `${testCase.id}: 最终回复缺少 marker`);
|
||
assert(check.toolCardVisible, `${testCase.id}: UI 未显示对应工具卡`);
|
||
assert(check.toolCallRecorded, `${testCase.id}: session JSONL 未记录工具调用`);
|
||
assert(check.expectedMcpToolRecorded, `${testCase.id}: session JSONL 未记录目标 MCP 工具`);
|
||
assert(
|
||
toolResults.length >= (testCase.minimumResults || 1),
|
||
`${testCase.id}: toolResult 数量不足，实际=${toolResults.length}`,
|
||
);
|
||
assert.equal(check.failedToolResults, 0, `${testCase.id}: 存在 isError=true 的 toolResult`);
|
||
assert(check.noReasoningLeak, `${testCase.id}: 工具时间线泄漏 thinking/reasoning`);
|
||
assert(check.noEmptyReplyError, `${testCase.id}: 仍出现 Pi runtime 空回复错误`);
|
||
assert(check.noBridgeSessionFailure, `${testCase.id}: 工具结果仍包含 Pi bridge/session 错误`);
|
||
if (testCase.id === "subagent") {
|
||
assert(
|
||
answer.includes(`CHILD_SUBAGENT_OK_${STAMP}`),
|
||
"subagent: 最终回复未包含子 agent 的真实返回",
|
||
);
|
||
}
|
||
} finally {
|
||
await requestJson(page, "/api/page-ai/pi/abort", {
|
||
method: "POST",
|
||
data: { sessionId: session.sessionId },
|
||
}).catch(() => ({}));
|
||
}
|
||
}
|
||
|
||
async function main() {
|
||
mkdirp(OUT);
|
||
const browser = await chromium.launch({
|
||
headless: process.env.MNOTE_PI_EXTENSION_MCP_MATRIX_HEADED === "1" ? false : true,
|
||
executablePath: CHROMIUM_EXECUTABLE || undefined,
|
||
});
|
||
const context = await browser.newContext({ viewport: { width: 1440, height: 980 } });
|
||
const page = await context.newPage();
|
||
const consoleMessages = [];
|
||
page.on("console", (message) => {
|
||
if (["error", "warning"].includes(message.type())) {
|
||
consoleMessages.push(`${message.type()}: ${message.text()}`);
|
||
}
|
||
});
|
||
page.on("pageerror", (error) => consoleMessages.push(`pageerror: ${error.message}`));
|
||
|
||
const result = {
|
||
base: BASE,
|
||
outputDir: OUT,
|
||
workspaceId: WORKSPACE_ID,
|
||
rootUri: ROOT_URI,
|
||
model: `${MODEL_PROVIDER}/${MODEL_ID}`,
|
||
sessions: {},
|
||
checks: {},
|
||
screenshots: {},
|
||
consoleMessages,
|
||
};
|
||
|
||
try {
|
||
await quickLogin(page);
|
||
result.seededEffective = await seedWorkspace(browser, page);
|
||
result.checks.policy = {
|
||
mcpServers: ["CodeGraph", "SearXNG", "Chrome Bridge"].every(
|
||
(name) => (result.seededEffective.mcpServers || []).some(
|
||
(server) => server.name === name && server.enabled !== false,
|
||
),
|
||
),
|
||
piExtensions: ["Pi Rust Official Todo", "Pi Rust Official Subagent"].every(
|
||
(name) => (result.seededEffective.piExtensions || []).some(
|
||
(extension) => extension.name === name && extension.enabled !== false,
|
||
),
|
||
),
|
||
};
|
||
assert(result.checks.policy.mcpServers, "有效策略未启用全部目标 MCP");
|
||
assert(result.checks.policy.piExtensions, "有效策略未启用 Todo/Subagent 扩展");
|
||
|
||
assert(ACTIVE_CASES.length > 0, "未匹配到需要执行的 matrix case");
|
||
for (const testCase of ACTIVE_CASES) {
|
||
await runCase(page, result, testCase);
|
||
}
|
||
result.ok = true;
|
||
} catch (error) {
|
||
result.ok = false;
|
||
result.error = error && error.stack ? error.stack : String(error);
|
||
await abortExistingSession(page).catch(() => null);
|
||
try {
|
||
const screenshot = path.join(OUT, "99-failure.png");
|
||
await page.screenshot({ path: screenshot, fullPage: true });
|
||
result.screenshots.failure = screenshot;
|
||
} catch {}
|
||
process.exitCode = 1;
|
||
} finally {
|
||
fs.writeFileSync(path.join(OUT, "result.json"), JSON.stringify(result, null, 2), "utf8");
|
||
await browser.close();
|
||
console.log(JSON.stringify({
|
||
ok: result.ok,
|
||
outputDir: OUT,
|
||
result: path.join(OUT, "result.json"),
|
||
}, null, 2));
|
||
}
|
||
}
|
||
|
||
main();
|