mnote/scripts/task-pi-lab-real-extension-mcp-matrix-smoke.js

#!/usr/bin/env node
"use strict";

const assert = require("node:assert");
const fs = require("node:fs");
const os = require("node:os");
const path = require("node:path");
const { chromium } = require("playwright");

const BASE = process.env.MNOTE_UI_BASE_URL || "http://127.0.0.1:3000";
const STAMP = Date.now();
const OUT = process.env.MNOTE_PI_EXTENSION_MCP_MATRIX_OUT
  || path.join(os.tmpdir(), `mnote-pi-extension-mcp-matrix-${STAMP}`);
const TIMEOUT = Number.parseInt(process.env.UI_TIMEOUT_MS || "420000", 10);
const ACTOR_ID = process.env.MNOTE_E2E_ACTOR_ID || "mnote-e2e";
const WORKSPACE_ID = process.env.MNOTE_PI_EXTENSION_MCP_MATRIX_WORKSPACE_ID
  || `local-ws:${ACTOR_ID}:my-space`;
const ROOT_PATH = process.env.MNOTE_PI_EXTENSION_MCP_MATRIX_ROOT_PATH
  || `/mnt/Data1T/Mnote_data/users/${ACTOR_ID}/workspaces/my-space`;
const ROOT_URI = process.env.MNOTE_PI_EXTENSION_MCP_MATRIX_ROOT_URI || `file://${ROOT_PATH}`;
const MODEL_PROVIDER = process.env.MNOTE_PI_EXTENSION_MCP_MATRIX_MODEL_PROVIDER || "omniroute";
const MODEL_ID = process.env.MNOTE_PI_EXTENSION_MCP_MATRIX_MODEL_ID || "gpt-5.4-mini";
const CHROMIUM_EXECUTABLE = process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE
  || (fs.existsSync("/usr/bin/chromium-browser") ? "/usr/bin/chromium-browser" : "")
  || (fs.existsSync("/usr/bin/chromium") ? "/usr/bin/chromium" : "")
  || (fs.existsSync("/usr/bin/google-chrome") ? "/usr/bin/google-chrome" : "");

const CASES = [
  {
    id: "codegraph",
    kind: "mcp",
    toolName: "mcp",
    serverName: "codegraph",
    expectedWireTool: "codegraph_search",
    marker: `PI_MATRIX_CODEGRAPH_OK_${STAMP}`,
    prompt: [
      "必须真实调用 MCP 工具，不能只描述。",
      "先调用 mcp 的 list 模式查看 codegraph 服务工具，再调用 codegraph_search。",
      "搜索词使用 PiLabSession，项目根使用 /mnt/Data1T/mnote，最多返回 3 条。",
      "简要写出一个实际命中的文件路径。",
    ],
  },
  {
    id: "searxng",
    kind: "mcp",
    toolName: "mcp",
    serverName: "searxng",
    expectedWireTool: "searxng_search",
    marker: `PI_MATRIX_SEARXNG_OK_${STAMP}`,
    prompt: [
      "必须真实调用 MCP 工具，不能只描述。",
      "先调用 mcp 的 list 模式查看 searxng 服务工具，再调用 searxng_search。",
      "搜索 Rust programming language official website，最多返回 3 条。",
      "简要写出一个实际返回的标题或网址域名。",
    ],
  },
  {
    id: "chrome-bridge",
    kind: "mcp",
    toolName: "mcp",
    serverName: "chrome-bridge",
    expectedWireTool: "chrome_bridge_session_summary",
    marker: `PI_MATRIX_CHROME_BRIDGE_OK_${STAMP}`,
    prompt: [
      "必须真实调用 MCP 工具，不能只描述。",
      "先调用 mcp 的 list 模式查看 chrome-bridge 服务工具，再调用 chrome_bridge_session_summary，参数为空对象。",
      "简要报告工具实际返回的 bridge 或 extension 状态。",
    ],
  },
  {
    id: "todo",
    kind: "extension",
    toolName: "todo",
    minimumResults: 4,
    marker: `PI_MATRIX_TODO_OK_${STAMP}`,
    prompt: [
      "必须真实调用 Pi 官方 todo 工具，不能只描述。",
      `先 add 一条文本为 TODO_MATRIX_${STAMP} 的任务，再 list，再 toggle id=1，最后再次 list。`,
      "简要确认任务已完成。",
    ],
  },
  {
    id: "subagent",
    kind: "extension",
    toolName: "subagent",
    minimumResults: 1,
    marker: `PI_MATRIX_SUBAGENT_OK_${STAMP}`,
    prompt: [
      "必须真实调用 Pi 官方 subagent 工具，不能只描述。",
      `使用 single 模式：agent=worker，agentScope=user，cwd=${ROOT_PATH}。`,
      `委派任务为：不要调用任何工具，只回复 CHILD_SUBAGENT_OK_${STAMP}。`,
      `最终回答必须包含 CHILD_SUBAGENT_OK_${STAMP}。`,
    ],
  },
];
const CASE_FILTER = new Set(
  String(process.env.MNOTE_PI_EXTENSION_MCP_MATRIX_CASES || "")
    .split(",")
    .map((value) => value.trim())
    .filter(Boolean),
);
const ACTIVE_CASES = CASE_FILTER.size > 0
  ? CASES.filter((testCase) => CASE_FILTER.has(testCase.id))
  : CASES;

function mkdirp(dir) {
  fs.mkdirSync(dir, { recursive: true });
}

async function quickLogin(page) {
  await page.goto(`${BASE}/auth`, { waitUntil: "commit", timeout: TIMEOUT });
  const quickLoginButton = page.getByRole("button", { name: "测试账号快速登录" });
  await quickLoginButton.waitFor({ state: "visible", timeout: TIMEOUT });
  await Promise.all([
    page.waitForURL((url) => !url.toString().includes("/auth"), { timeout: TIMEOUT }).catch(() => null),
    quickLoginButton.click(),
  ]);
}

async function requestJson(page, url, options = {}) {
  const response = await page.request.fetch(`${BASE}${url}`, {
    ...options,
    headers: {
      accept: "application/json",
      "content-type": "application/json",
      ...(options.headers || {}),
    },
    timeout: options.timeout || TIMEOUT,
  });
  const text = await response.text();
  let body = {};
  try {
    body = text ? JSON.parse(text) : {};
  } catch {
    body = { raw: text };
  }
  assert(response.ok(), `${options.method || "GET"} ${url} failed: ${response.status()} ${text.slice(0, 1000)}`);
  return body;
}

function skillConfig(name, description, source, riskLevel, requiredScopes = []) {
  return { name, description, source, riskLevel, requiredScopes, enabled: true };
}

function mcpConfig(name, description, transport, command, url, networkPolicy, secretRefs, riskLevel, requiredScopes = []) {
  return {
    name,
    description,
    transport,
    command,
    url,
    networkPolicy,
    secretRefs,
    riskLevel,
    requiredScopes,
    enabled: true,
    facadeOnly: true,
    sandbox: true,
  };
}

function piExtensionConfig(name, description, source, toolNames, riskLevel, requiredScopes = []) {
  return { name, description, source, toolNames, riskLevel, requiredScopes, enabled: true };
}

function policyForMatrix() {
  return {
    defaultModel: `${MODEL_PROVIDER}/${MODEL_ID}`,
    allowedModels: [`${MODEL_PROVIDER}/${MODEL_ID}`],
    tools: {
      "mnote.current_page.read": "allow",
      "mnote.allowed_roots.describe": "allow",
      "mnote.tool_receipt.write": "allow",
    },
    skills: {
      codegraph: skillConfig(
        "CodeGraph",
        "读取项目代码图、符号和调用关系。",
        "mcp://codegraph",
        "medium",
        ["workspace:code-read"],
      ),
      searxng: skillConfig(
        "SearXNG Search",
        "通过本地 SearXNG MCP 做网页检索。",
        "mcp://searxng",
        "medium",
        ["network:search"],
      ),
      "chrome-bridge": skillConfig(
        "Chrome Bridge",
        "通过受控浏览器桥接执行页面验证。",
        "mcp://chrome-bridge",
        "high",
        ["browser:automation", "qa:browser"],
      ),
    },
    mcpServers: {
      codegraph: mcpConfig(
        "CodeGraph",
        "代码图 MCP。",
        "stdio",
        "codegraph serve --mcp",
        "",
        "deny-all",
        [],
        "medium",
        ["workspace:code-read"],
      ),
      searxng: mcpConfig(
        "SearXNG",
        "本地 SearXNG 检索 MCP。",
        "stdio",
        "node /home/lix/.agent-infra/searxng/codex-searxng-mcp.cjs",
        "",
        "allow-local",
        [],
        "medium",
        ["network:search"],
      ),
      "chrome-bridge": mcpConfig(
        "Chrome Bridge",
        "本机 Chromium/Chrome 桥接。",
        "stdio",
        "node /home/lix/apps/codex-chrome-bridge/mcp/chrome-bridge-mcp.mjs",
        "",
        "allow-local",
        [],
        "high",
        ["browser:automation", "qa:browser"],
      ),
    },
    piExtensions: {
      "pi-rust-official-todo": piExtensionConfig(
        "Pi Rust Official Todo",
        "Pi Rust 官方 todo 扩展。",
        "pi-rust-official:todo",
        ["todo"],
        "medium",
        ["workflow:todo"],
      ),
      "pi-rust-official-subagent": piExtensionConfig(
        "Pi Rust Official Subagent",
        "Pi Rust 官方 subagent 扩展。",
        "pi-rust-official:subagent",
        ["subagent"],
        "high",
        ["agent:delegate"],
      ),
      "pi-rust-official-permission-gate": piExtensionConfig(
        "Pi Rust Official Permission Gate",
        "Pi Rust 官方 permission-gate 扩展。",
        "pi-rust-official:permission-gate",
        [],
        "high",
        ["tool:policy"],
      ),
    },
  };
}

async function seedWorkspace(page) {
  mkdirp(ROOT_PATH);
  fs.writeFileSync(
    path.join(ROOT_PATH, "pi-extension-mcp-matrix.md"),
    "# Pi extension and MCP matrix smoke\n",
    "utf8",
  );
  const grantResponse = await page.request.fetch(`${BASE}/api/admin/access-policy/grants`, {
    method: "POST",
    headers: {
      accept: "application/json",
      "content-type": "application/json",
    },
    data: {
      userId: ACTOR_ID,
      rootUri: ROOT_URI,
      rootPath: ROOT_PATH,
      permission: "write",
      recursive: true,
      capabilities: ["ai"],
    },
    timeout: TIMEOUT,
  });
  const grantText = await grantResponse.text();
  let grantBody = {};
  try {
    grantBody = grantText ? JSON.parse(grantText) : {};
  } catch {
    grantBody = { raw: grantText };
  }
  if (!grantResponse.ok() && grantBody.code !== "local_access_policy_grant_duplicate") {
    throw new Error(
      `POST /api/admin/access-policy/grants failed: ${grantResponse.status()} ${grantText.slice(0, 1000)}`,
    );
  }
  await requestJson(page, "/api/ai-admin/settings", {
    method: "PUT",
    data: {
      ...policyForMatrix(),
      quota: { daily: 200 },
    },
  });
  return requestJson(page, `/api/ai-settings/effective?workspaceId=${encodeURIComponent(WORKSPACE_ID)}`);
}

async function abortExistingSession(page) {
  const status = await requestJson(page, "/api/page-ai/pi/status").catch(() => null);
  const sessionId = status?.session?.sessionId;
  if (!sessionId) return null;
  await requestJson(page, "/api/page-ai/pi/abort", {
    method: "POST",
    data: { sessionId },
  }).catch(() => null);
  return sessionId;
}

async function startRealPi(page, testCase) {
  const sessionId = `pi-extension-mcp-matrix-${testCase.id}-${STAMP}`;
  const pagePath = `pi-extension-mcp-matrix-${testCase.id}-${STAMP}.md`;
  fs.writeFileSync(path.join(ROOT_PATH, pagePath), `# ${testCase.id} smoke\n`, "utf8");
  const start = await requestJson(page, "/api/page-ai/pi/start", {
    method: "POST",
    data: {
      sessionId,
      rootUri: ROOT_URI,
      workspaceId: WORKSPACE_ID,
      pagePath,
      pageTitle: `${testCase.id} smoke`,
      modelProvider: MODEL_PROVIDER,
      modelId: MODEL_ID,
      thinkingLevel: "off",
      permissionMode: "full_access",
    },
  });
  assert.equal(start.ok, true, `${testCase.id}: Pi start ok should be true`);
  assert.equal(start.permissionMode, "full_access", `${testCase.id}: start should expose full_access`);
  assert.equal(
    start.session.runtimePolicySnapshot.permissionMode,
    "full_access",
    `${testCase.id}: runtime policy should persist full_access`,
  );
  assert.equal(start.session.runtimeMode, "rpc", `${testCase.id}: Pi 必须以 rpc 模式启动`);
  assert(start.session.runtimePid, `${testCase.id}: 真实 Pi RPC 启动后应有 runtimePid`);
  return start.session;
}

async function openPiUi(page) {
  await page.goto(`${BASE}/page-ai/pi`, { waitUntil: "commit", timeout: TIMEOUT });
  await page.locator('[data-page-ai-pi-lab="drawer"]').waitFor({ state: "visible", timeout: TIMEOUT });
  await page.waitForFunction(() => {
    const text = document.querySelector("[data-page-ai-pi-lab-status-text]")?.textContent || "";
    return /ready|running|streaming/.test(text);
  }, null, { timeout: TIMEOUT }).catch(() => null);
}

async function expandToolTimelines(page) {
  const timelines = await page.locator(
    "[data-page-ai-pi-lab-tool-timeline], [data-page-ai-pi-tool-timeline]",
  ).all();
  for (const timeline of timelines) {
    await timeline.evaluate((node) => {
      node.open = true;
    }).catch(() => {});
  }
}

async function sendPrompt(page, testCase) {
  const fullPrompt = [
    ...testCase.prompt,
    `最后必须单独输出一行：${testCase.marker}`,
  ].join("\n");
  await page.locator("[data-page-ai-pi-lab-input]").fill(fullPrompt);
  await page.locator("[data-page-ai-pi-lab-btn-send]").click();
  const markerLocator = page
    .locator(
      '[data-page-ai-pi-lab-message-role="assistant"] .wolai-page-ai-pi-lab-text, '
      + '[data-page-ai-pi-lab-message-role="assistant"] .wolai-page-ai-pi-lab-markdown',
    )
    .filter({ hasText: testCase.marker })
    .last();
  await markerLocator.waitFor({ state: "visible", timeout: TIMEOUT });
  await page.waitForFunction(() => {
    const status = document.querySelector("[data-page-ai-pi-lab-status-text]")?.textContent || "";
    return status === "ready";
  }, null, { timeout: 30000 }).catch(() => null);
  await expandToolTimelines(page);
  return ((await markerLocator.textContent({ timeout: TIMEOUT })) || "").trim();
}

function readSessionJsonl(sessionDir) {
  const files = [];
  const walk = (dir) => {
    for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
      const target = path.join(dir, entry.name);
      if (entry.isDirectory()) walk(target);
      else if (entry.isFile() && entry.name.endsWith(".jsonl")) files.push(target);
    }
  };
  walk(sessionDir);
  files.sort();
  const sessionFile = files[files.length - 1];
  assert(sessionFile, `session jsonl not found in ${sessionDir}`);
  return { sessionFile, raw: fs.readFileSync(sessionFile, "utf8") };
}

function parseSessionMessages(raw) {
  return String(raw || "")
    .split("\n")
    .filter(Boolean)
    .map((line) => {
      try {
        return JSON.parse(line);
      } catch {
        return undefined;
      }
    })
    .filter(Boolean);
}

function readToolResults(raw, toolName) {
  return parseSessionMessages(raw)
    .filter((entry) => entry?.type === "message"
      && entry?.message?.role === "toolResult"
      && entry?.message?.toolName === toolName)
    .map((entry) => entry.message);
}

function readTargetToolResults(testCase, raw) {
  const entries = parseSessionMessages(raw);
  const targetCallIds = new Set();
  for (const entry of entries) {
    if (entry?.type !== "message" || entry?.message?.role !== "assistant") continue;
    for (const item of entry.message.content || []) {
      if (item?.type !== "toolCall" || item?.name !== testCase.toolName) continue;
      if (testCase.kind === "mcp") {
        if (item.arguments?.server !== testCase.serverName) continue;
        if (item.arguments?.mode !== "call" || item.arguments?.tool !== testCase.expectedWireTool) continue;
      }
      targetCallIds.add(item.id);
    }
  }
  return entries
    .filter((entry) => entry?.type === "message"
      && entry?.message?.role === "toolResult"
      && entry?.message?.toolName === testCase.toolName
      && targetCallIds.has(entry.message.toolCallId))
    .map((entry) => entry.message);
}

function containsBridgeFailure(text) {
  return /page_ai_pi_lab_session_not_found|page_ai_pi_lab_bridge_token_invalid|mnote_pi_rust_service_bridge_unavailable|mnote_pi_bridge_session_id_missing/i.test(String(text || ""));
}

function sessionEvidenceReady(testCase, raw) {
  const results = readTargetToolResults(testCase, raw);
  const minimumResults = testCase.minimumResults || 1;
  if (results.length < minimumResults || !raw.includes(testCase.marker)) return false;
  if (testCase.kind === "mcp") {
    return raw.includes(testCase.serverName) && raw.includes(testCase.expectedWireTool);
  }
  return raw.includes(`"name":"${testCase.toolName}"`);
}

async function waitForSessionEvidence(testCase, sessionDir) {
  const deadline = Date.now() + Math.min(TIMEOUT, 30000);
  let snapshot = readSessionJsonl(sessionDir);
  while (Date.now() < deadline) {
    if (sessionEvidenceReady(testCase, snapshot.raw)) return snapshot;
    await new Promise((resolve) => setTimeout(resolve, 200));
    snapshot = readSessionJsonl(sessionDir);
  }
  return snapshot;
}

async function runCase(page, result, testCase) {
  await abortExistingSession(page);
  const session = await startRealPi(page, testCase);
  result.sessions[testCase.id] = {
    sessionId: session.sessionId,
    runtimePid: session.runtimePid,
    piSessionDir: session.piSessionDir,
    runtimePolicySnapshot: session.runtimePolicySnapshot,
  };
  try {
    await openPiUi(page);
    const answer = await sendPrompt(page, testCase);
    const sessionJsonl = await waitForSessionEvidence(testCase, session.piSessionDir);
    const allToolResults = readToolResults(sessionJsonl.raw, testCase.toolName);
    const toolResults = readTargetToolResults(testCase, sessionJsonl.raw);
    const toolText = (await page.locator(
      "[data-page-ai-pi-lab-tool-timeline], [data-page-ai-pi-tool-timeline]",
    ).allTextContents()).join("\n");
    const screenshot = path.join(OUT, `${String(Object.keys(result.sessions).length).padStart(2, "0")}-${testCase.id}.png`);
    await page.screenshot({ path: screenshot, fullPage: false });

    const check = {
      answer,
      screenshot,
      sessionFile: sessionJsonl.sessionFile,
      toolResultCount: toolResults.length,
      allToolResultCount: allToolResults.length,
      failedToolResults: toolResults.filter((message) => message.isError === true).length,
      allFailedToolResults: allToolResults.filter((message) => message.isError === true).length,
      toolCardVisible: new RegExp(
        testCase.kind === "mcp"
          ? `mcp:${testCase.serverName}|${testCase.serverName}|${testCase.expectedWireTool}`
          : testCase.toolName,
        "i",
      ).test(toolText),
      toolCallRecorded: sessionJsonl.raw.includes(`"name":"${testCase.toolName}"`),
      expectedMcpToolRecorded: testCase.kind !== "mcp"
        || (sessionJsonl.raw.includes(testCase.serverName)
          && sessionJsonl.raw.includes(testCase.expectedWireTool)),
      noReasoningLeak: !/reasoning_content|"thinking"/i.test(toolText),
      noEmptyReplyError: !/Pi runtime 返回了空回复|empty response/i.test(`${answer}\n${toolText}`),
      noBridgeSessionFailure: !containsBridgeFailure(`${answer}\n${toolText}\n${sessionJsonl.raw}`),
    };
    result.checks[testCase.id] = check;
    result.screenshots[testCase.id] = screenshot;
    result.sessions[testCase.id].sessionFile = sessionJsonl.sessionFile;

    assert(answer.includes(testCase.marker), `${testCase.id}: 最终回复缺少 marker`);
    assert(check.toolCardVisible, `${testCase.id}: UI 未显示对应工具卡`);
    assert(check.toolCallRecorded, `${testCase.id}: session JSONL 未记录工具调用`);
    assert(check.expectedMcpToolRecorded, `${testCase.id}: session JSONL 未记录目标 MCP 工具`);
    assert(
      toolResults.length >= (testCase.minimumResults || 1),
      `${testCase.id}: toolResult 数量不足，实际=${toolResults.length}`,
    );
    assert.equal(check.failedToolResults, 0, `${testCase.id}: 存在 isError=true 的 toolResult`);
    assert(check.noReasoningLeak, `${testCase.id}: 工具时间线泄漏 thinking/reasoning`);
    assert(check.noEmptyReplyError, `${testCase.id}: 仍出现 Pi runtime 空回复错误`);
    assert(check.noBridgeSessionFailure, `${testCase.id}: 工具结果仍包含 Pi bridge/session 错误`);
    if (testCase.id === "subagent") {
      assert(
        answer.includes(`CHILD_SUBAGENT_OK_${STAMP}`),
        "subagent: 最终回复未包含子 agent 的真实返回",
      );
    }
  } finally {
    await requestJson(page, "/api/page-ai/pi/abort", {
      method: "POST",
      data: { sessionId: session.sessionId },
    }).catch(() => ({}));
  }
}

async function main() {
  mkdirp(OUT);
  const browser = await chromium.launch({
    headless: process.env.MNOTE_PI_EXTENSION_MCP_MATRIX_HEADED === "1" ? false : true,
    executablePath: CHROMIUM_EXECUTABLE || undefined,
  });
  const context = await browser.newContext({ viewport: { width: 1440, height: 980 } });
  const page = await context.newPage();
  const consoleMessages = [];
  page.on("console", (message) => {
    if (["error", "warning"].includes(message.type())) {
      consoleMessages.push(`${message.type()}: ${message.text()}`);
    }
  });
  page.on("pageerror", (error) => consoleMessages.push(`pageerror: ${error.message}`));

  const result = {
    base: BASE,
    outputDir: OUT,
    workspaceId: WORKSPACE_ID,
    rootUri: ROOT_URI,
    model: `${MODEL_PROVIDER}/${MODEL_ID}`,
    sessions: {},
    checks: {},
    screenshots: {},
    consoleMessages,
  };

  try {
    await quickLogin(page);
    result.seededEffective = await seedWorkspace(page);
    result.checks.policy = {
      mcpServers: ["CodeGraph", "SearXNG", "Chrome Bridge"].every(
        (name) => (result.seededEffective.mcpServers || []).some(
          (server) => server.name === name && server.enabled !== false,
        ),
      ),
      piExtensions: ["Pi Rust Official Todo", "Pi Rust Official Subagent"].every(
        (name) => (result.seededEffective.piExtensions || []).some(
          (extension) => extension.name === name && extension.enabled !== false,
        ),
      ),
    };
    assert(result.checks.policy.mcpServers, "有效策略未启用全部目标 MCP");
    assert(result.checks.policy.piExtensions, "有效策略未启用 Todo/Subagent 扩展");

    assert(ACTIVE_CASES.length > 0, "未匹配到需要执行的 matrix case");
    for (const testCase of ACTIVE_CASES) {
      await runCase(page, result, testCase);
    }
    result.ok = true;
  } catch (error) {
    result.ok = false;
    result.error = error && error.stack ? error.stack : String(error);
    await abortExistingSession(page).catch(() => null);
    try {
      const screenshot = path.join(OUT, "99-failure.png");
      await page.screenshot({ path: screenshot, fullPage: true });
      result.screenshots.failure = screenshot;
    } catch {}
    process.exitCode = 1;
  } finally {
    fs.writeFileSync(path.join(OUT, "result.json"), JSON.stringify(result, null, 2), "utf8");
    await browser.close();
    console.log(JSON.stringify({
      ok: result.ok,
      outputDir: OUT,
      result: path.join(OUT, "result.json"),
    }, null, 2));
  }
}

main();