fix: restore pi rust builtin tool surface
- expose Pi Rust built-in tools by permission mode instead of replacing them with MNote file tools - update Pi Lab smoke coverage for native ls/read/bash usage - document the overreplacement regression and verification evidence
This commit is contained in:
Agent Board
@@ -6,19 +6,25 @@ const fs = require("node:fs");
|
||||
const os = require("node:os");
|
||||
const path = require("node:path");
|
||||
const { chromium } = require("playwright");
|
||||
const {
|
||||
setupWorkspaceAccess,
|
||||
seedAiPolicy,
|
||||
} = require("./lib/control-plane-dev-seed");
|
||||
|
||||
const BASE = process.env.MNOTE_UI_BASE_URL || "http://127.0.0.1:3000";
|
||||
const STAMP = Date.now();
|
||||
const OUT = process.env.MNOTE_PI_FULL_ACCESS_BUILTIN_OUT || path.join(os.tmpdir(), `mnote-pi-full-access-builtin-disabled-${STAMP}`);
|
||||
const OUT = process.env.MNOTE_PI_FULL_ACCESS_BUILTIN_OUT || path.join(os.tmpdir(), `mnote-pi-full-access-builtin-tools-${STAMP}`);
|
||||
const TIMEOUT = Number.parseInt(process.env.UI_TIMEOUT_MS || "360000", 10);
|
||||
const ACTOR_ID = process.env.MNOTE_E2E_ACTOR_ID || "mnote-e2e";
|
||||
const WORKSPACE_ID = process.env.MNOTE_PI_FULL_ACCESS_BUILTIN_WORKSPACE_ID || "local-ws:mnote-e2e:my-space";
|
||||
const ROOT_PATH = process.env.MNOTE_PI_FULL_ACCESS_BUILTIN_ROOT_PATH || "/mnt/Data1T/Mnote_data/users/mnote-e2e/workspaces/my-space";
|
||||
const WORKSPACE_ID = process.env.MNOTE_PI_FULL_ACCESS_BUILTIN_WORKSPACE_ID || `local-ws:${ACTOR_ID}:pi-full-access-builtins-${STAMP}`;
|
||||
const ROOT_PATH = process.env.MNOTE_PI_FULL_ACCESS_BUILTIN_ROOT_PATH || path.join(OUT, "workspace");
|
||||
const ROOT_URI = process.env.MNOTE_PI_FULL_ACCESS_BUILTIN_ROOT_URI || `file://${ROOT_PATH}`;
|
||||
const MODEL_PROVIDER = process.env.MNOTE_PI_FULL_ACCESS_BUILTIN_MODEL_PROVIDER || "omniroute";
|
||||
const MODEL_ID = process.env.MNOTE_PI_FULL_ACCESS_BUILTIN_MODEL_ID || "freefirst";
|
||||
const MARKER = `PI_FULL_ACCESS_BUILTIN_DISABLED_OK_${STAMP}`;
|
||||
const PAGE_PATH = `pi-full-access-builtin-disabled-${STAMP}.md`;
|
||||
const MODEL_ID = process.env.MNOTE_PI_FULL_ACCESS_BUILTIN_MODEL_ID || "gpt-5.4-mini";
|
||||
const MARKER = `PI_FULL_ACCESS_CONTROLLED_TOOLS_OK_${STAMP}`;
|
||||
const PAGE_PATH = `pi-full-access-builtin-tools-${STAMP}.md`;
|
||||
const SCRATCH_PATH = `pi-full-access-builtin-tools-${STAMP}.txt`;
|
||||
const BUILTIN_TOOLS = ["read", "write", "edit", "bash", "grep", "find", "ls", "hashline_edit"];
|
||||
const CHROMIUM_EXECUTABLE = process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE
|
||||
|| (fs.existsSync("/usr/bin/chromium-browser") ? "/usr/bin/chromium-browser" : "")
|
||||
|| (fs.existsSync("/usr/bin/chromium") ? "/usr/bin/chromium" : "")
|
||||
@@ -63,7 +69,7 @@ function piExtensionConfig(name, description, source, toolNames, riskLevel, requ
|
||||
return { name, description, source, toolNames, riskLevel, requiredScopes, enabled: true };
|
||||
}
|
||||
|
||||
function policyForFullAccessBuiltinDisabled() {
|
||||
function policyForFullAccessBuiltinTools() {
|
||||
return {
|
||||
defaultModel: `${MODEL_PROVIDER}/${MODEL_ID}`,
|
||||
allowedModels: [`${MODEL_PROVIDER}/${MODEL_ID}`],
|
||||
@@ -84,39 +90,34 @@ function policyForFullAccessBuiltinDisabled() {
|
||||
|
||||
async function seedWorkspace(page) {
|
||||
mkdirp(ROOT_PATH);
|
||||
fs.writeFileSync(path.join(ROOT_PATH, PAGE_PATH), "# Pi full access builtin disabled smoke\n", "utf8");
|
||||
const grantResponse = await page.request.fetch(`${BASE}/api/admin/access-policy/grants`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
accept: "application/json",
|
||||
"content-type": "application/json",
|
||||
},
|
||||
data: {
|
||||
userId: ACTOR_ID,
|
||||
rootUri: ROOT_URI,
|
||||
rootPath: ROOT_PATH,
|
||||
permission: "write",
|
||||
recursive: true,
|
||||
capabilities: ["ai"],
|
||||
},
|
||||
timeout: TIMEOUT,
|
||||
fs.writeFileSync(
|
||||
path.join(ROOT_PATH, PAGE_PATH),
|
||||
"# Pi full access builtin tools smoke\n\nBUILTIN_READ_MARKER\n",
|
||||
"utf8",
|
||||
);
|
||||
fs.rmSync(path.join(ROOT_PATH, SCRATCH_PATH), { force: true });
|
||||
await setupWorkspaceAccess(page.request, BASE, {
|
||||
actorId: ACTOR_ID,
|
||||
email: "mnote.e2e@example.com",
|
||||
username: ACTOR_ID,
|
||||
displayName: ACTOR_ID,
|
||||
role: "admin",
|
||||
workspaceId: WORKSPACE_ID,
|
||||
workspaceName: "Pi full access builtin tools smoke",
|
||||
rootPath: ROOT_PATH,
|
||||
rootUri: ROOT_URI,
|
||||
permission: "write",
|
||||
capabilities: ["ai", "read", "write"],
|
||||
timeoutMs: TIMEOUT,
|
||||
});
|
||||
const grantText = await grantResponse.text();
|
||||
let grantBody = {};
|
||||
try {
|
||||
grantBody = grantText ? JSON.parse(grantText) : {};
|
||||
} catch {
|
||||
grantBody = { raw: grantText };
|
||||
}
|
||||
if (!grantResponse.ok() && grantBody.code !== "local_access_policy_grant_duplicate") {
|
||||
throw new Error(`POST /api/admin/access-policy/grants failed: ${grantResponse.status()} ${grantText.slice(0, 800)}`);
|
||||
}
|
||||
await requestJson(page, "/api/ai-admin/settings", {
|
||||
method: "PUT",
|
||||
data: {
|
||||
...policyForFullAccessBuiltinDisabled(),
|
||||
quota: { daily: 200 },
|
||||
},
|
||||
await seedAiPolicy(page.request, BASE, {
|
||||
id: `pi-full-access-builtins-${ACTOR_ID}-${WORKSPACE_ID}`,
|
||||
userId: ACTOR_ID,
|
||||
workspaceId: WORKSPACE_ID,
|
||||
allowedRootsJson: [{ rootUri: ROOT_URI, rootPath: ROOT_PATH, permission: "write" }],
|
||||
modelPolicyJson: policyForFullAccessBuiltinTools(),
|
||||
quotaJson: { daily: 200 },
|
||||
timeoutMs: TIMEOUT,
|
||||
});
|
||||
await requestJson(page, `/api/ai-settings/effective?workspaceId=${encodeURIComponent(WORKSPACE_ID)}`);
|
||||
}
|
||||
@@ -130,7 +131,7 @@ async function abortExistingSession(page) {
|
||||
}
|
||||
|
||||
async function startRealPi(page) {
|
||||
const sessionId = `pi-full-access-builtin-disabled-${STAMP}`;
|
||||
const sessionId = `pi-full-access-builtin-tools-${STAMP}`;
|
||||
const start = await requestJson(page, "/api/page-ai/pi/start", {
|
||||
method: "POST",
|
||||
data: {
|
||||
@@ -138,7 +139,7 @@ async function startRealPi(page) {
|
||||
rootUri: ROOT_URI,
|
||||
workspaceId: WORKSPACE_ID,
|
||||
pagePath: PAGE_PATH,
|
||||
pageTitle: "Pi full access builtin disabled smoke",
|
||||
pageTitle: "Pi full access builtin tools smoke",
|
||||
modelProvider: MODEL_PROVIDER,
|
||||
modelId: MODEL_ID,
|
||||
thinkingLevel: "medium",
|
||||
@@ -150,6 +151,7 @@ async function startRealPi(page) {
|
||||
assert.equal(start.session.runtimePolicySnapshot.permissionMode, "full_access", "runtime policy should persist full_access");
|
||||
assert.equal(start.session.runtimeMode, "rpc", `Pi 必须以 rpc 模式启动，实际=${start.session.runtimeMode}`);
|
||||
assert(start.session.runtimePid, "真实 Pi RPC 启动后应有 runtimePid");
|
||||
start.session.managedPiBuiltinTools = start.managedPiBuiltinTools || [];
|
||||
return start.session;
|
||||
}
|
||||
|
||||
@@ -181,6 +183,41 @@ function readSessionJsonl(sessionDir) {
|
||||
return { sessionFile, raw: fs.readFileSync(sessionFile, "utf8") };
|
||||
}
|
||||
|
||||
function readBuiltinToolResults(raw) {
|
||||
return raw
|
||||
.split("\n")
|
||||
.filter(Boolean)
|
||||
.map((line) => {
|
||||
try {
|
||||
return JSON.parse(line);
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
})
|
||||
.filter((entry) => entry?.type === "message"
|
||||
&& entry?.message?.role === "toolResult"
|
||||
&& BUILTIN_TOOLS.includes(entry.message.toolName))
|
||||
.map((entry) => entry.message);
|
||||
}
|
||||
|
||||
async function waitForCompleteSessionJsonl(sessionDir) {
|
||||
const deadline = Date.now() + Math.min(TIMEOUT, 15000);
|
||||
let snapshot = readSessionJsonl(sessionDir);
|
||||
while (Date.now() < deadline) {
|
||||
const calledTools = BUILTIN_TOOLS.filter((tool) => snapshot.raw.includes(`"name":"${tool}"`));
|
||||
if (calledTools.includes("ls")
|
||||
&& calledTools.includes("read")
|
||||
&& calledTools.includes("bash")
|
||||
&& !snapshot.raw.includes('"name":"mnote_local_file_read"')
|
||||
&& snapshot.raw.includes(MARKER)) {
|
||||
return snapshot;
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 100));
|
||||
snapshot = readSessionJsonl(sessionDir);
|
||||
}
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
mkdirp(OUT);
|
||||
const browser = await chromium.launch({
|
||||
@@ -223,20 +260,26 @@ async function main() {
|
||||
result.checks.noLegacyNpmAskUser = !enabledSources.includes("npm:pi-ask-user");
|
||||
result.checks.noExternalPermissionSystem = !enabledSources.includes("npm:@gotgenes/pi-permission-system");
|
||||
result.checks.officialPermissionGateConfigured = enabledSources.includes("pi-rust-official:permission-gate");
|
||||
result.checks.managedBuiltinToolsDisabledAtStart = (session.runtimePolicySnapshot.managedBuiltinTools || []).length === 0;
|
||||
result.checks.managedBuiltinToolsAtStart = session.managedPiBuiltinTools || [];
|
||||
assert.equal(result.checks.permissionSystemConfigAbsent, true, "full_access smoke should not generate legacy pi-permission-system config");
|
||||
assert.equal(result.checks.noLegacyNpmAskUser, true, "full_access smoke should not use unavailable npm:pi-ask-user");
|
||||
assert.equal(result.checks.noExternalPermissionSystem, true, "full_access smoke should not load incompatible pi-permission-system");
|
||||
assert.equal(result.checks.officialPermissionGateConfigured, true, "policy should include Pi Rust official permission-gate");
|
||||
assert.deepEqual(
|
||||
[...result.checks.managedBuiltinToolsAtStart].sort(),
|
||||
[...BUILTIN_TOOLS].sort(),
|
||||
"full_access should expose Pi Rust official built-in tools instead of replacing them with MNote file tools",
|
||||
);
|
||||
|
||||
await openPiUi(page);
|
||||
await page.screenshot({ path: path.join(OUT, "01-full-access-started.png"), fullPage: false });
|
||||
result.screenshots.started = path.join(OUT, "01-full-access-started.png");
|
||||
|
||||
const prompt = [
|
||||
"请调用 mnote_allowed_roots_describe 工具，读取 MNote 返回的 allowedRoots、deniedPiBuiltinTools、managedPiBuiltinTools、permissionProvider。",
|
||||
"不要调用 bash/read/write/edit/hashline_edit/grep/find/ls 这些 Pi 内置工具。",
|
||||
"用一句话说明：full_access 下 MNote 当前仍默认禁用 Pi Rust 内置文件/命令工具，文件权限由 MNote bridge 管控。",
|
||||
"当前是 full_access 验收。必须真实调用 Pi Rust 内置工具，不能只描述。",
|
||||
"先调用 ls 列出当前工作目录，再调用 read 读取当前页文件。",
|
||||
"然后调用 bash 执行 pwd。确认当前页内容包含 BUILTIN_READ_MARKER。",
|
||||
"不要调用 mnote_local_file_read 或 mnote_local_file_patch；MNote 工具只用于上下文和知识库，不应替代 Pi 原生文件工具。",
|
||||
`最终单独输出一行：${MARKER}`,
|
||||
].join("\n");
|
||||
await page.locator("[data-page-ai-pi-lab-input]").fill(prompt);
|
||||
@@ -254,21 +297,36 @@ async function main() {
|
||||
await page.waitForTimeout(500);
|
||||
}
|
||||
await markerLocator.waitFor({ state: "visible", timeout: 1000 });
|
||||
await page.screenshot({ path: path.join(OUT, "02-full-access-builtin-disabled-answer.png"), fullPage: false });
|
||||
result.screenshots.answer = path.join(OUT, "02-full-access-builtin-disabled-answer.png");
|
||||
await page.screenshot({ path: path.join(OUT, "02-full-access-builtin-tools-answer.png"), fullPage: false });
|
||||
result.screenshots.answer = path.join(OUT, "02-full-access-builtin-tools-answer.png");
|
||||
|
||||
const sessionJsonl = readSessionJsonl(session.piSessionDir);
|
||||
const sessionJsonl = await waitForCompleteSessionJsonl(session.piSessionDir);
|
||||
result.session.sessionFile = sessionJsonl.sessionFile;
|
||||
result.answerText = ((await markerLocator.textContent({ timeout: TIMEOUT })) || "").trim();
|
||||
result.checks.allowedRootsToolCalled = /mnote_allowed_roots_describe/.test(sessionJsonl.raw);
|
||||
result.checks.deniedBuiltinsRecorded = /deniedPiBuiltinTools/.test(sessionJsonl.raw) && /hashline_edit/.test(sessionJsonl.raw);
|
||||
result.checks.managedBuiltinsEmptyRecorded = /managedPiBuiltinTools/.test(sessionJsonl.raw);
|
||||
result.checks.noRawBuiltinCalled = !/"name":"(bash|read|write|edit|hashline_edit|grep|find|ls)"/.test(sessionJsonl.raw);
|
||||
result.checks.calledBuiltinTools = BUILTIN_TOOLS.filter((tool) => sessionJsonl.raw.includes(`"name":"${tool}"`));
|
||||
result.checks.calledMnoteLocalFileRead = sessionJsonl.raw.includes('"name":"mnote_local_file_read"');
|
||||
result.checks.calledMnoteLocalFilePatch = sessionJsonl.raw.includes('"name":"mnote_local_file_patch"');
|
||||
const builtinToolResults = readBuiltinToolResults(sessionJsonl.raw);
|
||||
result.checks.builtinToolResultCount = builtinToolResults.length;
|
||||
result.checks.failedBuiltinTools = builtinToolResults
|
||||
.filter((message) => message.isError === true)
|
||||
.map((message) => message.toolName);
|
||||
result.checks.readContainsMarker = sessionJsonl.raw.includes("BUILTIN_READ_MARKER");
|
||||
result.checks.lsSawPage = sessionJsonl.raw.includes(PAGE_PATH);
|
||||
result.checks.noBridgeSessionFailure = !/page_ai_pi_lab_session_not_found|page_ai_pi_lab_bridge_token_invalid|mnote_pi_rust_service_bridge_unavailable|mnote_pi_bridge_session_id_missing/i.test(sessionJsonl.raw);
|
||||
result.checks.scratchContent = fs.existsSync(path.join(ROOT_PATH, SCRATCH_PATH))
|
||||
? fs.readFileSync(path.join(ROOT_PATH, SCRATCH_PATH), "utf8")
|
||||
: "";
|
||||
result.checks.noPermissionRequiredPrompt = await page.locator("text=Permission Required").count() === 0;
|
||||
assert(result.checks.allowedRootsToolCalled, "Pi session JSONL should record mnote_allowed_roots_describe call");
|
||||
assert(result.checks.deniedBuiltinsRecorded, "Pi session JSONL should include deniedPiBuiltinTools");
|
||||
assert(result.checks.managedBuiltinsEmptyRecorded, "Pi session JSONL should include managedPiBuiltinTools");
|
||||
assert(result.checks.noRawBuiltinCalled, "Pi raw builtin tools should remain disabled by default");
|
||||
assert(result.checks.calledBuiltinTools.includes("ls"), "full_access should allow Pi Rust builtin ls");
|
||||
assert(result.checks.calledBuiltinTools.includes("read"), "full_access should allow Pi Rust builtin read");
|
||||
assert(result.checks.calledBuiltinTools.includes("bash"), "full_access should allow Pi Rust builtin bash");
|
||||
assert.equal(result.checks.calledMnoteLocalFileRead, false, "Pi Rust builtin read must not be replaced by mnote_local_file_read");
|
||||
assert.equal(result.checks.calledMnoteLocalFilePatch, false, "This smoke must not use mnote_local_file_patch");
|
||||
assert.equal(result.checks.readContainsMarker, true, "Pi Rust builtin read result should contain the seeded page marker");
|
||||
assert.equal(result.checks.lsSawPage, true, "Pi Rust builtin ls should list the seeded page file");
|
||||
assert.equal(result.checks.noBridgeSessionFailure, true, "MNote bridge context must not fail while Pi builtins are available");
|
||||
assert.equal(result.checks.scratchContent, "", "negative full_access smoke should not create scratch files via raw builtins");
|
||||
assert(result.checks.noPermissionRequiredPrompt, "official permission-gate smoke should not show legacy Permission Required prompt");
|
||||
|
||||
await requestJson(page, "/api/page-ai/pi/abort", { method: "POST", data: { sessionId: session.sessionId } }).catch(() => ({}));
|
||||
|
||||
Reference in New Issue
Block a user