feat(rag): replace LiteParse flows with LightRAG provider
This commit is contained in:
lix-2026
@@ -0,0 +1,234 @@
|
||||
#!/usr/bin/env node
|
||||
"use strict";
|
||||
|
||||
const assert = require("node:assert");
|
||||
const fs = require("node:fs");
|
||||
const path = require("node:path");
|
||||
const JSZip = require("jszip");
|
||||
const { chromium } = require("playwright");
|
||||
|
||||
const BASE_URL = (process.env.MNOTE_WEB_SMOKE_BASE_URL || "http://127.0.0.1:3000").replace(/\/+$/, "");
|
||||
const ROOT_PATH = process.env.MNOTE_KNOWLEDGE_RAG_ROOT_PATH || "/mnt/Data1T/Mnote_data/users/mnote-e2e/workspaces/my-space";
|
||||
const ROOT_URI = process.env.MNOTE_KNOWLEDGE_RAG_ROOT_URI || `file://${ROOT_PATH}`;
|
||||
const WORKSPACE_ID = process.env.MNOTE_KNOWLEDGE_RAG_WORKSPACE_ID || "local-ws:mnote-e2e:my-space";
|
||||
const SOURCE_DOCX = process.env.MNOTE_KNOWLEDGE_RAG_DOCX_SOURCE || path.join(process.cwd(), "tmp", "onlyoffice-direct-bridge.docx");
|
||||
const FIXTURE_DIR = "knowledge-rag-fixtures-7-50";
|
||||
const OUTPUT_DIR = path.join(process.cwd(), "tmp", "task532-knowledge-rag-docx-ingestion-smoke");
|
||||
const RESULT_PATH = path.join(OUTPUT_DIR, "result.json");
|
||||
const SCREENSHOT_PATH = path.join(OUTPUT_DIR, "docx-resource-tab.png");
|
||||
const POLL_TIMEOUT_MS = Number(process.env.MNOTE_KNOWLEDGE_RAG_DOCX_TIMEOUT_MS || 10 * 60_000);
|
||||
const UI_TIMEOUT_MS = Number(process.env.MNOTE_SMOKE_UI_TIMEOUT_MS || 30_000);
|
||||
const CHROMIUM_EXECUTABLE_PATH = process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH
|
||||
|| ["/usr/bin/google-chrome-stable", "/usr/bin/google-chrome", "/snap/bin/chromium", "/usr/bin/chromium"]
|
||||
.find((candidate) => fs.existsSync(candidate));
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
function xmlEscape(value) {
|
||||
return String(value)
|
||||
.replaceAll("&", "&")
|
||||
.replaceAll("<", "<")
|
||||
.replaceAll(">", ">")
|
||||
.replaceAll('"', """)
|
||||
.replaceAll("'", "'");
|
||||
}
|
||||
|
||||
async function writeDocxFixture(targetPath, marker) {
|
||||
assert(fs.existsSync(SOURCE_DOCX), `缺少 DOCX 源文件: ${SOURCE_DOCX}`);
|
||||
const zip = await JSZip.loadAsync(fs.readFileSync(SOURCE_DOCX));
|
||||
const documentFile = zip.file("word/document.xml");
|
||||
assert(documentFile, `DOCX 缺少 word/document.xml: ${SOURCE_DOCX}`);
|
||||
const originalXml = await documentFile.async("string");
|
||||
const paragraph = `<w:p><w:r><w:t>${xmlEscape(marker)}</w:t></w:r></w:p>`;
|
||||
const nextXml = originalXml.includes("</w:body>")
|
||||
? originalXml.replace("</w:body>", `${paragraph}</w:body>`)
|
||||
: originalXml + paragraph;
|
||||
zip.file("word/document.xml", nextXml);
|
||||
fs.mkdirSync(path.dirname(targetPath), { recursive: true });
|
||||
const buffer = await zip.generateAsync({ type: "nodebuffer" });
|
||||
fs.writeFileSync(targetPath, buffer);
|
||||
}
|
||||
|
||||
async function signIn(context) {
|
||||
const response = await context.request.post(`${BASE_URL}/api/auth`, {
|
||||
data: {
|
||||
action: "auth:signIn",
|
||||
args: {
|
||||
provider: "password",
|
||||
params: {
|
||||
email: "mnote.e2e@example.com",
|
||||
password: process.env.MNOTE_E2E_PASSWORD || "MnoteE2E123!",
|
||||
flow: "signIn",
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
assert(response.ok(), `登录失败: ${response.status()} ${await response.text()}`);
|
||||
}
|
||||
|
||||
async function apiJson(context, method, url, data) {
|
||||
const response = await context.request.fetch(url, {
|
||||
method,
|
||||
data,
|
||||
headers: {
|
||||
"x-mnote-actor-id": "mnote-e2e",
|
||||
"x-mnote-actor-type": "user",
|
||||
"accept": "application/json",
|
||||
},
|
||||
timeout: UI_TIMEOUT_MS,
|
||||
});
|
||||
const text = await response.text();
|
||||
let payload = null;
|
||||
try {
|
||||
payload = text ? JSON.parse(text) : null;
|
||||
} catch (_) {
|
||||
payload = { rawText: text };
|
||||
}
|
||||
return { ok: response.ok(), status: response.status(), payload, text };
|
||||
}
|
||||
|
||||
async function status(context) {
|
||||
const params = new URLSearchParams({ rootUri: ROOT_URI, workspaceId: WORKSPACE_ID });
|
||||
const result = await apiJson(context, "GET", `${BASE_URL}/api/knowledge-rag/status?${params.toString()}`);
|
||||
assert(result.ok, `knowledge-rag status 失败: ${result.status} ${result.text.slice(0, 1000)}`);
|
||||
return result.payload;
|
||||
}
|
||||
|
||||
function findRegistryEntry(payload, sourcePath) {
|
||||
const entries = payload?.registry?.entries;
|
||||
return Array.isArray(entries) ? entries.find((entry) => entry.sourceRootRelativePath === sourcePath) : null;
|
||||
}
|
||||
|
||||
async function ingest(context, sourcePath) {
|
||||
const result = await apiJson(context, "POST", `${BASE_URL}/api/knowledge-rag/ingest`, {
|
||||
rootUri: ROOT_URI,
|
||||
workspaceId: WORKSPACE_ID,
|
||||
sources: [{ sourcePath }],
|
||||
});
|
||||
assert(result.ok, `knowledge-rag ingest 失败: ${result.status} ${result.text.slice(0, 1000)}`);
|
||||
return result.payload;
|
||||
}
|
||||
|
||||
async function waitForIndexed(context, sourcePath) {
|
||||
const startedAt = Date.now();
|
||||
let attempts = 0;
|
||||
let lastStatus = null;
|
||||
let lastIngest = null;
|
||||
while (Date.now() - startedAt < POLL_TIMEOUT_MS) {
|
||||
attempts += 1;
|
||||
lastStatus = await status(context);
|
||||
const entry = findRegistryEntry(lastStatus, sourcePath);
|
||||
if (entry && entry.indexedAtMs && entry.lightRagDocId && !entry.stale && !entry.deletedAtMs) {
|
||||
return { entry, status: lastStatus, attempts, lastIngest };
|
||||
}
|
||||
if (!entry || attempts % 6 === 1) {
|
||||
lastIngest = await ingest(context, sourcePath);
|
||||
}
|
||||
await sleep(5_000);
|
||||
}
|
||||
throw new Error(`等待 DOCX 入库超时: ${JSON.stringify({ sourcePath, lastIngest, lastStatus }, null, 2).slice(0, 4000)}`);
|
||||
}
|
||||
|
||||
async function queryDocx(context, marker, sourcePath) {
|
||||
const result = await apiJson(context, "POST", `${BASE_URL}/api/knowledge-rag/query`, {
|
||||
rootUri: ROOT_URI,
|
||||
workspaceId: WORKSPACE_ID,
|
||||
query: marker,
|
||||
mode: "mix",
|
||||
topK: 8,
|
||||
chunkTopK: 8,
|
||||
includeChunkContent: true,
|
||||
});
|
||||
assert(result.ok, `knowledge-rag query 失败: ${result.status} ${result.text.slice(0, 1000)}`);
|
||||
const references = Array.isArray(result.payload?.references) ? result.payload.references : [];
|
||||
const reference = references.find((item) => item.sourceRootRelativePath === sourcePath);
|
||||
assert(reference, `DOCX 查询缺少目标引用 ${sourcePath}: ${JSON.stringify(references, null, 2).slice(0, 3000)}`);
|
||||
assert(String(reference.quote || reference.content || result.text).toLowerCase().includes(marker.toLowerCase()), `DOCX 引用缺少 marker: ${JSON.stringify(reference, null, 2).slice(0, 2000)}`);
|
||||
assert(String(reference.citationUrl || "").includes("resourceTab="), `DOCX 引用缺少 resourceTab citationUrl: ${JSON.stringify(reference, null, 2)}`);
|
||||
assert(String(reference.citationMarkdown || "").includes(path.basename(sourcePath)), `DOCX 引用缺少 citationMarkdown: ${JSON.stringify(reference, null, 2)}`);
|
||||
return { payload: result.payload, reference };
|
||||
}
|
||||
|
||||
async function openCitationInBrowser(context, reference, sourcePath) {
|
||||
const page = await context.newPage();
|
||||
await page.goto(`${BASE_URL}${reference.citationUrl}`, { waitUntil: "domcontentloaded", timeout: UI_TIMEOUT_MS });
|
||||
await page.waitForFunction(
|
||||
(expectedResource) => document.querySelector(`[data-mnote-resource-tab-panel][data-resource-path="${expectedResource}"]`),
|
||||
sourcePath,
|
||||
{ timeout: UI_TIMEOUT_MS },
|
||||
);
|
||||
await page.waitForTimeout(2_000);
|
||||
const state = await page.evaluate((expectedResource) => {
|
||||
const panel = document.querySelector(`[data-mnote-resource-tab-panel][data-resource-path="${expectedResource}"]`);
|
||||
const activeTab = document.querySelector(".mnote-main-tab.is-active, [data-mnote-tab-kind].is-active");
|
||||
return {
|
||||
url: location.href,
|
||||
panelVisible: !!panel && !panel.hidden,
|
||||
panelResourcePath: panel ? panel.getAttribute("data-resource-path") : "",
|
||||
panelKind: panel ? panel.getAttribute("data-resource-kind") : "",
|
||||
activeTabText: activeTab ? activeTab.textContent.trim().slice(0, 160) : "",
|
||||
bodyText: document.body.innerText.slice(0, 1000),
|
||||
};
|
||||
}, sourcePath);
|
||||
await page.screenshot({ path: SCREENSHOT_PATH, fullPage: true });
|
||||
await page.close().catch(() => undefined);
|
||||
assert.equal(state.panelVisible, true, `DOCX citationUrl 未打开资源 tab: ${JSON.stringify(state, null, 2)}`);
|
||||
assert.equal(state.panelResourcePath, sourcePath, `DOCX resource tab path 不匹配: ${JSON.stringify(state, null, 2)}`);
|
||||
return state;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
fs.mkdirSync(OUTPUT_DIR, { recursive: true });
|
||||
const marker = process.env.MNOTE_KNOWLEDGE_RAG_DOCX_MARKER || `DOCX RAG SMOKE ${Date.now()}`;
|
||||
const fileName = `docx-rag-smoke-${Date.now()}.docx`;
|
||||
const sourcePath = `${FIXTURE_DIR}/${fileName}`;
|
||||
const targetPath = path.join(ROOT_PATH, sourcePath);
|
||||
await writeDocxFixture(targetPath, marker);
|
||||
|
||||
const browser = await chromium.launch({
|
||||
headless: true,
|
||||
...(CHROMIUM_EXECUTABLE_PATH ? { executablePath: CHROMIUM_EXECUTABLE_PATH } : {}),
|
||||
});
|
||||
const context = await browser.newContext({ viewport: { width: 1360, height: 900 } });
|
||||
try {
|
||||
await signIn(context);
|
||||
const ingestPayload = await ingest(context, sourcePath);
|
||||
const indexed = await waitForIndexed(context, sourcePath);
|
||||
const queried = await queryDocx(context, marker, sourcePath);
|
||||
const browserState = await openCitationInBrowser(context, queried.reference, sourcePath);
|
||||
const result = {
|
||||
ok: true,
|
||||
baseUrl: BASE_URL,
|
||||
rootUri: ROOT_URI,
|
||||
workspaceId: WORKSPACE_ID,
|
||||
sourcePath,
|
||||
marker,
|
||||
lightRagDocId: indexed.entry.lightRagDocId,
|
||||
lightRagFilePath: indexed.entry.lightRagFilePath,
|
||||
ingestStatus: ingestPayload?.status,
|
||||
attempts: indexed.attempts,
|
||||
citationMarkdown: queried.reference.citationMarkdown,
|
||||
citationUrl: queried.reference.citationUrl,
|
||||
locatorDegraded: queried.reference.locatorDegraded,
|
||||
browserState,
|
||||
screenshot: SCREENSHOT_PATH,
|
||||
};
|
||||
fs.writeFileSync(RESULT_PATH, `${JSON.stringify(result, null, 2)}\n`, "utf8");
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
} finally {
|
||||
await browser.close().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
fs.mkdirSync(OUTPUT_DIR, { recursive: true });
|
||||
fs.writeFileSync(
|
||||
path.join(OUTPUT_DIR, "failure.json"),
|
||||
`${JSON.stringify({ ok: false, error: error.stack || error.message || String(error) }, null, 2)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
console.error(error.stack || error.message || String(error));
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user