feat(rag): replace LiteParse flows with LightRAG provider

2026-06-07 01:10:31 +08:00
parent f46c2fb5d0
commit 1f25364374
40 changed files with 7232 additions and 2350 deletions
@@ -0,0 +1,217 @@
+#!/usr/bin/env node
+"use strict";
+
+const assert = require("node:assert");
+const fs = require("node:fs");
+const path = require("node:path");
+const { chromium } = require("playwright");
+
+const BASE_URL = (process.env.MNOTE_WEB_SMOKE_BASE_URL || "http://127.0.0.1:3000").replace(/\/+$/, "");
+const ROOT_PATH = process.env.MNOTE_KNOWLEDGE_RAG_ROOT_PATH || "/mnt/Data1T/Mnote_data/users/mnote-e2e/workspaces/my-space";
+const ROOT_URI = process.env.MNOTE_KNOWLEDGE_RAG_ROOT_URI || `file://${ROOT_PATH}`;
+const WORKSPACE_ID = process.env.MNOTE_KNOWLEDGE_RAG_WORKSPACE_ID || "local-ws:mnote-e2e:my-space";
+const FIXTURE_DIR = "knowledge-rag-fixtures-7-50";
+const OUTPUT_DIR = path.join(process.cwd(), "tmp", "task534-knowledge-rag-source-management-scope-smoke");
+const RESULT_PATH = path.join(OUTPUT_DIR, "result.json");
+const SCREENSHOT_PATH = path.join(OUTPUT_DIR, "source-management.png");
+const UI_TIMEOUT_MS = Number(process.env.MNOTE_SMOKE_UI_TIMEOUT_MS || 30_000);
+const POLL_TIMEOUT_MS = Number(process.env.MNOTE_KNOWLEDGE_RAG_SCOPE_TIMEOUT_MS || 240_000);
+const CHROMIUM_EXECUTABLE_PATH = process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH
+  || ["/usr/bin/google-chrome-stable", "/usr/bin/google-chrome", "/snap/bin/chromium", "/usr/bin/chromium"]
+    .find((candidate) => fs.existsSync(candidate));
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+async function signIn(context) {
+  const response = await context.request.post(`${BASE_URL}/api/auth`, {
+    data: {
+      action: "auth:signIn",
+      args: {
+        provider: "password",
+        params: {
+          email: "mnote.e2e@example.com",
+          password: process.env.MNOTE_E2E_PASSWORD || "MnoteE2E123!",
+          flow: "signIn",
+        },
+      },
+    },
+  });
+  assert(response.ok(), `登录失败: ${response.status()} ${await response.text()}`);
+}
+
+async function apiJson(context, method, url, data) {
+  const response = await context.request.fetch(url, {
+    method,
+    data,
+    headers: {
+      "x-mnote-actor-id": "mnote-e2e",
+      "x-mnote-actor-type": "user",
+      accept: "application/json",
+    },
+    timeout: UI_TIMEOUT_MS,
+  });
+  const text = await response.text();
+  let payload = null;
+  try {
+    payload = text ? JSON.parse(text) : null;
+  } catch (_) {
+    payload = { rawText: text };
+  }
+  return { ok: response.ok(), status: response.status(), payload, text };
+}
+
+async function status(context) {
+  const params = new URLSearchParams({ rootUri: ROOT_URI, workspaceId: WORKSPACE_ID });
+  const result = await apiJson(context, "GET", `${BASE_URL}/api/knowledge-rag/status?${params.toString()}`);
+  assert(result.ok, `knowledge-rag status 失败: ${result.status} ${result.text.slice(0, 1000)}`);
+  return result.payload;
+}
+
+function entryFor(statusPayload, sourcePath) {
+  const entries = statusPayload?.registry?.entries;
+  return Array.isArray(entries) ? entries.find((entry) => entry.sourceRootRelativePath === sourcePath) : null;
+}
+
+async function ingest(context, sourcePaths) {
+  const result = await apiJson(context, "POST", `${BASE_URL}/api/knowledge-rag/ingest`, {
+    rootUri: ROOT_URI,
+    workspaceId: WORKSPACE_ID,
+    sources: sourcePaths.map((sourcePath) => ({ sourcePath })),
+  });
+  assert(result.ok, `knowledge-rag ingest 失败: ${result.status} ${result.text.slice(0, 1000)}`);
+  return result.payload;
+}
+
+async function waitForIndexed(context, sourcePaths) {
+  const startedAt = Date.now();
+  let lastStatus = null;
+  while (Date.now() - startedAt < POLL_TIMEOUT_MS) {
+    lastStatus = await status(context);
+    const allIndexed = sourcePaths.every((sourcePath) => {
+      const entry = entryFor(lastStatus, sourcePath);
+      return entry && entry.indexedAtMs && entry.lightRagDocId && !entry.stale && !entry.deletedAtMs;
+    });
+    if (allIndexed) return lastStatus;
+    await sleep(5_000);
+  }
+  throw new Error(`等待 source scope fixture 入库超时: ${JSON.stringify({ sourcePaths, lastStatus }, null, 2).slice(0, 4000)}`);
+}
+
+async function query(context, queryText, sourcePaths) {
+  const result = await apiJson(context, "POST", `${BASE_URL}/api/knowledge-rag/query`, {
+    rootUri: ROOT_URI,
+    workspaceId: WORKSPACE_ID,
+    query: queryText,
+    mode: "mix",
+    topK: 12,
+    chunkTopK: 12,
+    includeChunkContent: true,
+    sourcePaths,
+  });
+  assert(result.ok, `knowledge-rag query 失败: ${result.status} ${result.text.slice(0, 1000)}`);
+  return result.payload;
+}
+
+async function openKnowledgePanel(page) {
+  const url = new URL(`${BASE_URL}/`);
+  url.searchParams.set("sourceKind", "local_folder");
+  url.searchParams.set("rootUri", ROOT_URI);
+  url.searchParams.set("treeView", "filetree");
+  await page.goto(url.toString(), { waitUntil: "domcontentloaded", timeout: UI_TIMEOUT_MS });
+  await page.locator('[data-testid="mnote-knowledge-rag-settings-toggle"]').click({ timeout: UI_TIMEOUT_MS });
+  await page.locator('[data-testid="mnote-knowledge-rag-settings-popover"]').waitFor({
+    state: "visible",
+    timeout: UI_TIMEOUT_MS,
+  });
+}
+
+async function waitForSourceRow(page, sourcePath) {
+  const row = page.locator(`[data-knowledge-rag-source-row="true"][data-knowledge-rag-source-path="${sourcePath}"]`).first();
+  await row.waitFor({ state: "visible", timeout: UI_TIMEOUT_MS });
+  return row;
+}
+
+async function main() {
+  fs.mkdirSync(OUTPUT_DIR, { recursive: true });
+  fs.mkdirSync(path.join(ROOT_PATH, FIXTURE_DIR), { recursive: true });
+  const marker = `SOURCE SCOPE RAG ${Date.now()}`;
+  const sourceA = `${FIXTURE_DIR}/scope-alpha-${Date.now()}.md`;
+  const sourceB = `${FIXTURE_DIR}/scope-beta-${Date.now()}.md`;
+  fs.writeFileSync(path.join(ROOT_PATH, sourceA), `# Scope Alpha\n${marker}\nOnly alpha source should remain after scope filter.\n`, "utf8");
+  fs.writeFileSync(path.join(ROOT_PATH, sourceB), `# Scope Beta\n${marker}\nBeta source must be filtered when sourcePaths targets alpha.\n`, "utf8");
+
+  const browser = await chromium.launch({
+    headless: true,
+    ...(CHROMIUM_EXECUTABLE_PATH ? { executablePath: CHROMIUM_EXECUTABLE_PATH } : {}),
+  });
+  const context = await browser.newContext({ viewport: { width: 1360, height: 900 } });
+  const page = await context.newPage();
+  try {
+    await signIn(context);
+    await ingest(context, [sourceA, sourceB]);
+    const indexedStatus = await waitForIndexed(context, [sourceA, sourceB]);
+
+    const scoped = await query(context, marker, [sourceA]);
+    const scopedReferences = Array.isArray(scoped.references) ? scoped.references : [];
+    assert(scopedReferences.length > 0, `sourcePaths scope 应至少返回 alpha: ${JSON.stringify(scoped, null, 2).slice(0, 3000)}`);
+    assert(
+      scopedReferences.every((reference) => reference.sourceRootRelativePath === sourceA),
+      `sourcePaths scope 不应返回非 alpha 来源: ${JSON.stringify(scopedReferences, null, 2).slice(0, 3000)}`,
+    );
+
+    await openKnowledgePanel(page);
+    const row = await waitForSourceRow(page, sourceA);
+    await row.locator('[data-knowledge-rag-action="reindex-source"]').waitFor({ state: "visible", timeout: UI_TIMEOUT_MS });
+    const deleteButton = row.locator('[data-knowledge-rag-action="delete-source"]');
+    await deleteButton.waitFor({ state: "visible", timeout: UI_TIMEOUT_MS });
+    await deleteButton.click({ timeout: UI_TIMEOUT_MS });
+    await page.waitForFunction(
+      (expectedPath) => {
+        const row = document.querySelector(`[data-knowledge-rag-source-row="true"][data-knowledge-rag-source-path="${expectedPath}"]`);
+        return row && (row.textContent || "").includes("已删除");
+      },
+      sourceA,
+      { timeout: UI_TIMEOUT_MS },
+    );
+    await page.screenshot({ path: SCREENSHOT_PATH, fullPage: true });
+    assert(fs.existsSync(path.join(ROOT_PATH, sourceA)), "UI 删除索引不应删除原始 source 文件");
+
+    const afterDelete = await query(context, marker, [sourceA]);
+    const afterReferences = Array.isArray(afterDelete.references) ? afterDelete.references : [];
+    assert.equal(afterReferences.length, 0, `删除索引后 source-scoped query 不应继续返回 alpha: ${JSON.stringify(afterReferences, null, 2)}`);
+
+    const result = {
+      ok: true,
+      baseUrl: BASE_URL,
+      rootUri: ROOT_URI,
+      workspaceId: WORKSPACE_ID,
+      marker,
+      sourceA,
+      sourceB,
+      sourceScope: scoped.sourceScope,
+      scopedReferenceCount: scopedReferences.length,
+      alphaDocId: entryFor(indexedStatus, sourceA)?.lightRagDocId || null,
+      betaDocId: entryFor(indexedStatus, sourceB)?.lightRagDocId || null,
+      sourceAExistsAfterDelete: fs.existsSync(path.join(ROOT_PATH, sourceA)),
+      afterDeleteReferenceCount: afterReferences.length,
+      screenshot: SCREENSHOT_PATH,
+    };
+    fs.writeFileSync(RESULT_PATH, `${JSON.stringify(result, null, 2)}\n`, "utf8");
+    console.log(JSON.stringify(result, null, 2));
+  } finally {
+    await browser.close().catch(() => undefined);
+  }
+}
+
+main().catch((error) => {
+  fs.mkdirSync(OUTPUT_DIR, { recursive: true });
+  fs.writeFileSync(
+    path.join(OUTPUT_DIR, "failure.json"),
+    `${JSON.stringify({ ok: false, error: error.stack || error.message || String(error) }, null, 2)}\n`,
+    "utf8",
+  );
+  console.error(error.stack || error.message || String(error));
+  process.exit(1);
+});