mnote/scripts/task533-knowledge-rag-source-watcher-sync-smoke.js

#!/usr/bin/env node
"use strict";

const assert = require("node:assert");
const fs = require("node:fs");
const path = require("node:path");
const { chromium } = require("playwright");

const BASE_URL = (process.env.MNOTE_WEB_SMOKE_BASE_URL || "http://127.0.0.1:3000").replace(/\/+$/, "");
const ROOT_PATH = process.env.MNOTE_KNOWLEDGE_RAG_ROOT_PATH || "/mnt/Data1T/Mnote_data/users/mnote-e2e/workspaces/my-space";
const ROOT_URI = process.env.MNOTE_KNOWLEDGE_RAG_ROOT_URI || `file://${ROOT_PATH}`;
const WORKSPACE_ID = process.env.MNOTE_KNOWLEDGE_RAG_WORKSPACE_ID || "local-ws:mnote-e2e:my-space";
const FIXTURE_DIR = "knowledge-rag-fixtures-7-50";
const OUTPUT_DIR = path.join(process.cwd(), "tmp", "task533-knowledge-rag-source-watcher-sync-smoke");
const RESULT_PATH = path.join(OUTPUT_DIR, "result.json");
const UI_TIMEOUT_MS = Number(process.env.MNOTE_SMOKE_UI_TIMEOUT_MS || 30_000);
const POLL_TIMEOUT_MS = Number(process.env.MNOTE_KNOWLEDGE_RAG_WATCHER_TIMEOUT_MS || 180_000);
const CHROMIUM_EXECUTABLE_PATH = process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH
  || ["/usr/bin/google-chrome-stable", "/usr/bin/google-chrome", "/snap/bin/chromium", "/usr/bin/chromium"]
    .find((candidate) => fs.existsSync(candidate));

function sleep(ms) {
  return new Promise((resolve) => setTimeout(resolve, ms));
}

function registryPath() {
  return path.join(ROOT_PATH, ".mnote", "index", "lightrag-source-registry.json");
}

function readRegistry() {
  return JSON.parse(fs.readFileSync(registryPath(), "utf8"));
}

function findEntry(sourcePath) {
  const registry = readRegistry();
  const entries = Array.isArray(registry.entries) ? registry.entries : [];
  return entries.find((entry) => entry.sourceRootRelativePath === sourcePath) || null;
}

async function signIn(context) {
  const response = await context.request.post(`${BASE_URL}/api/auth`, {
    data: {
      action: "auth:signIn",
      args: {
        provider: "password",
        params: {
          email: "mnote.e2e@example.com",
          password: process.env.MNOTE_E2E_PASSWORD || "MnoteE2E123!",
          flow: "signIn",
        },
      },
    },
  });
  assert(response.ok(), `登录失败: ${response.status()} ${await response.text()}`);
}

async function apiJson(context, method, url, data) {
  const response = await context.request.fetch(url, {
    method,
    data,
    headers: {
      "x-mnote-actor-id": "mnote-e2e",
      "x-mnote-actor-type": "user",
      accept: "application/json",
    },
    timeout: UI_TIMEOUT_MS,
  });
  const text = await response.text();
  let payload = null;
  try {
    payload = text ? JSON.parse(text) : null;
  } catch (_) {
    payload = { rawText: text };
  }
  return { ok: response.ok(), status: response.status(), payload, text };
}

async function status(context) {
  const params = new URLSearchParams({ rootUri: ROOT_URI, workspaceId: WORKSPACE_ID });
  const result = await apiJson(context, "GET", `${BASE_URL}/api/knowledge-rag/status?${params.toString()}`);
  assert(result.ok, `knowledge-rag status 失败: ${result.status} ${result.text.slice(0, 1000)}`);
  return result.payload;
}

async function ingest(context, sourcePath) {
  const result = await apiJson(context, "POST", `${BASE_URL}/api/knowledge-rag/ingest`, {
    rootUri: ROOT_URI,
    workspaceId: WORKSPACE_ID,
    sources: [{ sourcePath }],
  });
  assert(result.ok, `knowledge-rag ingest 失败: ${result.status} ${result.text.slice(0, 1000)}`);
  return result.payload;
}

async function waitForIndexed(context, sourcePath) {
  const startedAt = Date.now();
  let attempts = 0;
  while (Date.now() - startedAt < POLL_TIMEOUT_MS) {
    attempts += 1;
    await status(context);
    const entry = findEntry(sourcePath);
    if (entry && entry.indexedAtMs && entry.lightRagDocId && !entry.stale && !entry.deletedAtMs) {
      return { entry, attempts };
    }
    if (!entry || attempts % 6 === 1) {
      await ingest(context, sourcePath);
    }
    await sleep(5_000);
  }
  throw new Error(`等待 watcher source 入库超时: ${sourcePath}`);
}

async function openTreeLiveWatcher(page) {
  await page.goto(`${BASE_URL}/`, { waitUntil: "domcontentloaded", timeout: UI_TIMEOUT_MS });
  return page.evaluate(({ rootUri }) => new Promise((resolve, reject) => {
    const params = new URLSearchParams({ rootUri, treeLive: "true" });
    const source = new EventSource(`/api/local-folder/events?${params.toString()}`);
    window.__task533Events = [];
    window.__task533Source = source;
    const timer = setTimeout(() => reject(new Error("treeLive watcher ready timeout")), 30_000);
    source.addEventListener("snapshot", () => {
      clearTimeout(timer);
      resolve(true);
    });
    source.onerror = () => {
      clearTimeout(timer);
      reject(new Error("treeLive watcher error"));
    };
  }), { rootUri: ROOT_URI });
}

async function waitForWatchBatch(page, relativePath) {
  return page.evaluate(({ expectedPath, timeoutMs }) => new Promise((resolve, reject) => {
    const source = window.__task533Source;
    if (!source) {
      reject(new Error("treeLive watcher not opened"));
      return;
    }
    const timer = setTimeout(() => reject(new Error("watch_batch timeout")), timeoutMs);
    source.addEventListener("watch_batch", (event) => {
      const payload = JSON.parse(event.data || "{}");
      window.__task533Events.push(payload);
      const changed = Array.isArray(payload.changedPaths) ? payload.changedPaths : [];
      if (changed.some((item) => item.relativePath === expectedPath)) {
        clearTimeout(timer);
        resolve(payload);
      }
    });
  }), { expectedPath: relativePath, timeoutMs: POLL_TIMEOUT_MS });
}

async function waitForRegistryStale(sourcePath) {
  const startedAt = Date.now();
  while (Date.now() - startedAt < POLL_TIMEOUT_MS) {
    const entry = findEntry(sourcePath);
    if (entry && entry.stale === true && entry.deletedAtMs && !entry.lightRagDocId && !entry.indexedAtMs) {
      return entry;
    }
    await sleep(1_000);
  }
  throw new Error(`watcher 未把 source 标记 stale/deleted: ${JSON.stringify(findEntry(sourcePath), null, 2)}`);
}

async function queryAfterDelete(context, marker, sourcePath) {
  const result = await apiJson(context, "POST", `${BASE_URL}/api/knowledge-rag/query`, {
    rootUri: ROOT_URI,
    workspaceId: WORKSPACE_ID,
    query: marker,
    mode: "mix",
    topK: 8,
    chunkTopK: 8,
    includeChunkContent: true,
  });
  assert(result.ok, `knowledge-rag query 失败: ${result.status} ${result.text.slice(0, 1000)}`);
  const references = Array.isArray(result.payload?.references) ? result.payload.references : [];
  assert(
    !references.some((item) => item.sourceRootRelativePath === sourcePath),
    `删除后的 source 不应继续作为有效 reference: ${JSON.stringify(references, null, 2).slice(0, 3000)}`,
  );
  return result.payload;
}

async function main() {
  fs.mkdirSync(OUTPUT_DIR, { recursive: true });
  fs.mkdirSync(path.join(ROOT_PATH, FIXTURE_DIR), { recursive: true });
  const marker = `WATCHER RAG STALE ${Date.now()}`;
  const sourcePath = `${FIXTURE_DIR}/watcher-stale-${Date.now()}.md`;
  const absoluteSource = path.join(ROOT_PATH, sourcePath);
  fs.writeFileSync(absoluteSource, `# Watcher stale\n${marker}\n`, "utf8");

  const browser = await chromium.launch({
    headless: true,
    ...(CHROMIUM_EXECUTABLE_PATH ? { executablePath: CHROMIUM_EXECUTABLE_PATH } : {}),
  });
  const context = await browser.newContext({ viewport: { width: 1280, height: 860 } });
  const page = await context.newPage();
  try {
    await signIn(context);
    await ingest(context, sourcePath);
    const indexed = await waitForIndexed(context, sourcePath);
    await openTreeLiveWatcher(page);
    fs.unlinkSync(absoluteSource);
    const watchBatch = await waitForWatchBatch(page, sourcePath);
    const staleEntry = await waitForRegistryStale(sourcePath);
    const queryPayload = await queryAfterDelete(context, marker, sourcePath);
    const result = {
      ok: true,
      baseUrl: BASE_URL,
      rootUri: ROOT_URI,
      workspaceId: WORKSPACE_ID,
      sourcePath,
      marker,
      initialDocId: indexed.entry.lightRagDocId,
      indexedAttempts: indexed.attempts,
      watchBatch,
      staleEntry,
      queryReferenceCount: Array.isArray(queryPayload.references) ? queryPayload.references.length : 0,
    };
    fs.writeFileSync(RESULT_PATH, `${JSON.stringify(result, null, 2)}\n`, "utf8");
    console.log(JSON.stringify(result, null, 2));
  } finally {
    await browser.close().catch(() => undefined);
  }
}

main().catch((error) => {
  fs.mkdirSync(OUTPUT_DIR, { recursive: true });
  fs.writeFileSync(
    path.join(OUTPUT_DIR, "failure.json"),
    `${JSON.stringify({ ok: false, error: error.stack || error.message || String(error) }, null, 2)}\n`,
    "utf8",
  );
  console.error(error.stack || error.message || String(error));
  process.exit(1);
});