lix/mnote
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: harden lightrag knowledge rag runtime

Browse Source
This commit is contained in:
lix-2026
2026-06-09 22:12:09 +08:00
parent 922965d30f
commit f698aad964
22 changed files with 899 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rust/crates/mnote-web/src/routes/command_support.rs
+1 -1
View File
@@ -1,7 +1,7 @@
use crate::app::{AppConfig, AppState};
use crate::context::RequestContext;
use crate::error::WebError;
use crate::transport::convex::{
use crate::transport::legacy_cloud_guard::{
execute_retired_command_plan, execute_retired_command_plan_with_artifacts,
RetiredCloudCommandExecution,
};
rust/crates/mnote-web/src/routes/documents.rs
+1 -1
View File
@@ -208,7 +208,7 @@ fn ok_response(context: &RequestContext, result: Value) -> (StatusCode, HeaderMa
}
fn execution_artifacts_json(
execution: &crate::transport::convex::RetiredCloudCommandExecution,
execution: &crate::transport::legacy_cloud_guard::RetiredCloudCommandExecution,
) -> Value {
execution
.artifacts
rust/crates/mnote-web/src/routes/gateway.rs
+1 -1
View File
@@ -18,7 +18,7 @@ use crate::routes::web_shell::{
render_local_file_tree_html_scoped, render_local_sidebar_tree_html,
render_local_sidebar_tree_html_from_snapshot,
};
use crate::transport::convex::execute_retired_mutation_by_name;
use crate::transport::legacy_cloud_guard::execute_retired_mutation_by_name;
use crate::workspace_shell::{
apply_active_page, build_workspace_shell_projection, render_workspace_shell_sidebar_html,
};
rust/crates/mnote-web/src/routes/hermes_client.rs
+3 -1
View File
@@ -3,7 +3,9 @@ use crate::app::AppState;
use crate::context::RequestContext;
use crate::error::WebError;
use crate::hermes_tools::manifest;
use crate::transport::convex::{execute_retired_mutation_by_name, execute_retired_query_by_name};
use crate::transport::legacy_cloud_guard::{
execute_retired_mutation_by_name, execute_retired_query_by_name,
};
use axum::body::Body;
use axum::extract::{Extension, Path, Query, State};
use axum::http::{header, HeaderMap, HeaderName, HeaderValue, StatusCode};
rust/crates/mnote-web/src/routes/knowledge_rag.rs
+132 -5
View File
@@ -3,11 +3,14 @@ use crate::context::RequestContext;
use crate::error::WebError;
use crate::routes::evidence::{citation_markdown_for_locator, citation_url_for_locator};
use crate::routes::local_folder_source;
use axum::body::Body;
use axum::extract::{Extension, Json, Query, State};
use axum::http::StatusCode;
use axum::http::{header, StatusCode};
use axum::response::Response;
use core_protocol::evidence::{
EvidenceBBox, EvidenceLocator, EvidenceOpenAction, EvidenceResourceKind,
};
use futures_util::TryStreamExt;
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
use std::collections::hash_map::DefaultHasher;
@@ -327,6 +330,59 @@ pub async fn status(
})))
}
pub async fn pipeline_events(
Extension(context): Extension<RequestContext>,
) -> Result<Response, WebError> {
let endpoint = lightrag_endpoint();
let url = format!(
"{}/documents/pipeline_status/events",
endpoint.trim_end_matches('/')
);
let client = reqwest::Client::builder()
.connect_timeout(Duration::from_secs(10))
.build()
.map_err(|error| {
WebError::internal(format!("LightRAG SSE client 初始化失败: {error}"))
.with_context(&context)
})?;
let mut request = client.get(&url).header("accept", "text/event-stream");
if let Some(api_key) = lightrag_api_key() {
request = request.header("X-API-Key", api_key);
}
let upstream = request.send().await.map_err(|error| {
WebError::bad_gateway_code(
"knowledge_rag_lightrag_events_unreachable",
format!("无法连接 LightRAG pipeline 事件流: {error}"),
)
.with_context(&context)
})?;
let upstream_status = upstream.status();
if !upstream_status.is_success() {
let text = upstream.text().await.unwrap_or_default();
return Err(WebError::bad_gateway_code(
"knowledge_rag_lightrag_events_error",
format!("LightRAG pipeline 事件流返回 HTTP {upstream_status}: {text}"),
)
.with_context(&context));
}
let stream = upstream.bytes_stream().map_err(|error| {
std::io::Error::new(
std::io::ErrorKind::Other,
format!("LightRAG pipeline 事件流读取失败: {error}"),
)
});
Response::builder()
.status(StatusCode::OK)
.header(header::CONTENT_TYPE, "text/event-stream; charset=utf-8")
.header(header::CACHE_CONTROL, "no-cache, no-transform")
.header("x-accel-buffering", "no")
.body(Body::from_stream(stream))
.map_err(|error| {
WebError::internal(format!("LightRAG pipeline 事件响应构造失败: {error}"))
.with_context(&context)
})
}
pub(crate) async fn sync_registry_for_root(
state: &AppState,
context: &RequestContext,
@@ -1447,13 +1503,29 @@ fn lightrag_pipeline_status_summary(value: &Value) -> Value {
.get("latest_message")
.and_then(Value::as_str)
.unwrap_or_default();
let progress = parse_lightrag_chunk_progress(latest_message);
let history_messages = lightrag_pipeline_history_messages(value);
let progress = parse_lightrag_chunk_progress(latest_message).or_else(|| {
history_messages
.iter()
.rev()
.find_map(|message| parse_lightrag_chunk_progress(message))
});
json!({
"ok": true,
"busy": value.get("busy").and_then(Value::as_bool).unwrap_or(false),
"destructiveBusy": value.get("destructive_busy").and_then(Value::as_bool).unwrap_or(false),
"scanning": value.get("scanning").and_then(Value::as_bool).unwrap_or(false),
"scanningExclusive": value.get("scanning_exclusive").and_then(Value::as_bool).unwrap_or(false),
"requestPending": value.get("request_pending").and_then(Value::as_bool).unwrap_or(false),
"pendingEnqueues": value.get("pending_enqueues").and_then(Value::as_u64).unwrap_or(0),
"pendingRequests": value.get("pending_requests").and_then(Value::as_bool).unwrap_or(false),
"docs": value.get("docs").and_then(Value::as_u64).unwrap_or(0),
"batches": value.get("batchs").and_then(Value::as_u64).unwrap_or(0),
"currentBatch": value.get("cur_batch").and_then(Value::as_u64).unwrap_or(0),
"jobName": value.get("job_name").and_then(Value::as_str).unwrap_or_default(),
"jobStart": value.get("job_start").cloned().unwrap_or(Value::Null),
"latestMessage": latest_message,
"historyMessages": history_messages,
"cancellationRequested": value.get("cancellation_requested").and_then(Value::as_bool).unwrap_or(false),
"cancellationReason": value.get("cancellation_reason").cloned().unwrap_or(Value::Null),
"progress": progress.map(|progress| json!({
@@ -1464,6 +1536,25 @@ fn lightrag_pipeline_status_summary(value: &Value) -> Value {
})
}
fn lightrag_pipeline_history_messages(value: &Value) -> Vec<String> {
value
.get("history_messages")
.and_then(Value::as_array)
.map(|messages| {
messages
.iter()
.filter_map(Value::as_str)
.rev()
.take(80)
.map(ToOwned::to_owned)
.collect::<Vec<_>>()
.into_iter()
.rev()
.collect()
})
.unwrap_or_default()
}
fn lightrag_rerank_status_summary(value: &Value) -> Value {
let configuration = value.get("configuration").unwrap_or(&Value::Null);
let queue = value.get("rerank_queue_status").unwrap_or(&Value::Null);
@@ -1510,13 +1601,15 @@ fn parse_lightrag_chunk_progress(message: &str) -> Option<LightRagChunkProgress>
let current = current.trim().parse::<u64>().ok()?;
let (total, _) = rest.split_once(' ')?;
let total = total.trim().parse::<u64>().ok()?;
let doc_token = message
.split_whitespace()
.find(|token| token.starts_with("doc-") && token.contains("-chunk-"))?;
let doc_token = message.split_whitespace().find(|token| {
token.starts_with("doc-") && (token.contains("-chunk-") || token.contains("-mm-"))
})?;
let doc_id = doc_token
.rsplit_once("-chunk-")
.or_else(|| doc_token.rsplit_once("-mm-"))
.map(|(doc_id, _)| doc_id)
.unwrap_or(doc_token)
.trim_end_matches(|ch: char| !ch.is_ascii_alphanumeric())
.to_string();
if total == 0 || current == 0 || doc_id.is_empty() {
return None;
@@ -3630,6 +3723,11 @@ mod tests {
let status = json!({
"busy": true,
"scanning": true,
"destructive_busy": false,
"pending_enqueues": 1,
"docs": 1,
"batchs": 2,
"cur_batch": 1,
"job_name": "book.docx",
"latest_message": "Chunk 212 of 333 extracted 4 Ent + 0 Rel doc-ff0b60997a285a85e5704a114d7b3ffa-chunk-212"
});
@@ -3638,6 +3736,10 @@ mod tests {
assert_eq!(summary["busy"], true);
assert_eq!(summary["scanning"], true);
assert_eq!(summary["pendingEnqueues"], 1);
assert_eq!(summary["docs"], 1);
assert_eq!(summary["batches"], 2);
assert_eq!(summary["currentBatch"], 1);
assert_eq!(summary["progress"]["current"], 212);
assert_eq!(summary["progress"]["total"], 333);
assert_eq!(
@@ -3646,6 +3748,31 @@ mod tests {
);
}
#[test]
fn lightrag_pipeline_summary_extracts_chunk_progress_from_history() {
let status = json!({
"busy": true,
"scanning": false,
"job_name": "book.pdf",
"latest_message": "Merging stage 1/1: book.pdf",
"history_messages": [
"Analyzing multimodal: doc-e51400269a2c41a2fcb22f1071193fad",
"Chunk 5 of 21 extracted 2 Ent + 1 Rel doc-e51400269a2c41a2fcb22f1071193fad-mm-drawing-001",
"Merging stage 1/1: book.pdf"
]
});
let summary = lightrag_pipeline_status_summary(&status);
assert_eq!(summary["progress"]["current"], 5);
assert_eq!(summary["progress"]["total"], 21);
assert_eq!(
summary["progress"]["docId"],
"doc-e51400269a2c41a2fcb22f1071193fad"
);
assert_eq!(summary["historyMessages"].as_array().unwrap().len(), 3);
}
#[test]
fn old_registry_json_defaults_indexed_roots() {
let registry: KnowledgeRagSourceRegistry = serde_json::from_value(json!({
rust/crates/mnote-web/src/routes/mindmap_api.rs
+1 -1
View File
@@ -129,7 +129,7 @@ async fn resolve_mindmap_workspace_id(
}
fn execution_artifacts_json(
execution: &crate::transport::convex::RetiredCloudCommandExecution,
execution: &crate::transport::legacy_cloud_guard::RetiredCloudCommandExecution,
) -> Value {
execution
.artifacts
rust/crates/mnote-web/src/routes/mod.rs
+4
View File
@@ -84,6 +84,10 @@ pub fn build_router(state: AppState) -> Router {
.route("/api/evidence/read", post(evidence::read))
.route("/api/evidence/open", post(evidence::open))
.route("/api/knowledge-rag/status", get(knowledge_rag::status))
.route(
"/api/knowledge-rag/pipeline-events",
get(knowledge_rag::pipeline_events),
)
.route("/api/knowledge-rag/ingest", post(knowledge_rag::ingest))
.route("/api/knowledge-rag/search", post(knowledge_rag::search))
.route("/api/knowledge-rag/query", post(knowledge_rag::query_rag))
rust/crates/mnote-web/src/routes/query_support.rs
+1 -1
View File
@@ -1,7 +1,7 @@
use crate::app::AppConfig;
use crate::context::RequestContext;
use crate::error::WebError;
use crate::transport::convex::execute_retired_query_plan;
use crate::transport::legacy_cloud_guard::execute_retired_query_plan;
use bridge_runtime::{
build_query_request, execute_runtime_input, execute_runtime_query, BridgeContext,
RuntimeActorWire, RuntimeBridgeContextWire, RuntimeExecutionPlan, RuntimeInput,
rust/crates/mnote-web/src/routes/resource_trash.rs
+1 -1
View File
@@ -7,7 +7,7 @@ use crate::routes::command_support::{
use crate::routes::local_folder_source::{
ensure_local_workspace_access, execute_local_tree_command,
};
use crate::transport::convex::{
use crate::transport::legacy_cloud_guard::{
execute_retired_mutation_by_name, execute_retired_query_by_name,
persist_runtime_command_artifacts,
};
rust/crates/mnote-web/src/routes/tree.rs
+1 -1
View File
@@ -15,7 +15,7 @@ use crate::routes::query_support::{
fetch_documents_meta_via_legacy_cloud, resolve_effective_workspace_id,
};
use crate::routes::snapshot_support::{load_projection_snapshot, ProjectionSnapshotSpec};
use crate::transport::convex::execute_retired_mutation_by_name;
use crate::transport::legacy_cloud_guard::execute_retired_mutation_by_name;
use crate::tree_shell::filetree_renderer::{
render_initial_filetree_html, FileTreeInitialRenderInput, FileTreeRenderRow,
};
rust/crates/mnote-web/src/routes/web_shell.rs
+75 -8
View File
@@ -79,8 +79,21 @@ pub async fn document_page_shell(
)
));
}
let primary_source_kind = normalize_source_kind(query.source_kind.as_deref());
let mut primary_source_kind = normalize_source_kind(query.source_kind.as_deref());
let primary_root_uri = normalize_optional_query_value(query.root_uri.as_deref());
if primary_source_kind.is_none()
&& document_id.trim().starts_with("local-md:")
&& primary_root_uri.is_some()
{
primary_source_kind = Some("local_folder");
}
if primary_source_kind.is_none() && document_id.trim().starts_with("local-md:") {
return redirect_response(&format!(
"/?routeGuard={}&missingPage={}",
query_escape("local_folder_source_required"),
query_escape(&document_id),
));
}
let aggregate = match build_page_aggregate_snapshot(
&state,
&context,
@@ -2249,12 +2262,25 @@ pub async fn pdf_preview_page(Query(query): Query<PdfPreviewQuery>) -> Response
if (viewer) viewer.append(canvas);
if (evidencePage === pageNumber) {{
canvas.setAttribute('data-mnote-evidence-page', 'true');
let evidenceTargetY = null;
if (evidenceBBox) {{
const rect = viewport.convertToViewportRectangle([evidenceBBox.x0, evidenceBBox.y0, evidenceBBox.x1, evidenceBBox.y1]);
const x = Math.min(rect[0], rect[2]);
const y = Math.min(rect[1], rect[3]);
const width = Math.max(1, Math.abs(rect[2] - rect[0]));
const height = Math.max(1, Math.abs(rect[3] - rect[1]));
const normalizedMineruBox = evidenceBBox.x0 >= 0 && evidenceBBox.y0 >= 0 && evidenceBBox.x1 <= 1000 && evidenceBBox.y1 <= 1000;
let x;
let y;
let width;
let height;
if (normalizedMineruBox) {{
x = Math.min(evidenceBBox.x0, evidenceBBox.x1) / 1000 * viewport.width;
y = Math.min(evidenceBBox.y0, evidenceBBox.y1) / 1000 * viewport.height;
width = Math.max(1, Math.abs(evidenceBBox.x1 - evidenceBBox.x0) / 1000 * viewport.width);
height = Math.max(1, Math.abs(evidenceBBox.y1 - evidenceBBox.y0) / 1000 * viewport.height);
}} else {{
const rect = viewport.convertToViewportRectangle([evidenceBBox.x0, evidenceBBox.y0, evidenceBBox.x1, evidenceBBox.y1]);
x = Math.min(rect[0], rect[2]);
y = Math.min(rect[1], rect[3]);
width = Math.max(1, Math.abs(rect[2] - rect[0]));
height = Math.max(1, Math.abs(rect[3] - rect[1]));
}}
context.save();
context.scale(outputScale, outputScale);
context.fillStyle = 'rgba(37, 99, 235, 0.18)';
@@ -2263,8 +2289,17 @@ pub async fn pdf_preview_page(Query(query): Query<PdfPreviewQuery>) -> Response
context.fillRect(x, y, width, height);
context.strokeRect(x, y, width, height);
context.restore();
evidenceTargetY = y + height / 2;
}}
window.setTimeout(() => canvas.scrollIntoView({{ block: 'center', inline: 'nearest' }}), 0);
window.setTimeout(() => {{
if (Number.isFinite(evidenceTargetY)) {{
const rect = canvas.getBoundingClientRect();
const absoluteTargetTop = rect.top + window.scrollY + evidenceTargetY;
window.scrollTo({{ top: Math.max(0, absoluteTargetTop - window.innerHeight / 2), behavior: 'auto' }});
return;
}}
canvas.scrollIntoView({{ block: 'center', inline: 'nearest' }});
}}, 0);
}}
}}
@@ -3869,6 +3904,38 @@ mod tests {
assert!(location.contains("missingPage=local-md%3Adocs%7E2FPlan.md"));
}
#[tokio::test]
async fn document_shell_local_markdown_without_source_kind_does_not_fall_back_to_convex() {
let response = app_with_unreachable_convex_without_fixture()
.oneshot(
Request::builder()
.uri("/documents/local-md:docs~2FPlan.md?resourceTab=primary%3A%3Aresource%3Afile%3Afile%3A%2F%2F%2Ftmp%3Adocs%2FPlan.pdf")
.header("x-mnote-actor-id", "user_test")
.header("x-mnote-actor-type", "user")
.body(Body::empty())
.expect("request"),
)
.await
.expect("response");
assert_eq!(response.status(), StatusCode::SEE_OTHER);
assert_ne!(
response
.headers()
.get("x-error-code")
.and_then(|value| value.to_str().ok()),
Some("convex_retired")
);
let location = response
.headers()
.get(header::LOCATION)
.and_then(|value| value.to_str().ok())
.unwrap_or_default();
assert!(location.starts_with("/?"));
assert!(location.contains("routeGuard=local_folder_source_required"));
assert!(location.contains("missingPage=local-md%3Adocs%7E2FPlan.md"));
}
#[tokio::test]
async fn document_shell_records_local_markdown_page_recent() {
let root = temp_root("mnote-document-shell-record-page-recent");
@@ -4490,7 +4557,7 @@ mod tests {
.headers()
.get("x-upstream-service")
.and_then(|value| value.to_str().ok()),
Some("convex-retired")
Some("legacy-cloud-retired")
);
let body = to_bytes(response.into_body(), usize::MAX)
.await