lix/mnote
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(rag): harden post-LightRAG runtime

Browse Source 
Retire legacy OCR/media/evidence fallbacks, add local-folder event bus and Page Aggregate guards, and archive completed design checklists.

Validation: cargo test -p mnote-web -- --test-threads=1; cargo test --workspace -- --test-threads=1; git diff --check; codegraph sync .; codegraph_status.
This commit is contained in:
lix-2026
2026-06-07 10:35:21 +08:00
parent 22a92edcda
commit 9551d4c1dc
59 changed files with 4053 additions and 5249 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rust/crates/mnote-web/src/routes/search.rs
+130 -307
View File
@@ -7,16 +7,15 @@ use crate::routes::query_support::{
resolve_effective_workspace_id,
};
use crate::routes::web_shell::load_sidebar_tree_html;
use crate::routes::{evidence, local_folder_source, local_search_index};
use crate::routes::{local_folder_source, local_search_index};
use crate::ssr::pages::search::SearchPage;
use axum::Json;
use axum::extract::{Extension, Query, State};
use axum::http::{HeaderMap, HeaderName, HeaderValue, StatusCode};
use axum::response::{Html, IntoResponse, Response};
use axum::Json;
use bridge_runtime::RuntimeQueryEnvelopeWire;
use core_protocol::{EvidenceSearchMode, EvidenceSearchResult};
use serde::Deserialize;
use serde_json::{Value, json};
use serde_json::{json, Value};
const HEADER_MNOTE_WEB_OWNER: &str = "x-mnote-web-owner";
const HEADER_QUERY_NAME: &str = "x-query-name";
@@ -183,100 +182,60 @@ pub async fn documents(
None
};
let (result, evidence_results) =
if body.source_kind.as_deref().map(str::trim) == Some("local_folder") {
let root_uri = body
.root_uri
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty())
.ok_or_else(|| {
WebError::bad_request_code("local_search_root_required", "本地搜索缺少 rootUri")
.with_context(&context)
})?;
let root_path = local_folder_source::ensure_local_workspace_read_access_with_state(
&state, &context, root_uri,
)
.map_err(|error| error.with_context(&context))?;
let user_settings = resolve_local_index_user_settings(
&state,
&context,
&effective_workspace_id,
&root_path,
)?;
let effective_settings = local_search_index::effective_local_index_settings_for_root(
state.control_plane(),
&effective_workspace_id,
&root_path,
)?;
let result = local_search_index::query_local_search_index_with_settings(
&root_path,
root_uri,
&effective_workspace_id,
&effective_settings,
&user_settings,
&normalized_query,
page_id.as_deref(),
body.limit.unwrap_or(30),
filters.title_only.unwrap_or(false),
filters.exact.unwrap_or(false),
filters.include_ocr.unwrap_or(false),
)?;
let evidence_results = evidence::evidence_results_from_local_search(
&result,
&root_path,
root_uri,
EvidenceSearchMode::Hybrid,
&normalized_query,
);
let limit = body.limit.unwrap_or(30).max(1) as usize;
let direct_evidence_results = if !filters.title_only.unwrap_or(false) {
local_search_index::query_evidence_sqlite_results_with_mode(
&root_path,
&normalized_query,
page_id.as_deref(),
body.limit.unwrap_or(30),
filters.exact.unwrap_or(false),
)?
.unwrap_or_default()
.into_iter()
.filter(|evidence| {
let path = evidence
.source
.resource_path
.as_deref()
.unwrap_or(evidence.source.owner_document_path.as_str());
local_search_index::local_index_relative_path_is_included(path, &user_settings)
})
.collect::<Vec<_>>()
} else {
Vec::new()
};
let (result, evidence_results) = merge_local_search_with_evidence_results(
result,
evidence_results,
direct_evidence_results,
limit,
);
(result, evidence_results)
} else {
let result = load_search_results_with_filters(
state.config(),
&context,
&effective_workspace_id,
&normalized_query,
page_id,
body.limit.unwrap_or(30),
filters,
)
.await?;
(result, Vec::new())
};
let result = if body.source_kind.as_deref().map(str::trim) == Some("local_folder") {
let root_uri = body
.root_uri
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty())
.ok_or_else(|| {
WebError::bad_request_code("local_search_root_required", "本地搜索缺少 rootUri")
.with_context(&context)
})?;
let root_path = local_folder_source::ensure_local_workspace_read_access_with_state(
&state, &context, root_uri,
)
.map_err(|error| error.with_context(&context))?;
let user_settings = resolve_local_index_user_settings(
&state,
&context,
&effective_workspace_id,
&root_path,
)?;
let effective_settings = local_search_index::effective_local_index_settings_for_root(
state.control_plane(),
&effective_workspace_id,
&root_path,
)?;
local_search_index::query_local_search_index_with_settings(
&root_path,
root_uri,
&effective_workspace_id,
&effective_settings,
&user_settings,
&normalized_query,
page_id.as_deref(),
body.limit.unwrap_or(30),
filters.title_only.unwrap_or(false),
filters.exact.unwrap_or(false),
filters.include_ocr.unwrap_or(false),
)?
} else {
load_search_results_with_filters(
state.config(),
&context,
&effective_workspace_id,
&normalized_query,
page_id,
body.limit.unwrap_or(30),
filters,
)
.await?
};
let results = result
.get("results")
.cloned()
.unwrap_or(Value::Array(vec![]));
let results = attach_evidence_to_search_results(results, &evidence_results);
let mut headers = HeaderMap::new();
stamp_search_headers(&mut headers);
@@ -285,13 +244,20 @@ pub async fn documents(
headers,
Json(json!({
"results": results,
"evidence": evidence_results,
"evidence": [],
"projectionOwner": result.get("projectionOwner").cloned().unwrap_or_else(|| json!("rust-kernel")),
"recent": result.get("recentChanges").cloned().unwrap_or_else(|| Value::Array(vec![])),
"meta": {
"owner": "mnote-web",
"projectionOwner": result.get("projectionOwner").cloned().unwrap_or_else(|| json!("rust-kernel")),
"queryName": "search.documents.query",
"boundary": {
"kind": "ordinary_local_search",
"knowledgeRag": false,
"evidenceSqliteFallback": false,
"liteParseFallback": false,
"ocrSidecarFallback": false
},
"degraded": result.get("degraded").cloned().unwrap_or_else(|| json!(false)),
"degradedReason": result.get("degradedReason").cloned().unwrap_or(Value::Null),
"requestId": context.trace.request_id,
@@ -475,158 +441,6 @@ pub async fn update_local_index_settings(
))
}
fn merge_local_search_with_evidence_results(
mut result: Value,
mut evidence_results: Vec<EvidenceSearchResult>,
direct_evidence_results: Vec<EvidenceSearchResult>,
limit: usize,
) -> (Value, Vec<EvidenceSearchResult>) {
if direct_evidence_results.is_empty() {
return (result, evidence_results);
}
let original_items = result
.get("results")
.and_then(Value::as_array)
.cloned()
.unwrap_or_default();
let original_evidence_results = std::mem::take(&mut evidence_results);
let mut result_items = Vec::new();
let mut merged_evidence_results = Vec::new();
let mut seen_result_ids = std::collections::HashSet::new();
let mut seen_evidence_ids = std::collections::HashSet::new();
let mut seen_paths = std::collections::HashSet::new();
for evidence in direct_evidence_results {
if result_items.len() >= limit || !seen_evidence_ids.insert(evidence.evidence_id.clone()) {
continue;
}
let item = search_result_from_evidence(&evidence);
if let Some(id) = item.get("id").and_then(Value::as_str) {
seen_result_ids.insert(id.to_string());
}
if let Some(path) = search_result_dedupe_path(&item) {
seen_paths.insert(path);
}
result_items.push(item);
merged_evidence_results.push(evidence);
}
for (index, item) in original_items.into_iter().enumerate() {
if result_items.len() >= limit {
break;
}
let item_id = item
.get("id")
.and_then(Value::as_str)
.map(ToOwned::to_owned)
.unwrap_or_default();
if !item_id.is_empty() && !seen_result_ids.insert(item_id) {
continue;
}
if let Some(path) = search_result_dedupe_path(&item) {
if !seen_paths.insert(path) {
continue;
}
}
if let Some(evidence) = original_evidence_results.get(index).cloned() {
merged_evidence_results.push(evidence);
}
result_items.push(item);
}
if let Some(map) = result.as_object_mut() {
map.insert("results".into(), Value::Array(result_items));
}
(result, merged_evidence_results)
}
fn search_result_dedupe_path(item: &Value) -> Option<String> {
let source_kind = item
.get("sourceKind")
.and_then(Value::as_str)
.unwrap_or_default();
if source_kind != "local_folder" {
return None;
}
item.get("path")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
}
fn search_result_from_evidence(evidence: &EvidenceSearchResult) -> Value {
let source = &evidence.source;
let evidence_value = serde_json::to_value(evidence).unwrap_or(Value::Null);
let resource_path = source
.resource_path
.as_deref()
.unwrap_or(source.owner_document_path.as_str());
let title = std::path::Path::new(resource_path)
.file_name()
.and_then(|value| value.to_str())
.unwrap_or(resource_path)
.to_string();
let resource_type = match source.resource_kind {
core_protocol::EvidenceResourceKind::Markdown => "markdown",
core_protocol::EvidenceResourceKind::Pdf => "pdf",
core_protocol::EvidenceResourceKind::Image => "image",
core_protocol::EvidenceResourceKind::Office => "office",
core_protocol::EvidenceResourceKind::Mindmap => "mindmap",
core_protocol::EvidenceResourceKind::RawFile => "resource",
};
json!({
"id": format!("evidence:{}", evidence.evidence_id),
"documentId": source.owner_document_id,
"title": title,
"path": source.owner_document_path,
"resourceType": resource_type,
"sourceKind": "local_folder",
"rootUri": source.root_uri,
"snippet": evidence.quote,
"score": evidence.score,
"matchInfo": evidence.match_info,
"publicPath": source.open_action.url,
"evidence": evidence_value,
"source": {
"locator": source
},
})
}
fn attach_evidence_to_search_results(
results: Value,
evidence_results: &[EvidenceSearchResult],
) -> Value {
let Value::Array(items) = results else {
return results;
};
Value::Array(
items
.into_iter()
.enumerate()
.map(|(index, item)| {
let Some(evidence) = evidence_results.get(index) else {
return item;
};
let mut item = item;
if let Some(map) = item.as_object_mut() {
if map.get("evidence").is_some() {
return item;
}
let evidence_value = serde_json::to_value(evidence).unwrap_or(Value::Null);
map.insert("evidence".into(), evidence_value);
let source = map.entry("source").or_insert_with(|| json!({}));
if let Some(source_map) = source.as_object_mut() {
source_map.insert(
"locator".into(),
serde_json::to_value(&evidence.source).unwrap_or(Value::Null),
);
}
}
item
})
.collect(),
)
}
fn resolve_local_index_user_settings(
state: &AppState,
context: &RequestContext,
@@ -940,12 +754,12 @@ fn stamp_search_headers(headers: &mut HeaderMap) {
#[cfg(test)]
mod tests {
use crate::app::{AppConfig, AppState, build_app};
use crate::app::{build_app, AppConfig, AppState};
use crate::routes::local_search_index;
use axum::body::{Body, to_bytes};
use axum::body::{to_bytes, Body};
use axum::http::{Request, StatusCode};
use control_plane::{DirectoryGrantInput, UpsertUserInput};
use serde_json::{Value, json};
use serde_json::{json, Value};
use std::fs;
use tower::util::ServiceExt;
@@ -1194,41 +1008,38 @@ mod tests {
assert_eq!(home["sourceKind"].as_str(), Some("local_folder"));
assert_eq!(home["resourceType"].as_str(), Some("markdown"));
assert_eq!(
home["source"]["locator"]["schema"].as_str(),
Some("mnote.evidence_locator.v1")
payload["meta"]["boundary"]["kind"].as_str(),
Some("ordinary_local_search")
);
assert_eq!(
home["evidence"]["source"]["ownerDocumentPath"].as_str(),
Some("README.md")
payload["meta"]["boundary"]["knowledgeRag"].as_bool(),
Some(false)
);
assert!(!payload["evidence"].as_array().expect("evidence").is_empty());
assert!(
home["tags"]
.as_array()
.unwrap()
.iter()
.any(|tag| tag.as_str() == Some("alpha"))
);
assert!(
home["backlinks"]
.as_array()
.unwrap()
.iter()
.any(|link| link.as_str() == Some("Daily"))
);
assert!(
home["resourceRefs"]
.as_array()
.unwrap()
.iter()
.any(|reference| reference.as_str() == Some("assets/spec.pdf"))
);
assert!(
root.join(".mnote")
.join("index")
.join("search-index.json")
.exists()
assert_eq!(
payload["meta"]["boundary"]["evidenceSqliteFallback"].as_bool(),
Some(false)
);
assert!(payload["evidence"].as_array().expect("evidence").is_empty());
assert!(home["tags"]
.as_array()
.unwrap()
.iter()
.any(|tag| tag.as_str() == Some("alpha")));
assert!(home["backlinks"]
.as_array()
.unwrap()
.iter()
.any(|link| link.as_str() == Some("Daily")));
assert!(home["resourceRefs"]
.as_array()
.unwrap()
.iter()
.any(|reference| reference.as_str() == Some("assets/spec.pdf")));
assert!(root
.join(".mnote")
.join("index")
.join("search-index.json")
.exists());
let evidence_db = root.join(".mnote").join("index").join("evidence.sqlite");
assert!(evidence_db.exists(), "evidence sqlite should be built");
let connection = rusqlite::Connection::open(&evidence_db).expect("open evidence sqlite");
@@ -1258,19 +1069,17 @@ mod tests {
locator["schema"].as_str(),
Some("mnote.evidence_locator.v1")
);
assert!(
payload["recent"]
.as_array()
.unwrap()
.iter()
.any(|item| item["documentId"].as_str() == Some("local-md:README.md"))
);
assert!(payload["recent"]
.as_array()
.unwrap()
.iter()
.any(|item| item["documentId"].as_str() == Some("local-md:README.md")));
let _ = fs::remove_dir_all(&root);
}
#[tokio::test]
async fn search_documents_local_folder_includes_evidence_sqlite_body_hits() {
async fn search_documents_local_folder_does_not_promote_evidence_sqlite_body_hits() {
let root = std::env::temp_dir().join(format!(
"mnote-local-search-evidence-route-{}",
std::process::id()
@@ -1350,20 +1159,18 @@ mod tests {
.expect("body");
let payload: Value = serde_json::from_slice(&body).expect("json");
let results = payload["results"].as_array().expect("results");
let hit = results
assert!(!results.iter().any(|item| item["id"]
.as_str()
.unwrap_or_default()
.starts_with("evidence:")));
assert!(results
.iter()
.find(|item| {
item["snippet"]
.as_str()
.unwrap_or("")
.contains("BodyOnlyEvidenceToken")
})
.expect("evidence sqlite body hit should be promoted to search result");
assert_eq!(hit["resourceType"].as_str(), Some("markdown"));
.any(|item| item["resourceType"].as_str() == Some("markdown")));
assert_eq!(
hit["evidence"]["source"]["schema"].as_str(),
Some("mnote.evidence_locator.v1")
payload["meta"]["boundary"]["evidenceSqliteFallback"].as_bool(),
Some(false)
);
assert!(payload["evidence"].as_array().expect("evidence").is_empty());
let _ = fs::remove_dir_all(&root);
}
@@ -1382,6 +1189,15 @@ mod tests {
.expect("manifest");
fs::write(root.join("README.md"), "# Refresh\nrefresh-token\n").expect("readme");
let root_uri = format!("file://{}", root.display());
local_search_index::write_local_index_settings(
&root,
&[String::from(".")],
None,
None,
None,
None,
)
.expect("settings");
let response = app()
.oneshot(
@@ -1582,6 +1398,15 @@ mod tests {
)
.expect("child");
let root_uri = format!("file://{}", root.display());
local_search_index::write_local_index_settings(
&root,
&[String::from(".")],
None,
None,
None,
None,
)
.expect("settings");
let encoded_root = query_escape(&root_uri);
let backlinks_response = app()
@@ -1607,13 +1432,11 @@ mod tests {
backlinks_payload["meta"]["queryName"].as_str(),
Some("search.local_index.backlinks")
);
assert!(
backlinks_payload["result"]["backlinks"]
.as_array()
.unwrap()
.iter()
.any(|item| item["documentId"].as_str() == Some("local-md:README.md"))
);
assert!(backlinks_payload["result"]["backlinks"]
.as_array()
.unwrap()
.iter()
.any(|item| item["documentId"].as_str() == Some("local-md:README.md")));
let tags_response = app()
.oneshot(