lix/mnote
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(rag): replace LiteParse flows with LightRAG provider

Browse Source
This commit is contained in:
lix-2026
2026-06-07 01:10:31 +08:00
parent f46c2fb5d0
commit 1f25364374
40 changed files with 7232 additions and 2350 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rust/crates/mnote-web/src/routes/local_search_index.rs
+137 -398
View File
@@ -1,7 +1,4 @@
use crate::error::WebError;
use crate::evidence_parse::{
ParseInput, ParseProviderMode, liteparse_runtime_available, parse_liteparse_input_blocking,
};
use crate::routes::local_folder_source::encode_local_id_segment;
use crate::routes::local_markdown_parser::{
parse_markdown_attachment_link, parse_markdown_page, split_frontmatter,
@@ -9,12 +6,12 @@ use crate::routes::local_markdown_parser::{
use crate::routes::local_ocr;
use control_plane::{ControlPlaneStore, UpsertUserInput, UpsertUserUiPreferenceInput};
use core_protocol::{
EvidenceLocator, EvidenceSearchMatchInfo, EvidenceSearchResult,
PARSED_RESOURCE_ARTIFACT_SCHEMA, ParsedResourceArtifact, ResourceSourceMap, SourceMapBlock,
EvidenceLocator, EvidenceSearchMatchInfo, EvidenceSearchResult, ParsedResourceArtifact,
ResourceSourceMap, SourceMapBlock, PARSED_RESOURCE_ARTIFACT_SCHEMA,
};
use rusqlite::{Connection, OptionalExtension, params};
use rusqlite::{params, Connection, OptionalExtension};
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use serde_json::{json, Value};
use std::collections::BTreeSet;
use std::fs;
use std::path::{Component, Path, PathBuf};
@@ -183,49 +180,7 @@ pub(crate) fn query_local_search_index_with_settings(
}
}
}
if include_ocr && results.len() < limit.max(1) as usize {
for entry in local_ocr::ocr_index_entries(root_path)? {
if !index_relative_path_is_included(
&entry.source_root_relative_path,
&result_settings.include_paths,
) {
continue;
}
if let Some(page_id) = page_id {
if let Some(resource_path) = page_resource_path.as_deref() {
if entry.source_root_relative_path != resource_path {
continue;
}
} else if entry.owner_document_id != page_id {
continue;
}
}
if entry.status != "done" {
continue;
}
let ocr_path = root_path.join(&entry.ocr_root_relative_path);
let markdown = match fs::read_to_string(&ocr_path) {
Ok(markdown) => markdown,
Err(_) => continue,
};
if local_ocr::parse_ocr_frontmatter(&markdown).is_none() {
continue;
}
let body = local_ocr::strip_ocr_frontmatter(&markdown);
if !local_search_ocr_matches(&entry, body, &normalized_query, title_only, exact) {
continue;
}
results.push(local_search_ocr_projection(
&entry,
body,
root_uri,
&normalized_query,
));
if results.len() >= limit.max(1) as usize {
break;
}
}
}
let _ = include_ocr;
Ok(json!({
"enqueueAssetIds": [],
"projectionOwner": "rust-kernel",
@@ -2244,14 +2199,6 @@ fn write_evidence_sqlite_index(root_path: &Path, index: &LocalSearchIndex) -> Re
for resource in &index.resources {
insert_resource_evidence(&tx, root_path, index, resource)?;
}
for entry in local_ocr::ocr_index_entries(root_path)?
.into_iter()
.filter(|entry| {
index_relative_path_is_included(&entry.source_root_relative_path, &index.indexed_paths)
})
{
insert_ocr_evidence(&tx, root_path, index, &entry)?;
}
for document in &index.documents {
insert_document_graph_edges(&tx, document)?;
}
@@ -2373,15 +2320,6 @@ fn refresh_evidence_sqlite_index_for_path(
{
insert_resource_evidence(&tx, root_path, index, resource)?;
}
for entry in local_ocr::ocr_index_entries(root_path)?
.into_iter()
.filter(|entry| {
entry.ocr_root_relative_path == relative_path
|| entry.source_root_relative_path == relative_path
})
{
insert_ocr_evidence(&tx, root_path, index, &entry)?;
}
tx.execute(
"INSERT OR REPLACE INTO evidence_meta(key, value) VALUES('schema_version', ?1)",
params![EVIDENCE_SQLITE_SCHEMA_VERSION.to_string()],
@@ -2664,147 +2602,18 @@ fn insert_resource_evidence(
}
fn parse_resource_evidence_artifact(
root_path: &Path,
index: &LocalSearchIndex,
resource: &LocalSearchResource,
_root_path: &Path,
_index: &LocalSearchIndex,
_resource: &LocalSearchResource,
) -> Result<Option<crate::evidence_parse::ParseProviderOutput>, WebError> {
if !resource_parse_supported(resource) || !liteparse_runtime_available() {
return Ok(None);
}
let owner = evidence_owner_for_resource(index, resource);
let mut output = match parse_liteparse_input_blocking(ParseInput {
root_path: root_path.to_path_buf(),
root_uri: index.root_uri.clone(),
owner_document_id: owner.document_id.clone(),
owner_document_path: owner.path.clone(),
source_root_relative_path: resource.path.clone(),
mode: ParseProviderMode::NoOcr,
}) {
Ok(output) => output,
Err(_) => return Ok(None),
};
let sidecar = resource_parse_sidecar_paths(&owner.path, &resource.path);
write_parsed_resource_sidecars(root_path, &sidecar, &output.markdown, &output.source_map)?;
output.artifact.artifact_root_relative_path = sidecar.parse_root_relative_path;
output.artifact.source_map_root_relative_path = sidecar.source_map_root_relative_path.clone();
output.source_map.owner_document_path = owner.path;
Ok(Some(output))
}
fn resource_parse_supported(resource: &LocalSearchResource) -> bool {
matches!(resource.resource_type.as_str(), "pdf" | "office")
Ok(None)
}
fn parsed_resource_id(resource: &LocalSearchResource) -> String {
format!("{}#parse", resource.resource_id)
}
#[derive(Debug, Clone)]
struct EvidenceResourceOwner {
document_id: String,
path: String,
}
fn evidence_owner_for_resource(
index: &LocalSearchIndex,
resource: &LocalSearchResource,
) -> EvidenceResourceOwner {
if let Some(document) = index.documents.iter().find(|document| {
document.resource_refs.iter().any(|reference| {
normalize_local_reference_path(&document.path, reference) == resource.path
})
}) {
return EvidenceResourceOwner {
document_id: document.document_id.clone(),
path: document.path.clone(),
};
}
EvidenceResourceOwner {
document_id: resource.resource_id.clone(),
path: resource.path.clone(),
}
}
#[derive(Debug, Clone)]
struct ResourceParseSidecarPaths {
parse_root_relative_path: String,
source_map_root_relative_path: String,
}
fn resource_parse_sidecar_paths(
owner_document_path: &str,
source_root_relative_path: &str,
) -> ResourceParseSidecarPaths {
let owner_parent = Path::new(owner_document_path)
.parent()
.map(Path::to_path_buf)
.unwrap_or_default();
let owner_stem = Path::new(owner_document_path)
.file_stem()
.and_then(|value| value.to_str())
.unwrap_or("Page");
let source_leaf = Path::new(source_root_relative_path)
.file_name()
.and_then(|value| value.to_str())
.unwrap_or("source");
let sidecar_dir = owner_parent.join(format!("{owner_stem}.ocr"));
let parse_root_relative_path = sidecar_dir
.join(format!("{source_leaf}.parse.md"))
.to_string_lossy()
.replace('\\', "/");
let source_map_root_relative_path = sidecar_dir
.join(format!("{source_leaf}.source-map.json"))
.to_string_lossy()
.replace('\\', "/");
ResourceParseSidecarPaths {
parse_root_relative_path,
source_map_root_relative_path,
}
}
fn write_parsed_resource_sidecars(
root_path: &Path,
sidecar: &ResourceParseSidecarPaths,
markdown: &str,
source_map: &ResourceSourceMap,
) -> Result<(), WebError> {
let parse_path = root_path.join(&sidecar.parse_root_relative_path);
let source_map_path = root_path.join(&sidecar.source_map_root_relative_path);
for path in [&parse_path, &source_map_path] {
if let Some(parent) = path.parent() {
fs::create_dir_all(parent).map_err(|error| {
WebError::bad_request_code(
"evidence_parse_sidecar_create_failed",
format!(
"无法创建证据解析 sidecar 目录 {}: {error}",
parent.display()
),
)
})?;
}
}
fs::write(&parse_path, markdown.trim_end()).map_err(|error| {
WebError::bad_request_code(
"evidence_parse_sidecar_write_failed",
format!(
"无法写入证据解析 Markdown {}: {error}",
parse_path.display()
),
)
})?;
let source_map_json = serde_json::to_string_pretty(source_map)
.map_err(|error| WebError::internal(format!("证据 source-map 序列化失败: {error}")))?;
fs::write(&source_map_path, format!("{source_map_json}\n")).map_err(|error| {
WebError::bad_request_code(
"evidence_parse_sidecar_write_failed",
format!(
"无法写入证据 source-map {}: {error}",
source_map_path.display()
),
)
})
}
#[allow(dead_code)]
fn insert_ocr_evidence(
connection: &Connection,
root_path: &Path,
@@ -2863,6 +2672,7 @@ fn insert_ocr_evidence(
insert_evidence_block(connection, &resource_id, &resource_id, body, locator)
}
#[allow(dead_code)]
fn ocr_parsed_artifact(
entry: &local_ocr::OcrIndexEntry,
source_map_root_relative_path: &str,
@@ -3461,12 +3271,14 @@ fn count_evidence_blocks(path: &Path) -> Result<u64, WebError> {
.map_err(sqlite_error)
}
#[allow(dead_code)]
#[derive(Debug, Clone, Default)]
pub(crate) struct LocalEvidenceSourceStatuses {
pub(crate) indexed_paths: BTreeSet<String>,
pub(crate) failed_paths: BTreeSet<String>,
}
#[allow(dead_code)]
pub(crate) fn local_evidence_source_statuses(
root_path: &Path,
) -> Result<LocalEvidenceSourceStatuses, WebError> {
@@ -3509,21 +3321,6 @@ pub(crate) fn local_evidence_source_statuses(
}
}
}
for entry in local_ocr::ocr_index_entries(root_path)? {
match entry.status.as_str() {
"done" => {
statuses
.indexed_paths
.insert(entry.source_root_relative_path);
}
"failed" | "interrupted" => {
statuses
.failed_paths
.insert(entry.source_root_relative_path);
}
_ => {}
}
}
Ok(statuses)
}
@@ -3563,6 +3360,7 @@ fn local_resource_path_from_document_id(document_id: &str) -> Option<String> {
.filter(|value| !value.trim().is_empty())
}
#[allow(dead_code)]
fn path_source_map_path(path: &str) -> Option<String> {
if let Some(stripped) = path.strip_suffix(".ocr.md") {
return Some(format!("{stripped}.source-map.json"));
@@ -3681,6 +3479,7 @@ fn local_search_resource_matches(
}
}
#[allow(dead_code)]
fn local_search_ocr_matches(
entry: &local_ocr::OcrIndexEntry,
body: &str,
@@ -3758,6 +3557,7 @@ fn local_search_resource_projection(resource: &LocalSearchResource, root_uri: &s
})
}
#[allow(dead_code)]
fn local_search_ocr_projection(
entry: &local_ocr::OcrIndexEntry,
body: &str,
@@ -4375,55 +4175,41 @@ mod tests {
.iter()
.find(|item| item["documentId"].as_str() == Some("local-md:README.md"))
.expect("home result");
assert!(
home["tags"]
.as_array()
.unwrap()
.iter()
.any(|tag| tag.as_str() == Some("alpha"))
);
assert!(
home["backlinks"]
.as_array()
.unwrap()
.iter()
.any(|link| link.as_str() == Some("Daily"))
);
assert!(
home["backlinks"]
.as_array()
.unwrap()
.iter()
.any(|link| link.as_str() == Some("docs/child.md"))
);
assert!(
home["resourceRefs"]
.as_array()
.unwrap()
.iter()
.any(|reference| reference.as_str() == Some("assets/spec.pdf"))
);
assert!(
home["resourceRefs"]
.as_array()
.unwrap()
.iter()
.any(|reference| reference.as_str() == Some("maps/idea.mindmap.json"))
);
assert!(
home["resourceRefs"]
.as_array()
.unwrap()
.iter()
.any(|reference| reference.as_str() == Some("office/report.xlsx"))
);
assert!(
home["publicPath"]
.as_str()
.is_some_and(|path| path.starts_with(
"/documents/local-md:README.md?sourceKind=local_folder&rootUri=file%3A%2F%2F"
))
);
assert!(home["tags"]
.as_array()
.unwrap()
.iter()
.any(|tag| tag.as_str() == Some("alpha")));
assert!(home["backlinks"]
.as_array()
.unwrap()
.iter()
.any(|link| link.as_str() == Some("Daily")));
assert!(home["backlinks"]
.as_array()
.unwrap()
.iter()
.any(|link| link.as_str() == Some("docs/child.md")));
assert!(home["resourceRefs"]
.as_array()
.unwrap()
.iter()
.any(|reference| reference.as_str() == Some("assets/spec.pdf")));
assert!(home["resourceRefs"]
.as_array()
.unwrap()
.iter()
.any(|reference| reference.as_str() == Some("maps/idea.mindmap.json")));
assert!(home["resourceRefs"]
.as_array()
.unwrap()
.iter()
.any(|reference| reference.as_str() == Some("office/report.xlsx")));
assert!(home["publicPath"]
.as_str()
.is_some_and(|path| path.starts_with(
"/documents/local-md:README.md?sourceKind=local_folder&rootUri=file%3A%2F%2F"
)));
let evidence_db = root.join(".mnote").join("index").join("evidence.sqlite");
let connection = Connection::open(&evidence_db).expect("open evidence sqlite");
let markdown_edge_count: i64 = connection
@@ -4479,12 +4265,11 @@ mod tests {
Some("local-mdid:child-page")
);
assert!(
root.join(".mnote")
.join("index")
.join("search-index.json")
.exists()
);
assert!(root
.join(".mnote")
.join("index")
.join("search-index.json")
.exists());
let mindmap_projection = query_local_search_index(
&root,
&format!("file://{}", root.display()),
@@ -4497,19 +4282,19 @@ mod tests {
false,
)
.expect("mindmap projection");
assert!(
mindmap_projection["results"]
.as_array()
.unwrap()
.iter()
.any(|item| item["resourceType"].as_str() == Some("mindmap")
&& item["path"].as_str() == Some("maps/idea.mindmap.json")
&& item["publicPath"].as_str().is_some_and(|path| path
assert!(mindmap_projection["results"]
.as_array()
.unwrap()
.iter()
.any(|item| item["resourceType"].as_str() == Some("mindmap")
&& item["path"].as_str() == Some("maps/idea.mindmap.json")
&& item["publicPath"]
.as_str()
.is_some_and(|path| path
.starts_with("/?treeView=filetree&sourceKind=local_folder&rootUri="))
&& item["publicPath"]
.as_str()
.is_some_and(|path| !path.starts_with("/tree?")))
);
&& item["publicPath"]
.as_str()
.is_some_and(|path| !path.starts_with("/tree?"))));
let office_projection = query_local_search_index(
&root,
&format!("file://{}", root.display()),
@@ -4522,14 +4307,12 @@ mod tests {
false,
)
.expect("office projection");
assert!(
office_projection["results"]
.as_array()
.unwrap()
.iter()
.any(|item| item["resourceType"].as_str() == Some("office")
&& item["path"].as_str() == Some("office/report.xlsx"))
);
assert!(office_projection["results"]
.as_array()
.unwrap()
.iter()
.any(|item| item["resourceType"].as_str() == Some("office")
&& item["path"].as_str() == Some("office/report.xlsx")));
let pdf_projection = query_local_search_index(
&root,
&format!("file://{}", root.display()),
@@ -4542,14 +4325,12 @@ mod tests {
false,
)
.expect("pdf projection");
assert!(
pdf_projection["results"]
.as_array()
.unwrap()
.iter()
.any(|item| item["resourceType"].as_str() == Some("pdf")
&& item["path"].as_str() == Some("assets/spec.pdf"))
);
assert!(pdf_projection["results"]
.as_array()
.unwrap()
.iter()
.any(|item| item["resourceType"].as_str() == Some("pdf")
&& item["path"].as_str() == Some("assets/spec.pdf")));
let image_projection = query_local_search_index(
&root,
&format!("file://{}", root.display()),
@@ -4562,14 +4343,12 @@ mod tests {
false,
)
.expect("image projection");
assert!(
image_projection["results"]
.as_array()
.unwrap()
.iter()
.any(|item| item["resourceType"].as_str() == Some("image")
&& item["path"].as_str() == Some("assets/diagram.png"))
);
assert!(image_projection["results"]
.as_array()
.unwrap()
.iter()
.any(|item| item["resourceType"].as_str() == Some("image")
&& item["path"].as_str() == Some("assets/diagram.png")));
let _ = fs::remove_dir_all(&root);
}
@@ -5194,12 +4973,10 @@ mod tests {
let index = read_local_search_index(&root)
.expect("read index")
.expect("index exists");
assert!(
index
.documents
.iter()
.any(|document| document.path == "README.md")
);
assert!(index
.documents
.iter()
.any(|document| document.path == "README.md"));
let child = index
.documents
.iter()
@@ -5227,18 +5004,14 @@ mod tests {
let index = read_local_search_index(&root)
.expect("read index")
.expect("index exists");
assert!(
!index
.documents
.iter()
.any(|document| document.path == "docs/child.md")
);
assert!(
index
.documents
.iter()
.any(|document| document.path == "README.md")
);
assert!(!index
.documents
.iter()
.any(|document| document.path == "docs/child.md"));
assert!(index
.documents
.iter()
.any(|document| document.path == "README.md"));
let removed_evidence = query_evidence_sqlite_results(&root, "ChangedToken", None, 10)
.expect("removed evidence")
.expect("sqlite exists");
@@ -5251,7 +5024,7 @@ mod tests {
}
#[test]
fn local_search_ocr_sidecar_requires_include_ocr_and_returns_owner_page() {
fn local_search_ocr_sidecar_is_hidden_after_lightrag_retirement() {
let root = temp_root("mnote-local-search-ocr");
let root_uri = format!("file://{}", root.display());
let workspace_id = "local-ws-ocr";
@@ -5344,38 +5117,18 @@ mod tests {
true,
)
.expect("with ocr");
let result = with_ocr["results"]
.as_array()
.and_then(|items| items.first())
.expect("ocr result");
assert_eq!(
result["documentId"].as_str(),
Some("local-md:docs~2FPage.md")
);
assert_eq!(result["hasOcr"].as_bool(), Some(true));
assert_eq!(
result["ocrEvidence"]["sourceRootRelativePath"].as_str(),
Some("docs/Page.assets/photo.png")
);
assert_eq!(
result["ocrEvidence"]["ocrRootRelativePath"].as_str(),
Some("docs/Page.ocr/photo.png.ocr.md")
);
assert_eq!(with_ocr["results"].as_array().map(Vec::len), Some(0));
let index = read_local_search_index(&root)
.expect("read search index")
.expect("search index");
assert!(
!index
.documents
.iter()
.any(|document| document.path == "docs/Page.ocr/photo.png.ocr.md")
);
assert!(
!index
.documents
.iter()
.any(|document| document.path == "docs/Page.ocr/photo.png-704905.ocr.md")
);
assert!(!index
.documents
.iter()
.any(|document| document.path == "docs/Page.ocr/photo.png.ocr.md"));
assert!(!index
.documents
.iter()
.any(|document| document.path == "docs/Page.ocr/photo.png-704905.ocr.md"));
refresh_local_search_index_for_path(
&root,
&root_uri,
@@ -5386,12 +5139,10 @@ mod tests {
let refreshed_index = read_local_search_index(&root)
.expect("read refreshed search index")
.expect("refreshed search index");
assert!(
!refreshed_index
.documents
.iter()
.any(|document| document.path == "docs/Page.ocr/photo.png-704905.ocr.md")
);
assert!(!refreshed_index
.documents
.iter()
.any(|document| document.path == "docs/Page.ocr/photo.png-704905.ocr.md"));
let _ = fs::remove_dir_all(&root);
}
@@ -5492,21 +5243,15 @@ mod tests {
.expect("sqlite query")
.expect("sqlite exists");
assert!(results.len() >= 2);
assert!(
results
.iter()
.any(|result| result.source.owner_document_path == "README.md")
);
assert!(
results
.iter()
.any(|result| result.source.owner_document_path == "docs/child.md")
);
assert!(
results
.iter()
.all(|result| result.source.schema == "mnote.evidence_locator.v1")
);
assert!(results
.iter()
.any(|result| result.source.owner_document_path == "README.md"));
assert!(results
.iter()
.any(|result| result.source.owner_document_path == "docs/child.md"));
assert!(results
.iter()
.all(|result| result.source.schema == "mnote.evidence_locator.v1"));
let home_result = results
.iter()
.find(|result| result.source.owner_document_path == "README.md")
@@ -5664,16 +5409,12 @@ mod tests {
.expect("sqlite exists");
assert_eq!(context.len(), 2);
assert!(
context
.iter()
.all(|item| item.source.owner_document_path == "README.md")
);
assert!(
context
.iter()
.any(|item| item.quote.contains("ReadContextToken"))
);
assert!(context
.iter()
.all(|item| item.source.owner_document_path == "README.md"));
assert!(context
.iter()
.any(|item| item.quote.contains("ReadContextToken")));
let _ = fs::remove_dir_all(&root);
}
@@ -5771,18 +5512,16 @@ JSON
Some("docs/Page.assets/spec.pdf"),
"页面内搜索打开资源时应只限制到当前资源，而不是它的 owner Markdown"
);
assert!(
root.join("docs")
.join("Page.ocr")
.join("spec.pdf.parse.md")
.exists()
);
assert!(
root.join("docs")
.join("Page.ocr")
.join("spec.pdf.source-map.json")
.exists()
);
assert!(root
.join("docs")
.join("Page.ocr")
.join("spec.pdf.parse.md")
.exists());
assert!(root
.join("docs")
.join("Page.ocr")
.join("spec.pdf.source-map.json")
.exists());
let _ = fs::remove_dir_all(&root);
}