feat(rag): harden post-LightRAG runtime
Retire legacy OCR/media/evidence fallbacks, add local-folder event bus and Page Aggregate guards, and archive completed design checklists. Validation: cargo test -p mnote-web -- --test-threads=1; cargo test --workspace -- --test-threads=1; git diff --check; codegraph sync .; codegraph_status.
This commit is contained in:
@@ -1217,11 +1217,8 @@ pub(crate) fn refresh_local_search_index_for_path_with_settings(
|
||||
if local_ocr::is_local_ocr_sidecar_relative_path(&relative_path) {
|
||||
index.built_at = now_ms();
|
||||
write_local_search_index_json(root_path, &index)?;
|
||||
if included {
|
||||
refresh_evidence_sqlite_index_for_path(root_path, &index, &relative_path)?;
|
||||
} else {
|
||||
remove_evidence_sqlite_index_for_path(root_path, &index, &relative_path)?;
|
||||
}
|
||||
let _ = included;
|
||||
remove_evidence_sqlite_index_for_path(root_path, &index, &relative_path)?;
|
||||
return Ok(json!({
|
||||
"version": index.version,
|
||||
"rootUri": index.root_uri,
|
||||
@@ -1948,13 +1945,6 @@ fn index_relative_path_is_included(relative_path: &str, include_paths: &[String]
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn local_index_relative_path_is_included(
|
||||
relative_path: &str,
|
||||
settings: &LocalIndexSettings,
|
||||
) -> bool {
|
||||
index_relative_path_is_included(relative_path, &settings.include_paths)
|
||||
}
|
||||
|
||||
fn collect_markdown_documents(
|
||||
root_path: &Path,
|
||||
current: &Path,
|
||||
@@ -2613,84 +2603,6 @@ fn parsed_resource_id(resource: &LocalSearchResource) -> String {
|
||||
format!("{}#parse", resource.resource_id)
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
fn insert_ocr_evidence(
|
||||
connection: &Connection,
|
||||
root_path: &Path,
|
||||
index: &LocalSearchIndex,
|
||||
entry: &local_ocr::OcrIndexEntry,
|
||||
) -> Result<(), WebError> {
|
||||
if entry.status != "done" {
|
||||
return Ok(());
|
||||
}
|
||||
let ocr_path = root_path.join(&entry.ocr_root_relative_path);
|
||||
let Ok(markdown) = fs::read_to_string(&ocr_path) else {
|
||||
return Ok(());
|
||||
};
|
||||
let body = local_ocr::strip_ocr_frontmatter(&markdown);
|
||||
let source_map = path_source_map_path(&entry.ocr_root_relative_path).unwrap_or_default();
|
||||
let resource_id = format!(
|
||||
"{}#ocr:{}",
|
||||
entry.owner_document_id, entry.source_root_relative_path
|
||||
);
|
||||
let artifact = ocr_parsed_artifact(entry, &source_map);
|
||||
if !source_map.is_empty() {
|
||||
let source_map_path = root_path.join(&source_map);
|
||||
if let Ok(source_map_content) = fs::read_to_string(&source_map_path) {
|
||||
if let Ok(resource_source_map) =
|
||||
serde_json::from_str::<ResourceSourceMap>(&source_map_content)
|
||||
{
|
||||
return insert_source_map_artifact_evidence(
|
||||
connection,
|
||||
index,
|
||||
&resource_id,
|
||||
&artifact,
|
||||
&resource_source_map,
|
||||
body,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
insert_evidence_resource_from_artifact(connection, &resource_id, &artifact)?;
|
||||
let locator = json!({
|
||||
"schema": "mnote.evidence_locator.v1",
|
||||
"rootUri": index.root_uri,
|
||||
"ownerDocumentId": entry.owner_document_id,
|
||||
"ownerDocumentPath": entry.owner_document_path,
|
||||
"resourcePath": entry.source_root_relative_path,
|
||||
"resourceKind": evidence_resource_kind_for_path(&entry.source_root_relative_path),
|
||||
"sourceMapPath": source_map,
|
||||
"openAction": {
|
||||
"actionType": "mnote.open_resource_locator",
|
||||
"url": format!("/documents/{}?sourceKind=local_folder&rootUri={}", entry.owner_document_id, encode_query_component(&index.root_uri)),
|
||||
"params": {
|
||||
"resourcePath": entry.source_root_relative_path,
|
||||
"sourceMapPath": source_map
|
||||
}
|
||||
}
|
||||
});
|
||||
insert_evidence_block(connection, &resource_id, &resource_id, body, locator)
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
fn ocr_parsed_artifact(
|
||||
entry: &local_ocr::OcrIndexEntry,
|
||||
source_map_root_relative_path: &str,
|
||||
) -> ParsedResourceArtifact {
|
||||
ParsedResourceArtifact {
|
||||
schema: PARSED_RESOURCE_ARTIFACT_SCHEMA.into(),
|
||||
provider: entry.provider.clone(),
|
||||
model_version: Some(entry.model_version.clone()),
|
||||
owner_document_id: entry.owner_document_id.clone(),
|
||||
owner_document_path: entry.owner_document_path.clone(),
|
||||
source_root_relative_path: entry.source_root_relative_path.clone(),
|
||||
source_hash: format!("size:{}:mtime:{}", entry.source_size, entry.source_mtime_ms),
|
||||
artifact_root_relative_path: entry.ocr_root_relative_path.clone(),
|
||||
source_map_root_relative_path: source_map_root_relative_path.to_string(),
|
||||
updated_at_ms: entry.updated_at_ms as u64,
|
||||
}
|
||||
}
|
||||
|
||||
fn insert_source_map_artifact_evidence(
|
||||
connection: &Connection,
|
||||
index: &LocalSearchIndex,
|
||||
@@ -3479,35 +3391,6 @@ fn local_search_resource_matches(
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
fn local_search_ocr_matches(
|
||||
entry: &local_ocr::OcrIndexEntry,
|
||||
body: &str,
|
||||
query: &str,
|
||||
title_only: bool,
|
||||
exact: bool,
|
||||
) -> bool {
|
||||
if query.is_empty() {
|
||||
return false;
|
||||
}
|
||||
let haystack = if title_only {
|
||||
normalize_search_text(&format!(
|
||||
"{}\n{}",
|
||||
entry.source_root_relative_path, entry.ocr_root_relative_path
|
||||
))
|
||||
} else {
|
||||
normalize_search_text(&format!(
|
||||
"{}\n{}\n{}",
|
||||
entry.source_root_relative_path, entry.ocr_root_relative_path, body
|
||||
))
|
||||
};
|
||||
if exact {
|
||||
haystack.contains(query)
|
||||
} else {
|
||||
token_search_match(&haystack, query)
|
||||
}
|
||||
}
|
||||
|
||||
fn local_search_document_projection(
|
||||
document: &LocalSearchDocument,
|
||||
root_uri: &str,
|
||||
@@ -3557,43 +3440,6 @@ fn local_search_resource_projection(resource: &LocalSearchResource, root_uri: &s
|
||||
})
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
fn local_search_ocr_projection(
|
||||
entry: &local_ocr::OcrIndexEntry,
|
||||
body: &str,
|
||||
root_uri: &str,
|
||||
query: &str,
|
||||
) -> Value {
|
||||
let title = Path::new(&entry.owner_document_path)
|
||||
.file_stem()
|
||||
.and_then(|value| value.to_str())
|
||||
.unwrap_or("OCR")
|
||||
.to_string();
|
||||
json!({
|
||||
"id": format!("{}#ocr:{}", entry.owner_document_id, entry.source_root_relative_path),
|
||||
"documentId": entry.owner_document_id,
|
||||
"title": title,
|
||||
"path": entry.owner_document_path,
|
||||
"resourceType": "markdown",
|
||||
"sourceKind": "local_folder",
|
||||
"rootUri": root_uri,
|
||||
"hasOcr": true,
|
||||
"snippet": ocr_search_snippet(body, query),
|
||||
"ocrEvidence": {
|
||||
"sourceRootRelativePath": entry.source_root_relative_path,
|
||||
"ocrRootRelativePath": entry.ocr_root_relative_path,
|
||||
"provider": entry.provider,
|
||||
"status": entry.status,
|
||||
},
|
||||
"updatedAt": entry.updated_at_ms,
|
||||
"publicPath": format!(
|
||||
"/documents/{}?sourceKind=local_folder&rootUri={}",
|
||||
entry.owner_document_id,
|
||||
encode_query_component(root_uri),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn encode_query_component(value: &str) -> String {
|
||||
let mut encoded = String::with_capacity(value.len());
|
||||
for byte in value.as_bytes() {
|
||||
@@ -5143,6 +4989,10 @@ mod tests {
|
||||
.documents
|
||||
.iter()
|
||||
.any(|document| document.path == "docs/Page.ocr/photo.png-704905.ocr.md"));
|
||||
let refreshed_evidence = query_evidence_sqlite_results(&root, "OCR-hash-token", None, 10)
|
||||
.expect("query refreshed evidence")
|
||||
.unwrap_or_default();
|
||||
assert!(refreshed_evidence.is_empty());
|
||||
let _ = fs::remove_dir_all(&root);
|
||||
}
|
||||
|
||||
@@ -5157,6 +5007,8 @@ mod tests {
|
||||
"---\ntitle: Child\n---\n# Child\nOriginalToken body.\n",
|
||||
)
|
||||
.expect("write child");
|
||||
write_local_index_settings(&root, &[String::from(".")], None, None, None, None)
|
||||
.expect("settings");
|
||||
|
||||
let first_projection = query_local_search_index(
|
||||
&root,
|
||||
@@ -5421,7 +5273,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
#[cfg(unix)]
|
||||
fn evidence_index_parses_resource_body_with_liteparse_sidecar() {
|
||||
fn evidence_index_does_not_parse_resource_body_with_retired_liteparse_sidecar() {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
let _guard = env_lock().lock().expect("env lock");
|
||||
@@ -5479,23 +5331,11 @@ JSON
|
||||
let results = query_evidence_sqlite_results(&root, "ResourceBodyToken", None, 10)
|
||||
.expect("sqlite query")
|
||||
.expect("sqlite exists");
|
||||
let hit = results
|
||||
.iter()
|
||||
.find(|result| result.quote.contains("ResourceBodyToken"))
|
||||
.expect("parsed resource body hit");
|
||||
assert_eq!(
|
||||
hit.source.owner_document_id, "local-md:docs~2FPage.md",
|
||||
"资源正文证据应归属引用它的 owner Markdown"
|
||||
);
|
||||
assert_eq!(
|
||||
hit.source.resource_path.as_deref(),
|
||||
Some("docs/Page.assets/spec.pdf")
|
||||
);
|
||||
assert_eq!(hit.source.page, Some(2));
|
||||
assert!(hit.source.bbox.is_some());
|
||||
assert_eq!(
|
||||
hit.source.source_map_path.as_deref(),
|
||||
Some("docs/Page.ocr/spec.pdf.source-map.json")
|
||||
assert!(
|
||||
!results
|
||||
.iter()
|
||||
.any(|result| result.quote.contains("ResourceBodyToken")),
|
||||
"LiteParse resource body fallback is retired from active evidence indexing"
|
||||
);
|
||||
let resource_scoped = query_evidence_sqlite_results_with_mode(
|
||||
&root,
|
||||
@@ -5506,18 +5346,16 @@ JSON
|
||||
)
|
||||
.expect("resource scoped sqlite query")
|
||||
.expect("sqlite exists");
|
||||
assert_eq!(resource_scoped.len(), 1);
|
||||
assert_eq!(
|
||||
resource_scoped[0].source.resource_path.as_deref(),
|
||||
Some("docs/Page.assets/spec.pdf"),
|
||||
"页面内搜索打开资源时应只限制到当前资源,而不是它的 owner Markdown"
|
||||
assert!(
|
||||
resource_scoped.is_empty(),
|
||||
"retired LiteParse sidecar must not create resource-scoped evidence hits"
|
||||
);
|
||||
assert!(root
|
||||
assert!(!root
|
||||
.join("docs")
|
||||
.join("Page.ocr")
|
||||
.join("spec.pdf.parse.md")
|
||||
.exists());
|
||||
assert!(root
|
||||
assert!(!root
|
||||
.join("docs")
|
||||
.join("Page.ocr")
|
||||
.join("spec.pdf.source-map.json")
|
||||
|
||||
Reference in New Issue
Block a user