2026-06-07 01:10:31 +08:00
use crate ::app ::AppState ;
use crate ::context ::RequestContext ;
use crate ::error ::WebError ;
use crate ::routes ::evidence ::{ citation_markdown_for_locator , citation_url_for_locator };
use crate ::routes ::local_folder_source ;
use axum ::extract ::{ Extension , Json , Query , State };
use axum ::http ::StatusCode ;
use core_protocol ::evidence ::{
EvidenceBBox , EvidenceLocator , EvidenceOpenAction , EvidenceResourceKind ,
};
use serde ::{ Deserialize , Serialize };
use serde_json ::{ json , Value };
use std ::collections ::hash_map ::DefaultHasher ;
use std ::collections ::{ BTreeMap , BTreeSet };
use std ::env ;
use std ::fs ;
use std ::hash ::{ Hash , Hasher };
use std ::path ::{ Path , PathBuf };
use std ::time ::{ Duration , SystemTime , UNIX_EPOCH };
const REGISTRY_SCHEMA : & str = "mnote.knowledge_rag.source_registry.v1" ;
const REFERENCE_SCHEMA : & str = "mnote.knowledge_rag.reference.v1" ;
const DEFAULT_LIGHTRAG_ENDPOINT : & str = "http://127.0.0.1:9621" ;
const DEFAULT_LIGHTRAG_INPUT_DIR : & str = "/mnt/Data1T/Mnote_data/lightrag/inputs" ;
const DEFAULT_LIGHTRAG_WORKING_DIR : & str = "/mnt/Data1T/Mnote_data/lightrag/rag_storage" ;
const MAX_INGEST_SOURCES_PER_REQUEST : usize = 200 ;
2026-06-07 10:35:21 +08:00
const SOURCE_SCOPE_MODE_POST_FILTER : & str = "post_filter_mapped_references" ;
2026-06-07 01:10:31 +08:00
const KNOWLEDGE_RAG_SOURCE_EXTENSIONS : & [ & str ] = & [
"md" , "markdown" , "txt" , "pdf" , "doc" , "docx" , "ppt" , "pptx" , "xls" , "xlsx" , "csv" , "png" ,
"jpg" , "jpeg" , "webp" , "gif" , "bmp" , "tif" , "tiff" ,
];
const LIGHTRAG_SCAN_SOURCE_EXTENSIONS : & [ & str ] = & [
"md" , "markdown" , "mdx" , "txt" , "pdf" , "docx" , "pptx" , "xlsx" , "rtf" , "odt" , "tex" , "epub" ,
"html" , "htm" ,
];
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase" )]
pub struct KnowledgeRagStatusQuery {
pub workspace_id : Option < String > ,
pub root_uri : Option < String > ,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase" )]
pub struct KnowledgeRagIngestRequest {
pub workspace_id : Option < String > ,
pub root_uri : String ,
pub sources : Vec < KnowledgeRagSourceInput > ,
pub force : Option < bool > ,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase" )]
pub struct KnowledgeRagSourceInput {
pub source_path : Option < String > ,
pub path : Option < String > ,
pub parser_hint : Option < String > ,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase" )]
pub struct KnowledgeRagQueryRequest {
pub workspace_id : Option < String > ,
pub root_uri : String ,
pub question : Option < String > ,
pub query : Option < String > ,
pub mode : Option < String > ,
pub top_k : Option < u32 > ,
pub chunk_top_k : Option < u32 > ,
pub include_chunk_content : Option < bool > ,
pub source_paths : Option < Vec < String >> ,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase" )]
pub struct KnowledgeRagOpenReferenceRequest {
pub workspace_id : Option < String > ,
pub root_uri : String ,
pub reference : Option < Value > ,
pub reference_id : Option < String > ,
pub file_path : Option < String > ,
pub chunk_id : Option < String > ,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase" )]
pub struct KnowledgeRagDeleteSourceRequest {
pub workspace_id : Option < String > ,
pub root_uri : String ,
pub source_path : Option < String > ,
pub light_rag_doc_id : Option < String > ,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase" )]
pub struct KnowledgeRagPruneRegistryRequest {
pub workspace_id : Option < String > ,
pub root_uri : String ,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(rename_all = "camelCase" )]
struct KnowledgeRagSourceRegistry {
schema : String ,
workspace_id : String ,
root_uri : String ,
updated_at_ms : u128 ,
entries : Vec < KnowledgeRagSourceRegistryEntry > ,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase" )]
struct KnowledgeRagSourceRegistryEntry {
source_id : String ,
workspace_id : String ,
root_uri : String ,
source_path : String ,
source_root_relative_path : String ,
source_hash : String ,
light_rag_doc_id : Option < String > ,
#[serde(default, skip_serializing_if = "Option::is_none" )]
light_rag_status : Option < String > ,
light_rag_file_path : String ,
symlink_path : String ,
parser_hint : Option < String > ,
indexed_at_ms : Option < u128 > ,
deleted_at_ms : Option < u128 > ,
stale : bool ,
updated_at_ms : u128 ,
}
#[derive(Debug, Clone, Default)]
pub ( crate ) struct KnowledgeRagSourceStatuses {
pub ( crate ) indexed_paths : BTreeSet < String > ,
pub ( crate ) indexing_paths : BTreeSet < String > ,
pub ( crate ) failed_paths : BTreeSet < String > ,
}
pub ( crate ) fn knowledge_rag_source_statuses (
root_path : & Path ,
workspace_id : & str ,
root_uri : & str ,
) -> Result < KnowledgeRagSourceStatuses , WebError > {
let registry = read_registry ( root_path , workspace_id , root_uri ) ? ;
let mut statuses = KnowledgeRagSourceStatuses ::default ();
for entry in registry . entries {
let path = entry . source_root_relative_path . trim ();
if path . is_empty () {
continue ;
}
let provider_status = entry . light_rag_status . as_deref (). unwrap_or_default ();
if provider_status == "delete_completed" {
continue ;
}
if provider_status == "failed" {
statuses . failed_paths . insert ( path . to_string ());
2026-06-07 10:35:21 +08:00
} else if provider_status == "delete_retry_required" {
statuses . failed_paths . insert ( path . to_string ());
2026-06-07 01:10:31 +08:00
} else if provider_status == "delete_submitted"
|| ( entry . deleted_at_ms . is_some () && entry . light_rag_doc_id . is_some ())
{
statuses . indexing_paths . insert ( path . to_string ());
} else if entry . stale {
statuses . failed_paths . insert ( path . to_string ());
} else if entry . deleted_at_ms . is_some () && entry . light_rag_doc_id . is_none () {
continue ;
} else if entry . indexed_at_ms . is_some () && entry . light_rag_doc_id . is_some () {
statuses . indexed_paths . insert ( path . to_string ());
} else {
statuses . indexing_paths . insert ( path . to_string ());
}
}
Ok ( statuses )
}
pub async fn retired_local_ocr_endpoint (
Extension ( context ) : Extension < RequestContext > ,
) -> ( StatusCode , Json < Value > ) {
(
StatusCode ::GONE ,
Json ( json! ({
"ok" : false ,
"code" : "mnote_local_ocr_retired" ,
"message" : "本地 OCR sidecar 已退役;图片、PDF、Office 与索引统一交给 LightRAG 资料库处理。" ,
"replacement" : {
"provider" : "lightrag" ,
"status" : "/api/knowledge-rag/status" ,
"ingest" : "/api/knowledge-rag/ingest" ,
"deleteSource" : "/api/knowledge-rag/delete-source"
},
"requestId" : context . trace . request_id ,
"traceId" : context . trace . trace_id ,
})),
)
}
pub async fn status (
State ( state ) : State < AppState > ,
Extension ( context ) : Extension < RequestContext > ,
Query ( query ) : Query < KnowledgeRagStatusQuery > ,
) -> Result < Json < Value > , WebError > {
let registry =
if let Some ( root_uri ) = query . root_uri . as_deref (). filter ( | value | ! value . is_empty ()) {
let root_path = local_folder_source ::ensure_local_workspace_read_access_with_state (
& state , & context , root_uri ,
)
. map_err ( | error | error . with_context ( & context )) ? ;
let workspace_id = effective_workspace_id ( query . workspace_id . as_deref (), root_uri );
let mut registry = read_registry ( & root_path , & workspace_id , root_uri ) ? ;
sync_registry_with_documents ( & root_path , & mut registry , & context ). await ? ;
Some ( registry )
} else {
None
};
let documents =
match lightrag_json ( reqwest ::Method ::GET , "/documents" , None , true , & context ). await {
Ok ( value ) => json! ({
"ok" : true ,
"rawStatusGroups" : lightrag_document_status_group_counts ( & value ),
"documents" : lightrag_document_summaries ( & value ),
}),
Err ( error ) => json! ({
"ok" : false ,
"code" : error . code (),
"message" : error . message (),
"rawStatusGroups" : {},
"documents" : [],
}),
};
let endpoint = lightrag_endpoint ();
let health = match lightrag_json ( reqwest ::Method ::GET , "/health" , None , false , & context ). await {
Ok ( value ) => json! ({
"ok" : true ,
"health" : value ,
}),
Err ( error ) => json! ({
"ok" : false ,
"code" : error . code (),
"message" : error . message (),
}),
};
Ok ( Json ( json! ({
"ok" : true ,
"schema" : "mnote.knowledge_rag.provider_status.v1" ,
"provider" : "lightrag" ,
"endpoint" : endpoint ,
"dashboardUrl" : lightrag_dashboard_url (),
"inputDir" : lightrag_input_dir (). display (). to_string (),
"health" : health ,
"documents" : documents ,
"registry" : registry ,
})))
}
pub ( crate ) async fn sync_registry_for_root (
state : & AppState ,
context : & RequestContext ,
root_uri : & str ,
workspace_id : Option <& str > ,
) -> Result < (), WebError > {
let root_path = local_folder_source ::ensure_local_workspace_read_access_with_state (
state , context , root_uri ,
)
. map_err ( | error | error . with_context ( context )) ? ;
let workspace_id = effective_workspace_id ( workspace_id , root_uri );
let mut registry = read_registry ( & root_path , & workspace_id , root_uri ) ? ;
sync_registry_with_documents ( & root_path , & mut registry , context ). await
}
pub async fn ingest (
State ( state ) : State < AppState > ,
Extension ( context ) : Extension < RequestContext > ,
Json ( body ) : Json < KnowledgeRagIngestRequest > ,
) -> Result < Json < Value > , WebError > {
let root_path = local_folder_source ::ensure_local_workspace_read_access_with_state (
& state ,
& context ,
& body . root_uri ,
)
. map_err ( | error | error . with_context ( & context )) ? ;
let workspace_id = effective_workspace_id ( body . workspace_id . as_deref (), & body . root_uri );
let mut registry = read_registry ( & root_path , & workspace_id , & body . root_uri ) ? ;
sync_registry_with_documents ( & root_path , & mut registry , & context ). await ? ;
let input_dir = lightrag_input_dir ();
fs ::create_dir_all ( & input_dir ). map_err ( | error | {
WebError ::bad_request_code (
"knowledge_rag_input_dir_unavailable" ,
format! ( "无法创建 LightRAG input 目录: {error} " ),
)
. with_context ( & context )
}) ? ;
let mut configured = Vec ::new ();
let mut expanded_count = 0 usize ;
let mut seen_sources = BTreeSet ::< String > ::new ();
let force = body . force . unwrap_or ( false );
for source in body . sources {
let source_path = source . source_path . or ( source . path ). ok_or_else ( || {
WebError ::bad_request_code ( "knowledge_rag_source_required" , "资料库索引缺少 sourcePath" )
. with_context ( & context )
}) ? ;
let parser_hint = normalize_parser_hint ( source . parser_hint . as_deref (), & context ) ? ;
let resolved_sources = resolve_knowledge_rag_sources ( & root_path , & source_path , & context ) ? ;
for resolved in resolved_sources {
expanded_count += 1 ;
if expanded_count > MAX_INGEST_SOURCES_PER_REQUEST {
return Err ( WebError ::bad_request_code (
"knowledge_rag_source_limit_exceeded" ,
format! (
"单次资料库索引最多支持 {MAX_INGEST_SOURCES_PER_REQUEST} 个文件,请缩小目录范围"
),
)
. with_context ( & context ));
}
let canonical = resolved . canonical_path ;
let canonical_key = canonical . display (). to_string ();
if ! seen_sources . insert ( canonical_key . clone ()) {
configured . push ( json! ({
"sourcePath" : canonical_key ,
"requestedSourcePath" : resolved . requested_path ,
"sourceKind" : resolved . source_kind ,
"skipped" : true ,
"skipReason" : "duplicate_request" ,
}));
continue ;
}
let relative = root_relative_path ( & root_path , & canonical ) ? ;
let source_hash = source_hash ( & canonical ) ? ;
let file_name = canonical
. file_name ()
. and_then ( | value | value . to_str ())
. unwrap_or ( "source" );
let direct_scan_source = lightrag_scan_supported_file ( & canonical );
if ! force {
if let Some ( existing ) = registry . entries . iter (). find ( | entry | {
entry . source_path == canonical_key
&& entry . source_hash == source_hash
&& entry . deleted_at_ms . is_none ()
&& ! entry . stale
&& ( entry . indexed_at_ms . is_some ()
|| entry . light_rag_doc_id . is_some ()
|| matches! (
entry . light_rag_status . as_deref (),
Some ( "processing" | "pending" | "submitted" | "parsed" )
))
}) {
configured . push ( json! ({
"sourceId" : existing . source_id ,
"sourcePath" : existing . source_path ,
"sourceRootRelativePath" : existing . source_root_relative_path ,
"requestedSourcePath" : resolved . requested_path ,
"sourceKind" : resolved . source_kind ,
"lightRagFilePath" : existing . light_rag_file_path ,
"lightRagDocId" : existing . light_rag_doc_id ,
"lightRagStatus" : existing . light_rag_status ,
"skipped" : true ,
"skipReason" : "already_registered" ,
}));
continue ;
}
}
let staged_source = stage_lightrag_source (
& canonical ,
file_name ,
parser_hint . as_deref (),
& input_dir ,
& context ,
) ? ;
let light_rag_file_path = staged_source . light_rag_file_path . clone ();
let symlink_path = staged_source . staged_path . clone ();
let now = now_ms ();
let source_id = format! (
"lightrag-source- {} " ,
short_hash ( & canonical . display (). to_string ())
);
upsert_registry_entry (
& mut registry ,
KnowledgeRagSourceRegistryEntry {
source_id : source_id . clone (),
workspace_id : workspace_id . clone (),
root_uri : body . root_uri . clone (),
source_path : canonical_key . clone (),
source_root_relative_path : relative . clone (),
source_hash : source_hash . clone (),
light_rag_doc_id : None ,
light_rag_status : Some ( "submitted" . into ()),
light_rag_file_path : light_rag_file_path . clone (),
symlink_path : symlink_path . display (). to_string (),
parser_hint : parser_hint . clone (),
indexed_at_ms : None ,
deleted_at_ms : None ,
stale : false ,
updated_at_ms : now ,
},
);
configured . push ( json! ({
"sourceId" : source_id ,
"sourcePath" : canonical_key ,
"sourceRootRelativePath" : relative ,
"requestedSourcePath" : resolved . requested_path ,
"sourceKind" : resolved . source_kind ,
"lightRagFilePath" : light_rag_file_path ,
"symlinkPath" : symlink_path . display (). to_string (),
"scanMode" : if direct_scan_source { "direct" } else { "markdown_wrapper" },
"lightRagStatus" : "submitted" ,
}));
}
}
write_registry ( & root_path , & mut registry ) ? ;
let scan = lightrag_json (
reqwest ::Method ::POST ,
"/documents/scan" ,
None ,
true ,
& context ,
)
. await ? ;
let scan_status = scan . get ( "status" ). and_then ( Value ::as_str ). unwrap_or ( "" );
let retry_required = scan_status == "scanning_skipped_pipeline_busy" ;
Ok ( Json ( json! ({
"ok" : ! retry_required ,
"schema" : "mnote.knowledge_rag.ingest_result.v1" ,
"provider" : "lightrag" ,
"force" : body . force . unwrap_or ( false ),
"configuredSources" : configured ,
"scan" : scan ,
"retryRequired" : retry_required ,
"registry" : registry ,
})))
}
pub async fn query_rag (
State ( state ) : State < AppState > ,
Extension ( context ) : Extension < RequestContext > ,
Json ( body ) : Json < KnowledgeRagQueryRequest > ,
) -> Result < Json < Value > , WebError > {
let root_path = local_folder_source ::ensure_local_workspace_read_access_with_state (
& state ,
& context ,
& body . root_uri ,
)
. map_err ( | error | error . with_context ( & context )) ? ;
let workspace_id = effective_workspace_id ( body . workspace_id . as_deref (), & body . root_uri );
let mut registry = read_registry ( & root_path , & workspace_id , & body . root_uri ) ? ;
sync_registry_with_documents ( & root_path , & mut registry , & context ). await ? ;
let query = body . question . or ( body . query ). ok_or_else ( || {
WebError ::bad_request_code ( "knowledge_rag_query_required" , "资料库问答缺少 query" )
. with_context ( & context )
}) ? ;
let mode = normalize_lightrag_query_mode ( body . mode . as_deref ());
let raw = lightrag_json (
reqwest ::Method ::POST ,
"/query/data" ,
Some ( json! ({
"query" : query . clone (),
"mode" : mode ,
"top_k" : body . top_k ,
"chunk_top_k" : body . chunk_top_k ,
"include_references" : true ,
"include_chunk_content" : body . include_chunk_content . unwrap_or ( true ),
})),
true ,
& context ,
)
. await ? ;
let source_scope = normalize_source_scope ( body . source_paths . as_deref ());
let mut references = mapped_references ( & raw , & registry , & body . root_uri , & root_path );
filter_mapped_references_by_source_scope ( & mut references , & source_scope );
rank_mapped_references_for_query ( & mut references , & query );
Ok ( Json ( json! ({
"ok" : true ,
"schema" : "mnote.knowledge_rag.query_result.v1" ,
"provider" : "lightrag" ,
"sourceScope" : source_scope ,
2026-06-07 10:35:21 +08:00
"sourceScopeMode" : SOURCE_SCOPE_MODE_POST_FILTER ,
"rawScopeFiltered" : false ,
2026-06-07 01:10:31 +08:00
"raw" : raw ,
"references" : references ,
})))
}
fn normalize_source_scope ( source_paths : Option <& [ String ] > ) -> Vec < String > {
source_paths
. unwrap_or ( & [])
. iter ()
. map ( | value | value . trim (). trim_matches ( '/' ). replace ( '\\' , "/" ))
. filter ( | value | ! value . is_empty () && value != "." )
. collect ::< std ::collections ::BTreeSet < _ >> ()
. into_iter ()
. collect ()
}
fn filter_mapped_references_by_source_scope ( references : & mut Vec < Value > , source_scope : & [ String ]) {
if source_scope . is_empty () {
return ;
}
references . retain ( | reference | {
let source_path = reference
. get ( "sourceRootRelativePath" )
. and_then ( Value ::as_str )
. unwrap_or_default ()
. trim_matches ( '/' )
. replace ( '\\' , "/" );
source_scope
. iter ()
. any ( | scope | source_path == * scope || source_path . starts_with ( & format! ( " {scope} /" )))
});
}
fn rank_mapped_references_for_query ( references : & mut Vec < Value > , query : & str ) {
let mut indexed = references
. drain ( .. )
. enumerate ()
. map ( | ( index , reference ) | {
let score = reference_query_score ( & reference , query );
( index , score , reference )
})
. collect ::< Vec < _ >> ();
indexed . sort_by ( | left , right | right . 1. cmp ( & left . 1 ). then_with ( || left . 0. cmp ( & right . 0 )));
references . extend ( indexed . into_iter (). map ( | ( _ , _ , reference ) | reference ));
}
fn reference_query_score ( reference : & Value , query : & str ) -> i64 {
let query_normalized = query . to_ascii_lowercase ();
let tokens = query_normalized
. split ( | ch : char | ! ch . is_ascii_alphanumeric ())
. map ( str ::trim )
. filter ( | token | token . len () >= 2 )
. collect ::< Vec < _ >> ();
if tokens . is_empty () {
return 0 ;
}
let source_path = reference
. get ( "sourceRootRelativePath" )
. and_then ( Value ::as_str )
. unwrap_or_default ()
. to_ascii_lowercase ();
let file_path = reference
. get ( "filePath" )
. and_then ( Value ::as_str )
. unwrap_or_default ()
. to_ascii_lowercase ();
let quote = reference
. get ( "quote" )
. and_then ( Value ::as_str )
. unwrap_or_default ()
. to_ascii_lowercase ();
let citation = reference
. get ( "citationMarkdown" )
. and_then ( Value ::as_str )
. unwrap_or_default ()
. to_ascii_lowercase ();
let mut score = 0 ;
if quote . contains ( & query_normalized ) {
score += 100 ;
}
if source_path . contains ( & query_normalized ) || file_path . contains ( & query_normalized ) {
score += 60 ;
}
for token in tokens {
if source_path . contains ( token ) {
score += 20 ;
}
if file_path . contains ( token ) {
score += 12 ;
}
if quote . contains ( token ) {
score += 8 ;
}
if citation . contains ( token ) {
score += 4 ;
}
}
score
}
fn normalize_lightrag_query_mode ( mode : Option <& str > ) -> String {
match mode . map ( str ::trim ). filter ( | value | ! value . is_empty ()) {
Some ( "hybrid" ) => "mix" . into (),
Some ( value ) => value . into (),
None => "mix" . into (),
}
}
pub async fn open_reference (
State ( state ) : State < AppState > ,
Extension ( context ) : Extension < RequestContext > ,
Json ( body ) : Json < KnowledgeRagOpenReferenceRequest > ,
) -> Result < Json < Value > , WebError > {
let root_path = local_folder_source ::ensure_local_workspace_read_access_with_state (
& state ,
& context ,
& body . root_uri ,
)
. map_err ( | error | error . with_context ( & context )) ? ;
let workspace_id = effective_workspace_id ( body . workspace_id . as_deref (), & body . root_uri );
let mut registry = read_registry ( & root_path , & workspace_id , & body . root_uri ) ? ;
sync_registry_with_documents ( & root_path , & mut registry , & context ). await ? ;
let file_path = body
. file_path
. or_else ( || {
body . reference
. as_ref ()
. and_then ( | value | value . get ( "file_path" ))
. and_then ( Value ::as_str )
. map ( ToOwned ::to_owned )
})
. ok_or_else ( || {
WebError ::bad_request_code (
"knowledge_rag_reference_file_path_required" ,
"资料库引用打开缺少 filePath" ,
)
. with_context ( & context )
}) ? ;
let reference = map_reference_plan (
& json! ({
"reference_id" : body . reference_id ,
"file_path" : file_path ,
"chunk_id" : body . chunk_id ,
}),
& registry ,
& body . root_uri ,
& root_path ,
);
Ok ( Json ( json! ({
"ok" : true ,
"schema" : "mnote.knowledge_rag.open_reference_result.v1" ,
"reference" : reference ,
"registry" : registry ,
})))
}
pub async fn delete_source (
State ( state ) : State < AppState > ,
Extension ( context ) : Extension < RequestContext > ,
Json ( body ) : Json < KnowledgeRagDeleteSourceRequest > ,
) -> Result < Json < Value > , WebError > {
let root_path = local_folder_source ::ensure_local_workspace_read_access_with_state (
& state ,
& context ,
& body . root_uri ,
)
. map_err ( | error | error . with_context ( & context )) ? ;
let workspace_id = effective_workspace_id ( body . workspace_id . as_deref (), & body . root_uri );
let mut registry = read_registry ( & root_path , & workspace_id , & body . root_uri ) ? ;
sync_registry_with_documents ( & root_path , & mut registry , & context ). await ? ;
let source_path = body . source_path . as_deref (). map ( str ::trim );
let doc_id = body . light_rag_doc_id . as_deref (). map ( str ::trim );
let now = now_ms ();
let mut matched_doc_ids = Vec ::new ();
for entry in & mut registry . entries {
let source_matches = source_path
. filter ( | value | ! value . is_empty ())
. is_some_and ( | value | {
value == entry . source_path || value == entry . source_root_relative_path
});
let doc_matches = doc_id
. filter ( | value | ! value . is_empty ())
. is_some_and ( | value | entry . light_rag_doc_id . as_deref () == Some ( value ));
if source_matches || doc_matches {
if let Some ( doc_id ) = entry . light_rag_doc_id . clone () {
matched_doc_ids . push ( doc_id );
}
entry . stale = true ;
entry . deleted_at_ms = Some ( now );
entry . light_rag_status = Some (
if entry . light_rag_doc_id . is_some () {
"delete_submitted"
} else {
"delete_completed"
}
. into (),
);
entry . updated_at_ms = now ;
}
}
matched_doc_ids . sort ();
matched_doc_ids . dedup ();
let delete_result = if matched_doc_ids . is_empty () {
json! ({
"status" : "no_lightrag_doc" ,
"message" : "没有可删除的 LightRAG doc id; registry 已按 source 标记 stale" ,
})
} else {
lightrag_json (
reqwest ::Method ::DELETE ,
"/documents/delete_document" ,
Some ( json! ({
"doc_ids" : matched_doc_ids ,
"delete_file" : false ,
"delete_llm_cache" : false ,
})),
true ,
& context ,
)
. await ?
};
write_registry ( & root_path , & mut registry ) ? ;
Ok ( Json ( json! ({
"ok" : true ,
"schema" : "mnote.knowledge_rag.delete_source_result.v1" ,
"provider" : "lightrag" ,
"deleteResult" : delete_result ,
"registry" : registry ,
})))
}
pub async fn prune_registry (
State ( state ) : State < AppState > ,
Extension ( context ) : Extension < RequestContext > ,
Json ( body ) : Json < KnowledgeRagPruneRegistryRequest > ,
) -> Result < Json < Value > , WebError > {
let root_path = local_folder_source ::ensure_local_workspace_read_access_with_state (
& state ,
& context ,
& body . root_uri ,
)
. map_err ( | error | error . with_context ( & context )) ? ;
let workspace_id = effective_workspace_id ( body . workspace_id . as_deref (), & body . root_uri );
let mut registry = read_registry ( & root_path , & workspace_id , & body . root_uri ) ? ;
sync_registry_with_documents ( & root_path , & mut registry , & context ). await ? ;
let before = registry . entries . len ();
registry
. entries
. retain ( | entry | ! knowledge_rag_registry_entry_prunable ( entry ));
let removed = before . saturating_sub ( registry . entries . len ());
write_registry ( & root_path , & mut registry ) ? ;
Ok ( Json ( json! ({
"ok" : true ,
"schema" : "mnote.knowledge_rag.prune_registry_result.v1" ,
"provider" : "lightrag" ,
"removed" : removed ,
"registry" : registry ,
})))
}
fn knowledge_rag_registry_entry_prunable ( entry : & KnowledgeRagSourceRegistryEntry ) -> bool {
2026-06-07 10:35:21 +08:00
if matches! (
entry . light_rag_status . as_deref (),
Some ( "delete_submitted" | "delete_retry_required" )
) {
return false ;
}
if entry . light_rag_doc_id . is_some () && ( entry . deleted_at_ms . is_some () || entry . stale ) {
2026-06-07 01:10:31 +08:00
return false ;
}
entry . deleted_at_ms . is_some ()
|| entry . stale
|| matches! (
entry . light_rag_status . as_deref (),
Some ( "delete_completed" | "failed" )
)
}
2026-06-07 10:35:21 +08:00
fn knowledge_rag_provider_delete_confirmed ( entry : & KnowledgeRagSourceRegistryEntry ) -> bool {
entry . light_rag_doc_id . is_some ()
&& ( entry . deleted_at_ms . is_some ()
|| entry . stale
|| matches! (
entry . light_rag_status . as_deref (),
Some ( "delete_submitted" | "delete_retry_required" )
))
}
fn mark_registry_entry_delete_completed ( entry : & mut KnowledgeRagSourceRegistryEntry , now : u128 ) {
entry . light_rag_doc_id = None ;
entry . indexed_at_ms = None ;
entry . light_rag_status = Some ( "delete_completed" . into ());
entry . updated_at_ms = now ;
}
2026-06-07 01:10:31 +08:00
async fn sync_registry_with_documents (
root_path : & Path ,
registry : & mut KnowledgeRagSourceRegistry ,
context : & RequestContext ,
) -> Result < (), WebError > {
if registry . entries . is_empty () {
return Ok (());
}
let docs = lightrag_json ( reqwest ::Method ::GET , "/documents" , None , true , context ). await ? ;
let by_file_path = lightrag_documents_by_file_path ( & docs );
let now = now_ms ();
let mut changed = false ;
2026-06-07 10:35:21 +08:00
let mut retry_doc_ids = Vec ::new ();
2026-06-07 01:10:31 +08:00
for entry in & mut registry . entries {
if let Some ( doc ) = document_for_registry_entry ( & by_file_path , entry ) {
if let Some ( id ) = doc . get ( "id" ). and_then ( Value ::as_str ) {
entry . light_rag_doc_id = Some ( id . to_string ());
}
2026-06-07 10:35:21 +08:00
let delete_pending = matches! (
entry . light_rag_status . as_deref (),
Some ( "delete_submitted" | "delete_retry_required" )
) || entry . deleted_at_ms . is_some ();
if delete_pending {
if let Some ( doc_id ) = entry . light_rag_doc_id . clone () {
retry_doc_ids . push ( doc_id );
}
if entry . light_rag_status . is_none () {
entry . light_rag_status = Some ( "delete_submitted" . into ());
}
} else if let Some ( status ) = doc . get ( "status" ). and_then ( Value ::as_str ) {
entry . light_rag_status = Some ( status . to_string ());
2026-06-07 01:10:31 +08:00
}
2026-06-07 10:35:21 +08:00
if ! entry . stale
&& entry . deleted_at_ms . is_none ()
2026-06-07 01:10:31 +08:00
&& doc . get ( "status" ). and_then ( Value ::as_str ) == Some ( "processed" )
{
entry . indexed_at_ms . get_or_insert ( now );
entry . stale = ! Path ::new ( & entry . source_path ). exists ();
}
entry . updated_at_ms = now ;
changed = true ;
2026-06-07 10:35:21 +08:00
} else if knowledge_rag_provider_delete_confirmed ( entry ) {
mark_registry_entry_delete_completed ( entry , now );
2026-06-07 01:10:31 +08:00
changed = true ;
} else if entry . deleted_at_ms . is_some ()
&& entry . light_rag_doc_id . is_none ()
&& entry . light_rag_status . as_deref () != Some ( "delete_completed" )
{
entry . indexed_at_ms = None ;
entry . light_rag_status = Some ( "delete_completed" . into ());
entry . updated_at_ms = now ;
changed = true ;
} else if ! Path ::new ( & entry . source_path ). exists () {
entry . stale = true ;
entry . updated_at_ms = now ;
changed = true ;
}
}
2026-06-07 10:35:21 +08:00
let mut stale_doc_ids = sync_registry_source_state ( registry , now ) ? ;
stale_doc_ids . extend ( retry_doc_ids );
stale_doc_ids . sort ();
stale_doc_ids . dedup ();
2026-06-07 01:10:31 +08:00
if ! stale_doc_ids . is_empty () {
2026-06-07 10:35:21 +08:00
let delete_result = lightrag_json (
2026-06-07 01:10:31 +08:00
reqwest ::Method ::DELETE ,
"/documents/delete_document" ,
Some ( json! ({
"doc_ids" : stale_doc_ids ,
"delete_file" : false ,
"delete_llm_cache" : false ,
})),
true ,
context ,
)
. await ;
2026-06-07 10:35:21 +08:00
for entry in & mut registry . entries {
if entry
. light_rag_doc_id
. as_deref ()
. is_some_and ( | doc_id | stale_doc_ids . iter (). any ( | item | item == doc_id ))
{
entry . light_rag_status = Some (
if delete_result . is_err () {
"delete_retry_required"
} else {
"delete_submitted"
}
. into (),
);
entry . updated_at_ms = now ;
}
}
2026-06-07 01:10:31 +08:00
changed = true ;
}
if changed {
write_registry ( root_path , registry ) ? ;
}
Ok (())
}
fn lightrag_document_status_group_counts ( docs : & Value ) -> BTreeMap < String , usize > {
let mut counts = BTreeMap ::new ();
for ( status , items ) in docs
. get ( "statuses" )
. and_then ( Value ::as_object )
. into_iter ()
. flat_map ( | map | map . iter ())
{
counts . insert ( status . clone (), items . as_array (). map_or ( 0 , Vec ::len ));
}
counts
}
fn lightrag_document_summaries ( docs : & Value ) -> Vec < Value > {
let mut items = Vec ::new ();
for ( status_group , doc ) in docs
. get ( "statuses" )
. and_then ( Value ::as_object )
. into_iter ()
. flat_map ( | map | map . iter ())
. flat_map ( | ( status , items ) | {
items
. as_array ()
. into_iter ()
. flat_map ( move | docs | docs . iter (). map ( move | doc | ( status , doc )))
})
{
items . push ( json! ({
"id" : doc . get ( "id" ). and_then ( Value ::as_str ). unwrap_or_default (),
"filePath" : doc . get ( "file_path" ). and_then ( Value ::as_str ). unwrap_or_default (),
"status" : doc . get ( "status" ). and_then ( Value ::as_str ). unwrap_or ( status_group ),
"statusGroup" : status_group ,
"summary" : doc . get ( "summary" ). and_then ( Value ::as_str ). unwrap_or_default (),
"chunksCount" : doc . get ( "chunks_count" ). or_else ( || doc . get ( "chunks" )). cloned (). unwrap_or ( Value ::Null ),
"createdAt" : doc . get ( "created_at" ). or_else ( || doc . get ( "created" )). cloned (). unwrap_or ( Value ::Null ),
"updatedAt" : doc . get ( "updated_at" ). or_else ( || doc . get ( "updated" )). cloned (). unwrap_or ( Value ::Null ),
}));
}
items
}
fn lightrag_documents_by_file_path ( docs : & Value ) -> BTreeMap < String , Value > {
let mut by_file_path = BTreeMap ::< String , Value > ::new ();
for doc in docs
. get ( "statuses" )
. and_then ( Value ::as_object )
. into_iter ()
. flat_map ( | map | map . values ())
. filter_map ( Value ::as_array )
. flat_map ( | items | items . iter ())
{
if let Some ( file_path ) = doc . get ( "file_path" ). and_then ( Value ::as_str ) {
by_file_path . insert ( file_path . to_string (), doc . clone ());
}
}
by_file_path
}
fn sync_registry_source_state (
registry : & mut KnowledgeRagSourceRegistry ,
now : u128 ,
) -> Result < Vec < String > , WebError > {
let mut stale_doc_ids = Vec ::new ();
for entry in & mut registry . entries {
if entry . deleted_at_ms . is_some () {
continue ;
}
let Some ( doc_id ) = entry . light_rag_doc_id . clone () else {
continue ;
};
let source_path = Path ::new ( & entry . source_path );
if ! source_path . exists () {
entry . stale = true ;
entry . deleted_at_ms = Some ( now );
entry . indexed_at_ms = None ;
2026-06-07 10:35:21 +08:00
entry . light_rag_status = Some ( "delete_submitted" . into ());
2026-06-07 01:10:31 +08:00
entry . updated_at_ms = now ;
stale_doc_ids . push ( doc_id );
continue ;
}
let current_hash = source_hash ( source_path ) ? ;
if current_hash != entry . source_hash {
entry . stale = true ;
entry . source_hash = current_hash ;
entry . indexed_at_ms = None ;
2026-06-07 10:35:21 +08:00
entry . light_rag_status = Some ( "delete_submitted" . into ());
2026-06-07 01:10:31 +08:00
entry . updated_at_ms = now ;
stale_doc_ids . push ( doc_id );
}
}
stale_doc_ids . sort ();
stale_doc_ids . dedup ();
Ok ( stale_doc_ids )
}
async fn lightrag_json (
method : reqwest ::Method ,
path : & str ,
body : Option < Value > ,
use_api_key : bool ,
context : & RequestContext ,
) -> Result < Value , WebError > {
let endpoint = lightrag_endpoint ();
let url = format! ( " {}{} " , endpoint . trim_end_matches ( '/' ), path );
let client = reqwest ::Client ::builder ()
. timeout ( Duration ::from_secs ( 180 ))
. build ()
. map_err ( | error | {
WebError ::internal ( format! ( "LightRAG HTTP client 初始化失败: {error} " ))
. with_context ( context )
}) ? ;
let mut request = client . request ( method , & url );
if use_api_key {
if let Some ( api_key ) = lightrag_api_key () {
request = request . header ( "X-API-Key" , api_key );
}
}
if let Some ( body ) = body {
request = request . json ( & body );
}
let response = request . send (). await . map_err ( | error | {
WebError ::bad_gateway_code (
"knowledge_rag_lightrag_unreachable" ,
format! ( "无法访问 LightRAG provider: {error} " ),
)
. with_context ( context )
}) ? ;
let status = response . status ();
let text = response . text (). await . unwrap_or_default ();
if ! status . is_success () {
return Err ( WebError ::bad_gateway_code (
"knowledge_rag_lightrag_error" ,
format! ( "LightRAG provider 返回 HTTP {status} : {text} " ),
)
. with_context ( context ));
}
serde_json ::from_str ( & text ). map_err ( | error | {
WebError ::bad_gateway_code (
"knowledge_rag_lightrag_json_invalid" ,
format! ( "LightRAG provider 返回非 JSON 响应: {error} " ),
)
. with_context ( context )
})
}
fn mapped_references (
raw : & Value ,
registry : & KnowledgeRagSourceRegistry ,
root_uri : & str ,
root_path : & Path ,
) -> Vec < Value > {
reference_array ( raw )
. into_iter ()
. map ( | reference | {
let enriched = enrich_reference_with_chunks ( raw , & reference );
map_reference_plan ( & enriched , registry , root_uri , root_path )
})
. filter ( | reference | {
! reference
. get ( "stale" )
. and_then ( Value ::as_bool )
. unwrap_or ( false )
&& ! reference
. get ( "deleted" )
. and_then ( Value ::as_bool )
. unwrap_or ( false )
2026-06-07 10:35:21 +08:00
&& ! reference
. get ( "unmapped" )
. and_then ( Value ::as_bool )
. unwrap_or ( false )
2026-06-07 01:10:31 +08:00
})
. collect ()
}
fn reference_array ( raw : & Value ) -> Vec < Value > {
if let Some ( references ) = raw
. get ( "references" )
. or_else ( || raw . get ( "data" ). and_then ( | value | value . get ( "references" )))
. and_then ( Value ::as_array )
{
return references . clone ();
}
let mut by_key = BTreeMap ::< String , Value > ::new ();
for chunk in chunk_array ( raw ) {
let file_path = chunk . get ( "file_path" ). and_then ( Value ::as_str ). unwrap_or ( "" );
if file_path . is_empty () {
continue ;
}
let reference_id = chunk
. get ( "reference_id" )
. and_then ( Value ::as_str )
. unwrap_or ( file_path );
let key = format! ( " {file_path} \n {reference_id} " );
by_key . entry ( key ). or_insert_with ( || {
json! ({
"file_path" : file_path ,
"reference_id" : reference_id ,
"chunks" : [ chunk . clone ()],
})
});
}
by_key . into_values (). collect ()
}
fn chunk_array ( raw : & Value ) -> Vec <& Value > {
raw . get ( "chunks" )
. or_else ( || raw . get ( "data" ). and_then ( | value | value . get ( "chunks" )))
. and_then ( Value ::as_array )
. map ( | items | items . iter (). collect ())
. unwrap_or_default ()
}
fn enrich_reference_with_chunks ( raw : & Value , reference : & Value ) -> Value {
let reference_id = reference . get ( "reference_id" ). and_then ( Value ::as_str );
let file_path = reference . get ( "file_path" ). and_then ( Value ::as_str );
let chunks = chunk_array ( raw )
. into_iter ()
. filter ( | chunk | {
let chunk_reference_id = chunk . get ( "reference_id" ). and_then ( Value ::as_str );
let chunk_file_path = chunk . get ( "file_path" ). and_then ( Value ::as_str );
reference_id
. zip ( chunk_reference_id )
. is_some_and ( | ( left , right ) | left == right )
|| file_path
. zip ( chunk_file_path )
. is_some_and ( | ( left , right ) | left == right )
})
. cloned ()
. collect ::< Vec < _ >> ();
if chunks . is_empty () {
return reference . clone ();
}
let mut enriched = reference . clone ();
if let Some ( map ) = enriched . as_object_mut () {
map . insert ( "chunks" . into (), Value ::Array ( chunks ));
}
enriched
}
fn map_reference_plan (
reference : & Value ,
registry : & KnowledgeRagSourceRegistry ,
root_uri : & str ,
root_path : & Path ,
) -> Value {
let file_path = reference
. get ( "file_path" )
. and_then ( Value ::as_str )
. unwrap_or_default ();
let doc_id = reference . get ( "doc_id" ). and_then ( Value ::as_str );
let entry = registry . entries . iter (). find ( | entry | {
lightrag_file_path_matches ( entry , file_path )
|| doc_id . is_some_and ( | doc_id | entry . light_rag_doc_id . as_deref () == Some ( doc_id ))
});
let locator_degraded = entry . is_none_or ( | entry | entry . stale || entry . deleted_at_ms . is_some ());
let source_path = entry . map ( | entry | entry . source_path . clone ());
let source_root_relative_path = entry . map ( | entry | entry . source_root_relative_path . clone ());
let primary_chunk = reference
. get ( "chunks" )
. and_then ( Value ::as_array )
. and_then ( | chunks | chunks . first ());
let chunk_id = reference
. get ( "chunk_id" )
. or_else ( || primary_chunk . and_then ( | chunk | chunk . get ( "chunk_id" )))
. cloned ()
. unwrap_or ( Value ::Null );
let quote = primary_chunk
. and_then ( | chunk | chunk . get ( "content" ))
. and_then ( Value ::as_str )
. map ( ToOwned ::to_owned )
. or_else ( || chunk_id . as_str (). and_then ( lightrag_chunk_content_for_id ))
. map ( | value | value . chars (). take ( 500 ). collect ::< String > ());
let locator = entry . and_then ( | entry | {
quote . as_deref (). and_then ( | quote | {
lightrag_locator_for_reference ( root_path , root_uri , entry , & chunk_id , quote )
})
});
let locator_degraded = locator_degraded || locator . is_none ();
let fallback_citation_url =
entry . and_then ( | entry | fallback_resource_citation_url ( root_path , root_uri , entry ));
let citation_url = locator
. as_ref ()
. map ( citation_url_for_locator )
. or ( fallback_citation_url );
let citation_markdown = locator
. as_ref ()
. map ( citation_markdown_for_locator )
. or_else ( || {
citation_url . as_ref (). map ( | url | {
format! (
"[来源定位降级: {} ]( {} )" ,
markdown_link_label_escape ( file_path ),
url . replace ( ')' , "%29" )
)
})
});
json! ({
"schema" : REFERENCE_SCHEMA ,
"provider" : "lightrag" ,
"reference" : reference ,
"filePath" : file_path ,
"chunkId" : chunk_id ,
"quote" : quote ,
"locator" : locator ,
"citationUrl" : citation_url ,
"sourceId" : entry . map ( | entry | entry . source_id . clone ()),
"sourcePath" : source_path ,
"sourceRootRelativePath" : source_root_relative_path ,
2026-06-07 10:35:21 +08:00
"unmapped" : entry . is_none (),
2026-06-07 01:10:31 +08:00
"stale" : entry . is_some_and ( | entry | entry . stale ),
"deleted" : entry . is_some_and ( | entry | entry . deleted_at_ms . is_some ()),
"locatorDegraded" : locator_degraded ,
"openAction" : {
"kind" : "mnote.local_resource.open" ,
"params" : {
"rootUri" : root_uri ,
"path" : entry . map ( | entry | entry . source_root_relative_path . clone ()),
"provider" : "lightrag" ,
"filePath" : file_path ,
"chunkId" : chunk_id ,
},
},
"citationMarkdown" : citation_markdown . unwrap_or_else ( || format! ( "[来源定位降级: {} ](#)" , file_path )),
})
}
fn fallback_resource_citation_url (
root_path : & Path ,
root_uri : & str ,
entry : & KnowledgeRagSourceRegistryEntry ,
) -> Option < String > {
if entry . stale || entry . deleted_at_ms . is_some () {
return None ;
}
let owner_document_id =
fallback_owner_document_id ( root_path , & entry . source_root_relative_path ) ? ;
let mut url = format! ( "/documents/ {owner_document_id} " );
append_query_param ( & mut url , "sourceKind" , "local_folder" );
append_query_param ( & mut url , "rootUri" , root_uri );
append_query_param (
& mut url ,
"resourceTab" ,
& format! (
"resource:file: {} : {} " ,
root_uri , entry . source_root_relative_path
),
);
append_query_param ( & mut url , "resourcePath" , & entry . source_root_relative_path );
Some ( url )
}
fn fallback_owner_document_id ( root_path : & Path , relative_path : & str ) -> Option < String > {
if is_markdown_like_path ( relative_path ) {
return Some ( local_markdown_document_id ( relative_path ));
}
let source_path = Path ::new ( relative_path );
if let Some ( parent ) = source_path
. parent ()
. filter ( | parent | ! parent . as_os_str (). is_empty ())
{
let parent_path = root_path . join ( parent );
if let Ok ( entries ) = fs ::read_dir ( parent_path ) {
let mut candidates = entries
. filter_map ( Result ::ok )
. filter_map ( | entry | {
let path = entry . path ();
if ! path . is_file () || ! is_markdown_like_path ( & path . to_string_lossy ()) {
return None ;
}
let file_name = path . file_name () ? . to_string_lossy ();
if file_name . starts_with ( '.' ) {
return None ;
}
let rel = parent . join ( file_name . as_ref ());
Some ( rel . to_string_lossy (). replace ( '\\' , "/" ))
})
. collect ::< Vec < _ >> ();
candidates . sort ();
if let Some ( candidate ) = candidates . first () {
return Some ( local_markdown_document_id ( candidate ));
}
}
}
for root_candidate in [ "README.md" , "index.md" ] {
if root_path . join ( root_candidate ). is_file () {
return Some ( local_markdown_document_id ( root_candidate ));
}
}
None
}
fn is_markdown_like_path ( value : & str ) -> bool {
Path ::new ( value )
. extension ()
. and_then ( | extension | extension . to_str ())
. is_some_and ( | extension | {
extension . eq_ignore_ascii_case ( "md" ) || extension . eq_ignore_ascii_case ( "markdown" )
})
}
fn local_markdown_document_id ( relative_path : & str ) -> String {
format! (
"local-md: {} " ,
encode_local_document_id_segment ( relative_path )
)
}
fn append_query_param ( url : & mut String , key : & str , value : & str ) {
let value = value . trim ();
if value . is_empty () {
return ;
}
let separator = if url . contains ( '?' ) { '&' } else { '?' };
url . push ( separator );
url . push_str ( & encode_query_component ( key ));
url . push ( '=' );
url . push_str ( & encode_query_component ( value ));
}
fn encode_query_component ( value : & str ) -> String {
let mut encoded = String ::with_capacity ( value . len ());
for byte in value . as_bytes () {
if byte . is_ascii_alphanumeric () || matches! ( byte , b '-' | b '_' | b '.' | b '~' ) {
encoded . push ( * byte as char );
} else {
encoded . push_str ( & format! ( "% {byte:02X} " ));
}
}
encoded
}
fn encode_local_document_id_segment ( relative_path : & str ) -> String {
relative_path
. replace ( '\\' , "/" )
. bytes ()
. flat_map ( | byte | match byte {
b 'A' ..= b 'Z' | b 'a' ..= b 'z' | b '0' ..= b '9' | b '-' | b '_' | b '.' => {
vec! [ byte as char ]
}
b '/' => "~2F" . chars (). collect ::< Vec < _ >> (),
_ => format! ( "~ {byte:02X} " ). chars (). collect ::< Vec < _ >> (),
})
. collect ()
}
fn markdown_link_label_escape ( value : & str ) -> String {
value . replace ( '[' , " \\ [" ). replace ( ']' , " \\ ]" )
}
fn lightrag_locator_for_reference (
root_path : & Path ,
root_uri : & str ,
entry : & KnowledgeRagSourceRegistryEntry ,
chunk_id : & Value ,
quote : & str ,
) -> Option < EvidenceLocator > {
let block = find_lightrag_sidecar_block ( entry , quote ) ? ;
let position = block
. get ( "positions" ) ?
. as_array () ?
. iter ()
. find_map ( parse_position ) ? ;
let resource_path = entry . source_root_relative_path . clone ();
let mut locator = EvidenceLocator ::new (
root_uri ,
"" ,
& resource_path ,
EvidenceResourceKind ::Pdf ,
EvidenceOpenAction {
action_type : "mnote.open_resource_locator" . into (),
url : "/" . into (),
params : json ! ({
"rootUri" : root_uri ,
"resourcePath" : resource_path ,
"provider" : "lightrag" ,
"chunkId" : chunk_id ,
}),
},
);
locator . resource_path = Some ( entry . source_root_relative_path . clone ());
locator . page = Some ( position . page );
locator . bbox = Some ( position . bbox );
locator . block_id = block
. get ( "blockid" )
. and_then ( Value ::as_str )
. map ( ToOwned ::to_owned )
. or_else ( || chunk_id . as_str (). map ( ToOwned ::to_owned ));
locator . source_map_path =
sidecar_blocks_path ( entry ). and_then ( | path | root_relative_path ( root_path , & path ). ok ());
Some ( locator )
}
#[derive(Debug, Clone)]
struct LightRagPosition {
page : u32 ,
bbox : EvidenceBBox ,
}
fn parse_position ( value : & Value ) -> Option < LightRagPosition > {
let page = value
. get ( "anchor" )
. and_then ( Value ::as_str )
. and_then ( | anchor | anchor . trim (). parse ::< u32 > (). ok ()) ? ;
let range = value . get ( "range" ) ? . as_array () ? ;
if range . len () != 4 {
return None ;
}
Some ( LightRagPosition {
page ,
bbox : EvidenceBBox {
x0 : range . first () ? . as_f64 () ? ,
y0 : range . get ( 1 ) ? . as_f64 () ? ,
x1 : range . get ( 2 ) ? . as_f64 () ? ,
y1 : range . get ( 3 ) ? . as_f64 () ? ,
},
})
}
fn find_lightrag_sidecar_block (
entry : & KnowledgeRagSourceRegistryEntry ,
quote : & str ,
) -> Option < Value > {
let quote_normalized = normalize_text_for_match ( quote );
if quote_normalized . is_empty () {
return None ;
}
let path = sidecar_blocks_path ( entry ) ? ;
let content = fs ::read_to_string ( path ). ok () ? ;
for line in content . lines () {
let block = serde_json ::from_str ::< Value > ( line ). ok () ? ;
if block . get ( "positions" ). and_then ( Value ::as_array ). is_none () {
continue ;
}
let block_text = block
. get ( "content" )
. and_then ( Value ::as_str )
. unwrap_or_default ();
let block_normalized = normalize_text_for_match ( block_text );
if block_normalized . is_empty () {
continue ;
}
if quote_normalized . contains ( & match_prefix ( & block_normalized , 80 ))
|| block_normalized . contains ( & match_prefix ( & quote_normalized , 80 ))
{
return Some ( block );
}
}
2026-06-07 10:35:21 +08:00
None
2026-06-07 01:10:31 +08:00
}
fn normalize_text_for_match ( value : & str ) -> String {
value
. split_whitespace ()
. collect ::< Vec < _ >> ()
. join ( " " )
. chars ()
. collect ::< String > ()
}
fn match_prefix ( value : & str , limit : usize ) -> String {
value
. chars ()
. take ( limit . min ( value . chars (). count ()))
. collect ()
}
fn sidecar_blocks_path ( entry : & KnowledgeRagSourceRegistryEntry ) -> Option < PathBuf > {
for file_path in lightrag_sidecar_file_path_candidates ( entry ) {
let stem = Path ::new ( & file_path ). file_stem () ? . to_string_lossy ();
let path = lightrag_input_dir ()
. join ( "__parsed__" )
. join ( format! ( " {file_path} .parsed" ))
. join ( format! ( " {stem} .blocks.jsonl" ));
if path . exists () {
return Some ( path );
}
}
None
}
fn lightrag_sidecar_file_path_candidates ( entry : & KnowledgeRagSourceRegistryEntry ) -> Vec < String > {
let mut candidates = vec! [ entry . light_rag_file_path . clone ()];
if let Some ( stripped ) = strip_one_supported_parser_hint ( & entry . light_rag_file_path ) {
candidates . push ( stripped );
}
candidates . sort ();
candidates . dedup ();
candidates
}
fn lightrag_chunk_content_for_id ( chunk_id : & str ) -> Option < String > {
let content =
fs ::read_to_string ( lightrag_working_dir (). join ( "kv_store_text_chunks.json" )). ok () ? ;
let chunks = serde_json ::from_str ::< Value > ( & content ). ok () ? ;
chunks
. get ( chunk_id )
. and_then ( | chunk | chunk . get ( "content" ))
. and_then ( Value ::as_str )
. map ( ToOwned ::to_owned )
}
fn read_registry (
root_path : & Path ,
workspace_id : & str ,
root_uri : & str ,
) -> Result < KnowledgeRagSourceRegistry , WebError > {
let path = registry_path ( root_path );
if ! path . exists () {
return Ok ( KnowledgeRagSourceRegistry {
schema : REGISTRY_SCHEMA . to_string (),
workspace_id : workspace_id . to_string (),
root_uri : root_uri . to_string (),
updated_at_ms : now_ms (),
entries : Vec ::new (),
});
}
let content = fs ::read_to_string ( & path ). map_err ( | error | {
WebError ::bad_request_code (
"knowledge_rag_registry_read_failed" ,
format! ( "无法读取 LightRAG source registry: {error} " ),
)
}) ? ;
serde_json ::from_str ( & content ). map_err ( | error | {
WebError ::bad_request_code (
"knowledge_rag_registry_invalid" ,
format! ( "LightRAG source registry JSON 无效: {error} " ),
)
})
}
fn write_registry (
root_path : & Path ,
registry : & mut KnowledgeRagSourceRegistry ,
) -> Result < (), WebError > {
registry . updated_at_ms = now_ms ();
let path = registry_path ( root_path );
if let Some ( parent ) = path . parent () {
fs ::create_dir_all ( parent ). map_err ( | error | {
WebError ::bad_request_code (
"knowledge_rag_registry_dir_failed" ,
format! ( "无法创建 LightRAG source registry 目录: {error} " ),
)
}) ? ;
}
let content = serde_json ::to_string_pretty ( registry ). map_err ( | error | {
WebError ::internal ( format! ( "无法序列化 LightRAG source registry: {error} " ))
}) ? ;
fs ::write ( & path , format! ( " {content} \n " )). map_err ( | error | {
WebError ::bad_request_code (
"knowledge_rag_registry_write_failed" ,
format! ( "无法写入 LightRAG source registry: {error} " ),
)
})
}
fn upsert_registry_entry (
registry : & mut KnowledgeRagSourceRegistry ,
next : KnowledgeRagSourceRegistryEntry ,
) {
if let Some ( existing ) = registry
. entries
. iter_mut ()
. find ( | entry | entry . source_path == next . source_path )
{
* existing = next ;
return ;
}
registry . entries . push ( next );
}
fn registry_path ( root_path : & Path ) -> PathBuf {
root_path
. join ( ".mnote" )
. join ( "index" )
. join ( "lightrag-source-registry.json" )
}
#[derive(Debug, Clone)]
struct ResolvedKnowledgeRagSource {
canonical_path : PathBuf ,
requested_path : String ,
source_kind : & 'static str ,
}
fn resolve_knowledge_rag_sources (
root_path : & Path ,
source_path : & str ,
context : & RequestContext ,
) -> Result < Vec < ResolvedKnowledgeRagSource > , WebError > {
let canonical = ensure_source_path_in_root ( root_path , source_path , context ) ? ;
if canonical . is_file () {
if ! knowledge_rag_source_supported_file ( & canonical ) {
return Err ( WebError ::bad_request_code (
"knowledge_rag_source_unsupported" ,
"资料库 source 文件类型暂不支持" ,
)
. with_context ( context ));
}
return Ok ( vec! [ ResolvedKnowledgeRagSource {
canonical_path : canonical ,
requested_path : source_path . trim (). to_string (),
source_kind : "file" ,
}]);
}
if canonical . is_dir () {
let mut files = collect_knowledge_rag_directory_sources ( root_path , & canonical , context ) ? ;
if files . is_empty () {
return Err ( WebError ::bad_request_code (
"knowledge_rag_source_directory_empty" ,
"资料库目录中没有可索引文件" ,
)
. with_context ( context ));
}
files . sort ();
files . dedup ();
return Ok ( files
. into_iter ()
. map ( | path | ResolvedKnowledgeRagSource {
canonical_path : path ,
requested_path : source_path . trim (). to_string (),
source_kind : "directory" ,
})
. collect ());
}
Err ( WebError ::bad_request_code (
"knowledge_rag_source_not_file_or_directory" ,
"资料库 source 必须是文件或目录" ,
)
. with_context ( context ))
}
fn collect_knowledge_rag_directory_sources (
root_path : & Path ,
directory : & Path ,
context : & RequestContext ,
) -> Result < Vec < PathBuf > , WebError > {
let mut files = Vec ::new ();
let mut stack = vec! [ directory . to_path_buf ()];
while let Some ( current_dir ) = stack . pop () {
let mut entries = fs ::read_dir ( & current_dir )
. map_err ( | error | {
WebError ::bad_request_code (
"knowledge_rag_source_directory_read_failed" ,
format! ( "无法读取资料库目录 {} : {error} " , current_dir . display ()),
)
. with_context ( context )
}) ?
. filter_map ( Result ::ok )
. collect ::< Vec < _ >> ();
entries . sort_by_key ( | entry | entry . path ());
for entry in entries {
let path = entry . path ();
let canonical = path . canonicalize (). map_err ( | error | {
WebError ::bad_request_code (
"knowledge_rag_source_unavailable" ,
format! ( "无法访问资料库 source: {error} " ),
)
. with_context ( context )
}) ? ;
if ! canonical . starts_with ( root_path ) {
continue ;
}
if canonical . is_dir () {
if should_skip_knowledge_rag_directory ( & canonical ) {
continue ;
}
stack . push ( canonical );
continue ;
}
if canonical . is_file () && knowledge_rag_source_supported_file ( & canonical ) {
files . push ( canonical );
if files . len () > MAX_INGEST_SOURCES_PER_REQUEST {
return Err ( WebError ::bad_request_code (
"knowledge_rag_source_limit_exceeded" ,
format! (
"单次资料库索引最多支持 {MAX_INGEST_SOURCES_PER_REQUEST} 个文件,请缩小目录范围"
),
)
. with_context ( context ));
}
}
}
}
Ok ( files )
}
fn should_skip_knowledge_rag_directory ( path : & Path ) -> bool {
let name = path
. file_name ()
. and_then ( | value | value . to_str ())
. unwrap_or_default ();
matches! (
name ,
".git" | ".mnote" | "node_modules" | "target" | "__parsed__" | ".venv"
)
}
fn knowledge_rag_source_supported_file ( path : & Path ) -> bool {
path . extension ()
. and_then ( | value | value . to_str ())
. map ( | value | {
KNOWLEDGE_RAG_SOURCE_EXTENSIONS
. iter ()
. any ( | extension | value . eq_ignore_ascii_case ( extension ))
})
. unwrap_or ( false )
}
fn lightrag_scan_supported_file ( path : & Path ) -> bool {
path . extension ()
. and_then ( | value | value . to_str ())
. map ( | value | {
LIGHTRAG_SCAN_SOURCE_EXTENSIONS
. iter ()
. any ( | extension | value . eq_ignore_ascii_case ( extension ))
})
. unwrap_or ( false )
}
struct StagedLightRagSource {
light_rag_file_path : String ,
staged_path : PathBuf ,
}
fn stage_lightrag_source (
canonical : & Path ,
file_name : & str ,
parser_hint : Option <& str > ,
input_dir : & Path ,
context : & RequestContext ,
) -> Result < StagedLightRagSource , WebError > {
if lightrag_scan_supported_file ( canonical ) {
let light_rag_file_path = lightrag_symlink_name ( canonical , file_name , parser_hint );
let symlink_path = input_dir . join ( & light_rag_file_path );
replace_lightrag_input_file ( & symlink_path , context ) ? ;
create_symlink ( canonical , & symlink_path , context ) ? ;
return Ok ( StagedLightRagSource {
light_rag_file_path ,
staged_path : symlink_path ,
});
}
let asset_file_path = lightrag_symlink_name ( canonical , file_name , None );
let asset_symlink_path = input_dir . join ( & asset_file_path );
replace_lightrag_input_file ( & asset_symlink_path , context ) ? ;
create_symlink ( canonical , & asset_symlink_path , context ) ? ;
let wrapper_file_path = format! ( " {asset_file_path} .md" );
let wrapper_path = input_dir . join ( & wrapper_file_path );
replace_lightrag_input_file ( & wrapper_path , context ) ? ;
let wrapper_markdown = format! ( "# {file_name} \n\n  \n " );
fs ::write ( & wrapper_path , wrapper_markdown ). map_err ( | error | {
WebError ::bad_request_code (
"knowledge_rag_wrapper_write_failed" ,
format! ( "无法写入 LightRAG 图片包装 Markdown: {error} " ),
)
. with_context ( context )
}) ? ;
Ok ( StagedLightRagSource {
light_rag_file_path : wrapper_file_path ,
staged_path : wrapper_path ,
})
}
fn replace_lightrag_input_file ( path : & Path , context : & RequestContext ) -> Result < (), WebError > {
if path . exists () || path . symlink_metadata (). is_ok () {
fs ::remove_file ( path ). map_err ( | error | {
WebError ::bad_request_code (
"knowledge_rag_symlink_replace_failed" ,
format! ( "无法替换 LightRAG source 文件: {error} " ),
)
. with_context ( context )
}) ? ;
}
Ok (())
}
fn ensure_source_path_in_root (
root_path : & Path ,
source_path : & str ,
context : & RequestContext ,
) -> Result < PathBuf , WebError > {
let raw_path = source_path
. trim ()
. strip_prefix ( "file://" )
. map ( PathBuf ::from )
. unwrap_or_else ( || PathBuf ::from ( source_path . trim ()));
let target = if raw_path . is_absolute () {
raw_path
} else {
root_path . join ( raw_path )
};
let canonical = target . canonicalize (). map_err ( | error | {
WebError ::bad_request_code (
"knowledge_rag_source_unavailable" ,
format! ( "无法访问资料库 source: {error} " ),
)
. with_context ( context )
}) ? ;
if ! canonical . starts_with ( root_path ) {
return Err ( WebError ::bad_request_code (
"knowledge_rag_source_root_escape" ,
"资料库 source 不能越过授权目录" ,
)
. with_context ( context ));
}
Ok ( canonical )
}
fn root_relative_path ( root_path : & Path , source_path : & Path ) -> Result < String , WebError > {
let relative = source_path . strip_prefix ( root_path ). map_err ( | error | {
WebError ::bad_request_code (
"knowledge_rag_source_relative_failed" ,
format! ( "无法生成资料库 source 相对路径: {error} " ),
)
}) ? ;
Ok ( relative . to_string_lossy (). replace ( '\\' , "/" ))
}
fn create_symlink (
source : & Path ,
symlink_path : & Path ,
context : & RequestContext ,
) -> Result < (), WebError > {
#[cfg(unix)]
{
std ::os ::unix ::fs ::symlink ( source , symlink_path ). map_err ( | error | {
WebError ::bad_request_code (
"knowledge_rag_symlink_failed" ,
format! ( "无法创建 LightRAG source symlink: {error} " ),
)
. with_context ( context )
})
}
#[cfg(not(unix))]
{
fs ::copy ( source , symlink_path ). map ( | _ | ()). map_err ( | error | {
WebError ::bad_request_code (
"knowledge_rag_staging_copy_failed" ,
format! ( "无法创建 LightRAG source staging 文件: {error} " ),
)
. with_context ( context )
})
}
}
fn source_hash ( path : & Path ) -> Result < String , WebError > {
let bytes = fs ::read ( path ). map_err ( | error | {
WebError ::bad_request_code (
"knowledge_rag_source_hash_failed" ,
format! ( "无法读取资料库 source 以计算 hash: {error} " ),
)
}) ? ;
Ok ( format! ( "mnote-fnv64: {:016x} " , fnv64 ( & bytes )))
}
fn fnv64 ( bytes : & [ u8 ]) -> u64 {
let mut hash = 0xcbf29ce484222325 u64 ;
for byte in bytes {
hash ^= u64 ::from ( * byte );
hash = hash . wrapping_mul ( 0x100000001b3 );
}
hash
}
fn lightrag_symlink_name ( path : & Path , file_name : & str , parser_hint : Option <& str > ) -> String {
let prefix = format! ( "mnote- {} -" , short_hash ( & path . display (). to_string ()));
let Some ( parser_hint ) = parser_hint else {
return format! ( " {prefix}{file_name} " );
};
if strip_one_supported_parser_hint ( file_name ). is_some () {
return format! ( " {prefix}{file_name} " );
}
let path = Path ::new ( file_name );
let stem = path
. file_stem ()
. and_then ( | value | value . to_str ())
. unwrap_or ( file_name );
let extension = path . extension (). and_then ( | value | value . to_str ());
match extension {
Some ( extension ) if ! extension . is_empty () => {
format! ( " {prefix}{stem} .[ {parser_hint} ]. {extension} " )
}
_ => format! ( " {prefix}{stem} .[ {parser_hint} ]" ),
}
}
fn normalize_parser_hint (
value : Option <& str > ,
context : & RequestContext ,
) -> Result < Option < String > , WebError > {
let Some ( value ) = value . map ( str ::trim ). filter ( | value | ! value . is_empty ()) else {
return Ok ( None );
};
let normalized = value
. trim_matches ( '[' )
. trim_matches ( ']' )
. trim ()
. to_ascii_lowercase ();
match normalized . as_str () {
"legacy" | "native" | "mineru" | "docling" => Ok ( Some ( normalized )),
_ => Err ( WebError ::bad_request_code (
"knowledge_rag_parser_hint_invalid" ,
"LightRAG parserHint 仅支持 legacy/native/mineru/docling" ,
)
. with_context ( context )),
}
}
fn document_for_registry_entry < 'a > (
by_file_path : & 'a BTreeMap < String , Value > ,
entry : & KnowledgeRagSourceRegistryEntry ,
) -> Option <& 'a Value > {
by_file_path . get ( & entry . light_rag_file_path ). or_else ( || {
strip_one_supported_parser_hint ( & entry . light_rag_file_path )
. as_ref ()
. and_then ( | path | by_file_path . get ( path ))
})
}
fn lightrag_file_path_matches ( entry : & KnowledgeRagSourceRegistryEntry , file_path : & str ) -> bool {
entry . light_rag_file_path == file_path
|| strip_one_supported_parser_hint ( & entry . light_rag_file_path ). as_deref () == Some ( file_path )
}
fn strip_one_supported_parser_hint ( file_name : & str ) -> Option < String > {
for hint in [ "legacy" , "native" , "mineru" , "docling" ] {
let needle = format! ( ".[ {hint} ]" );
if file_name . contains ( & needle ) {
return Some ( file_name . replacen ( & needle , "" , 1 ));
}
}
None
}
fn short_hash ( value : & str ) -> String {
let mut hasher = DefaultHasher ::new ();
value . hash ( & mut hasher );
format! ( " {:016x} " , hasher . finish ())
}
fn lightrag_endpoint () -> String {
env ::var ( "MNOTE_LIGHTRAG_ENDPOINT" )
. or_else ( | _ | env ::var ( "LIGHTRAG_ENDPOINT" ))
. unwrap_or_else ( | _ | DEFAULT_LIGHTRAG_ENDPOINT . into ())
. trim ()
. trim_end_matches ( '/' )
. to_string ()
}
fn lightrag_dashboard_url () -> String {
env ::var ( "MNOTE_LIGHTRAG_DASHBOARD_URL" ). unwrap_or_else ( | _ | lightrag_endpoint ())
}
fn lightrag_input_dir () -> PathBuf {
if let Ok ( value ) = env ::var ( "MNOTE_LIGHTRAG_INPUT_DIR" ) {
return PathBuf ::from ( value );
}
if let Some ( value ) = read_lightrag_dotenv_value ( "INPUT_DIR" ) {
return PathBuf ::from ( value );
}
env ::var ( "INPUT_DIR" )
. map ( PathBuf ::from )
. unwrap_or_else ( | _ | PathBuf ::from ( DEFAULT_LIGHTRAG_INPUT_DIR ))
}
fn lightrag_working_dir () -> PathBuf {
if let Ok ( value ) = env ::var ( "MNOTE_LIGHTRAG_WORKING_DIR" ) {
return PathBuf ::from ( value );
}
if let Some ( value ) = read_lightrag_dotenv_value ( "WORKING_DIR" ) {
return PathBuf ::from ( value );
}
env ::var ( "WORKING_DIR" )
. map ( PathBuf ::from )
. unwrap_or_else ( | _ | PathBuf ::from ( DEFAULT_LIGHTRAG_WORKING_DIR ))
}
fn lightrag_api_key () -> Option < String > {
env ::var ( "MNOTE_LIGHTRAG_API_KEY" )
. ok ()
. map ( | value | value . trim (). trim_matches ( '"' ). to_string ())
. filter ( | value | ! value . is_empty ())
. or_else ( || read_lightrag_dotenv_value ( "LIGHTRAG_API_KEY" ))
. or_else ( || env ::var ( "LIGHTRAG_API_KEY" ). ok ())
. map ( | value | value . trim (). trim_matches ( '"' ). to_string ())
. filter ( | value | ! value . is_empty ())
}
fn read_lightrag_dotenv_value ( key : & str ) -> Option < String > {
let path = env ::var ( "MNOTE_LIGHTRAG_ENV_FILE" )
. map ( PathBuf ::from )
. unwrap_or_else ( | _ | PathBuf ::from ( "/mnt/Data1T/Mnote_data/lightrag/LightRAG/.env" ));
let content = fs ::read_to_string ( path ). ok () ? ;
for line in content . lines () {
let line = line . trim ();
if line . is_empty () || line . starts_with ( '#' ) {
continue ;
}
let Some (( name , value )) = line . split_once ( '=' ) else {
continue ;
};
if name . trim () != key {
continue ;
}
let value = value . trim (). trim_matches ( '"' ). to_string ();
if ! value . is_empty () {
return Some ( value );
}
}
None
}
fn effective_workspace_id ( input : Option <& str > , root_uri : & str ) -> String {
input
. map ( str ::trim )
. filter ( | value | ! value . is_empty ())
. map ( ToOwned ::to_owned )
. unwrap_or_else ( || {
local_folder_source ::local_workspace_id_from_root_uri ( root_uri )
. unwrap_or_else ( | _ | format! ( "local-ws: {} " , short_hash ( root_uri )))
})
}
fn now_ms () -> u128 {
SystemTime ::now ()
. duration_since ( UNIX_EPOCH )
. map ( | value | value . as_millis ())
. unwrap_or_default ()
}
#[cfg(test)]
mod tests {
use super ::* ;
use axum ::http ::{ HeaderMap , Method , Uri };
use std ::sync ::{ Mutex , OnceLock };
fn env_lock () -> & 'static Mutex < () > {
static LOCK : OnceLock < Mutex < () >> = OnceLock ::new ();
LOCK . get_or_init ( || Mutex ::new (()))
}
fn test_context () -> RequestContext {
RequestContext ::from_http_parts ( & Method ::GET , & Uri ::from_static ( "/" ), & HeaderMap ::new ())
}
fn temp_root ( label : & str ) -> PathBuf {
let root = std ::env ::temp_dir (). join ( format! ( " {label} - {} " , now_ms ()));
fs ::create_dir_all ( & root ). expect ( "temp root" );
root
}
fn test_registry_entry (
root : & Path ,
relative : & str ,
doc_id : Option <& str > ,
provider_status : Option <& str > ,
indexed_at_ms : Option < u128 > ,
deleted_at_ms : Option < u128 > ,
stale : bool ,
) -> KnowledgeRagSourceRegistryEntry {
KnowledgeRagSourceRegistryEntry {
source_id : format ! ( "src-{relative}" ),
workspace_id : "ws" . into (),
root_uri : format ! ( "file://{}" , root . display ()),
source_path : root . join ( relative ). display (). to_string (),
source_root_relative_path : relative . into (),
source_hash : "mnote-fnv64:test" . into (),
light_rag_doc_id : doc_id . map ( ToOwned ::to_owned ),
light_rag_status : provider_status . map ( ToOwned ::to_owned ),
light_rag_file_path : relative . into (),
symlink_path : format ! ( "/tmp/{relative}" ),
parser_hint : None ,
indexed_at_ms ,
deleted_at_ms ,
stale ,
updated_at_ms : 2 ,
}
}
#[tokio::test]
async fn retired_local_ocr_endpoint_points_to_lightrag_replacements () {
let ( status , Json ( payload )) = retired_local_ocr_endpoint ( Extension ( test_context ())). await ;
assert_eq! ( status , StatusCode ::GONE );
assert_eq! ( payload [ "code" ], "mnote_local_ocr_retired" );
assert_eq! ( payload [ "replacement" ][ "provider" ], "lightrag" );
assert_eq! (
payload [ "replacement" ][ "ingest" ],
"/api/knowledge-rag/ingest"
);
}
#[test]
fn lightrag_document_summaries_expose_status_groups_for_ui_sync () {
let docs = json! ({
"statuses" : {
"processed" : [
{ "id" : "doc-ok" , "file_path" : "ok.pdf" , "status" : "processed" , "summary" : "ok" }
],
"failed" : [
{ "id" : "doc-fail" , "file_path" : "fail.pdf" , "status" : "failed" , "summary" : "bad" }
]
}
});
let counts = lightrag_document_status_group_counts ( & docs );
assert_eq! ( counts . get ( "processed" ), Some ( & 1 ));
assert_eq! ( counts . get ( "failed" ), Some ( & 1 ));
let summaries = lightrag_document_summaries ( & docs );
assert_eq! ( summaries . len (), 2 );
assert! ( summaries . iter (). any ( | item | item [ "filePath" ] == "ok.pdf" ));
assert! ( summaries . iter (). any ( | item | item [ "statusGroup" ] == "failed" ));
let by_path = lightrag_documents_by_file_path ( & docs );
assert! ( by_path . contains_key ( "ok.pdf" ));
assert! ( by_path . contains_key ( "fail.pdf" ));
}
#[test]
fn reference_mapping_uses_registry_source_path () {
let registry = KnowledgeRagSourceRegistry {
schema : REGISTRY_SCHEMA . to_string (),
workspace_id : "ws" . into (),
root_uri : "file:///tmp/root" . into (),
updated_at_ms : 1 ,
entries : vec ! [ KnowledgeRagSourceRegistryEntry {
source_id : "src1" . into (),
workspace_id : "ws" . into (),
root_uri : "file:///tmp/root" . into (),
source_path : "/tmp/root/books/a.pdf" . into (),
source_root_relative_path : "books/a.pdf" . into (),
source_hash : "mnote-fnv64:1" . into (),
light_rag_doc_id : Some ( "doc1" . into ()),
light_rag_status : Some ( "processed" . into ()),
light_rag_file_path : "mnote-hash-a.pdf" . into (),
symlink_path : "/tmp/input/mnote-hash-a.pdf" . into (),
parser_hint : None ,
indexed_at_ms : Some ( 2 ),
deleted_at_ms : None ,
stale : false ,
updated_at_ms : 2 ,
}],
};
let mapped = map_reference_plan (
& json! ({ "file_path" :"mnote-hash-a.pdf" , "chunk_id" :"chunk1" }),
& registry ,
"file:///tmp/root" ,
Path ::new ( "/tmp/root" ),
);
assert_eq! ( mapped [ "sourceRootRelativePath" ], "books/a.pdf" );
assert_eq! ( mapped [ "locatorDegraded" ], true );
assert_eq! ( mapped [ "openAction" ][ "params" ][ "path" ], "books/a.pdf" );
}
#[test]
fn missing_registry_reference_is_degraded () {
let registry = KnowledgeRagSourceRegistry {
schema : REGISTRY_SCHEMA . to_string (),
workspace_id : "ws" . into (),
root_uri : "file:///tmp/root" . into (),
updated_at_ms : 1 ,
entries : vec ! [],
};
let mapped = map_reference_plan (
& json! ({ "file_path" :"unknown.pdf" }),
& registry ,
"file:///tmp/root" ,
Path ::new ( "/tmp/root" ),
);
assert_eq! ( mapped [ "locatorDegraded" ], true );
assert! ( mapped [ "citationMarkdown" ]
. as_str ()
. unwrap ()
. contains ( "来源定位降级" ));
}
#[test]
fn reference_mapping_does_not_match_deleted_entry_when_doc_id_missing () {
let registry = KnowledgeRagSourceRegistry {
schema : REGISTRY_SCHEMA . to_string (),
workspace_id : "ws" . into (),
root_uri : "file:///tmp/root" . into (),
updated_at_ms : 1 ,
entries : vec ! [ KnowledgeRagSourceRegistryEntry {
source_id : "deleted" . into (),
workspace_id : "ws" . into (),
root_uri : "file:///tmp/root" . into (),
source_path : "/tmp/root/deleted.md" . into (),
source_root_relative_path : "deleted.md" . into (),
source_hash : "mnote-fnv64:1" . into (),
light_rag_doc_id : None ,
light_rag_status : None ,
light_rag_file_path : "deleted.md" . into (),
symlink_path : "/tmp/input/deleted.md" . into (),
parser_hint : None ,
indexed_at_ms : None ,
deleted_at_ms : Some ( 2 ),
stale : true ,
updated_at_ms : 2 ,
}],
};
let mapped = map_reference_plan (
& json! ({ "file_path" :"active.md" , "chunk_id" :"chunk-active" }),
& registry ,
"file:///tmp/root" ,
Path ::new ( "/tmp/root" ),
);
assert! ( mapped [ "sourceRootRelativePath" ]. is_null ());
assert_eq! ( mapped [ "stale" ], false );
assert_eq! ( mapped [ "deleted" ], false );
}
#[test]
fn degraded_known_resource_keeps_clickable_resource_tab_url () {
let root = temp_root ( "mnote-knowledge-rag-docx-fallback" );
fs ::create_dir_all ( root . join ( "docs" )). expect ( "docs" );
fs ::write ( root . join ( "docs" ). join ( "Host.md" ), "# Host \n " ). expect ( "host" );
fs ::write ( root . join ( "docs" ). join ( "a.docx" ), b "docx" ). expect ( "docx" );
let registry = KnowledgeRagSourceRegistry {
schema : REGISTRY_SCHEMA . to_string (),
workspace_id : "ws" . into (),
root_uri : "file:///tmp/root" . into (),
updated_at_ms : 1 ,
entries : vec ! [ KnowledgeRagSourceRegistryEntry {
source_id : "src1" . into (),
workspace_id : "ws" . into (),
root_uri : "file:///tmp/root" . into (),
source_path : root . join ( "docs" ). join ( "a.docx" ). display (). to_string (),
source_root_relative_path : "docs/a.docx" . into (),
source_hash : "mnote-fnv64:1" . into (),
light_rag_doc_id : Some ( "doc1" . into ()),
light_rag_status : Some ( "processed" . into ()),
light_rag_file_path : "mnote-hash-a.docx" . into (),
symlink_path : "/tmp/input/mnote-hash-a.docx" . into (),
parser_hint : None ,
indexed_at_ms : Some ( 2 ),
deleted_at_ms : None ,
stale : false ,
updated_at_ms : 2 ,
}],
};
let mapped = map_reference_plan (
& json! ({ "file_path" :"mnote-hash-a.docx" , "chunk_id" :"chunk1" }),
& registry ,
"file:///tmp/root" ,
& root ,
);
assert_eq! ( mapped [ "locatorDegraded" ], true );
let citation_url = mapped [ "citationUrl" ]. as_str (). expect ( "citation url" );
assert! ( citation_url . starts_with ( "/documents/local-md:docs~2FHost.md?" ));
assert! ( citation_url . contains ( "resourceTab=" ));
assert! ( citation_url . contains ( "resourcePath=docs%2Fa.docx" ));
assert! ( mapped [ "citationMarkdown" ]
. as_str ()
. unwrap ()
. contains ( "来源定位降级" ));
let _ = fs ::remove_dir_all ( root );
}
#[test]
fn source_state_marks_deleted_and_changed_entries_stale () {
let root = temp_root ( "mnote-knowledge-rag-source-state" );
fs ::create_dir_all ( root . join ( "docs" )). expect ( "docs" );
let changed_path = root . join ( "docs" ). join ( "changed.md" );
fs ::write ( & changed_path , "new content" ). expect ( "changed" );
let missing_path = root . join ( "docs" ). join ( "missing.md" );
let mut registry = KnowledgeRagSourceRegistry {
schema : REGISTRY_SCHEMA . to_string (),
workspace_id : "ws" . into (),
root_uri : "file:///tmp/root" . into (),
updated_at_ms : 1 ,
entries : vec ! [
KnowledgeRagSourceRegistryEntry {
source_id : "missing" . into (),
workspace_id : "ws" . into (),
root_uri : "file:///tmp/root" . into (),
source_path : missing_path . display (). to_string (),
source_root_relative_path : "docs/missing.md" . into (),
source_hash : "mnote-fnv64:old" . into (),
light_rag_doc_id : Some ( "doc-missing" . into ()),
light_rag_status : Some ( "processed" . into ()),
light_rag_file_path : "missing.md" . into (),
symlink_path : "/tmp/input/missing.md" . into (),
parser_hint : None ,
indexed_at_ms : Some ( 2 ),
deleted_at_ms : None ,
stale : false ,
updated_at_ms : 2 ,
},
KnowledgeRagSourceRegistryEntry {
source_id : "changed" . into (),
workspace_id : "ws" . into (),
root_uri : "file:///tmp/root" . into (),
source_path : changed_path . display (). to_string (),
source_root_relative_path : "docs/changed.md" . into (),
source_hash : "mnote-fnv64:old" . into (),
light_rag_doc_id : Some ( "doc-changed" . into ()),
light_rag_status : Some ( "processed" . into ()),
light_rag_file_path : "changed.md" . into (),
symlink_path : "/tmp/input/changed.md" . into (),
parser_hint : None ,
indexed_at_ms : Some ( 2 ),
deleted_at_ms : None ,
stale : false ,
updated_at_ms : 2 ,
},
],
};
let stale_doc_ids = sync_registry_source_state ( & mut registry , 42 ). expect ( "sync" );
assert_eq! ( stale_doc_ids , vec! [ "doc-changed" , "doc-missing" ]);
assert_eq! ( registry . entries [ 0 ]. deleted_at_ms , Some ( 42 ));
2026-06-07 10:35:21 +08:00
assert_eq! (
registry . entries [ 0 ]. light_rag_doc_id . as_deref (),
Some ( "doc-missing" )
);
assert_eq! (
registry . entries [ 0 ]. light_rag_status . as_deref (),
Some ( "delete_submitted" )
);
2026-06-07 01:10:31 +08:00
assert_eq! ( registry . entries [ 0 ]. indexed_at_ms , None );
assert! ( registry . entries [ 0 ]. stale );
assert_eq! ( registry . entries [ 1 ]. deleted_at_ms , None );
2026-06-07 10:35:21 +08:00
assert_eq! (
registry . entries [ 1 ]. light_rag_doc_id . as_deref (),
Some ( "doc-changed" )
);
assert_eq! (
registry . entries [ 1 ]. light_rag_status . as_deref (),
Some ( "delete_submitted" )
);
2026-06-07 01:10:31 +08:00
assert_eq! ( registry . entries [ 1 ]. indexed_at_ms , None );
assert! ( registry . entries [ 1 ]. stale );
assert_ne! ( registry . entries [ 1 ]. source_hash , "mnote-fnv64:old" );
let _ = fs ::remove_dir_all ( root );
}
2026-06-07 10:35:21 +08:00
#[test]
fn provider_delete_confirmation_clears_doc_id_for_deleted_or_stale_entries () {
let root = temp_root ( "mnote-knowledge-rag-delete-confirmed" );
let mut deleted = test_registry_entry (
& root ,
"deleted.pdf" ,
Some ( "doc-deleted" ),
Some ( "delete_submitted" ),
Some ( 2 ),
Some ( 3 ),
true ,
);
let mut changed = test_registry_entry (
& root ,
"changed.pdf" ,
Some ( "doc-changed" ),
Some ( "delete_submitted" ),
Some ( 2 ),
None ,
true ,
);
let active = test_registry_entry (
& root ,
"active.pdf" ,
Some ( "doc-active" ),
Some ( "processed" ),
Some ( 2 ),
None ,
false ,
);
assert! ( knowledge_rag_provider_delete_confirmed ( & deleted ));
mark_registry_entry_delete_completed ( & mut deleted , 42 );
assert_eq! ( deleted . light_rag_doc_id , None );
assert_eq! ( deleted . indexed_at_ms , None );
assert_eq! (
deleted . light_rag_status . as_deref (),
Some ( "delete_completed" )
);
assert! ( knowledge_rag_provider_delete_confirmed ( & changed ));
mark_registry_entry_delete_completed ( & mut changed , 43 );
assert_eq! ( changed . light_rag_doc_id , None );
assert_eq! (
changed . light_rag_status . as_deref (),
Some ( "delete_completed" )
);
assert! ( ! knowledge_rag_provider_delete_confirmed ( & active ));
assert_eq! ( active . light_rag_doc_id . as_deref (), Some ( "doc-active" ));
let _ = fs ::remove_dir_all ( root );
}
2026-06-07 01:10:31 +08:00
#[test]
fn source_statuses_distinguish_indexed_processing_failed_and_deleted () {
let root = temp_root ( "mnote-knowledge-rag-source-statuses" );
fs ::write ( root . join ( "indexed.pdf" ), b "indexed" ). expect ( "indexed" );
fs ::write ( root . join ( "submitted.pdf" ), b "submitted" ). expect ( "submitted" );
fs ::write ( root . join ( "failed.pdf" ), b "failed" ). expect ( "failed" );
fs ::write ( root . join ( "deleting.pdf" ), b "deleting" ). expect ( "deleting" );
fs ::write ( root . join ( "removed.pdf" ), b "removed" ). expect ( "removed" );
let root_uri = format! ( "file:// {} " , root . display ());
let mut registry = KnowledgeRagSourceRegistry {
schema : REGISTRY_SCHEMA . to_string (),
workspace_id : "ws" . into (),
root_uri : root_uri . clone (),
updated_at_ms : 1 ,
entries : vec ! [
test_registry_entry (
& root ,
"indexed.pdf" ,
Some ( "doc-indexed" ),
Some ( "processed" ),
Some ( 2 ),
None ,
false ,
),
test_registry_entry (
& root ,
"submitted.pdf" ,
None ,
Some ( "submitted" ),
None ,
None ,
false ,
),
test_registry_entry ( & root , "failed.pdf" , None , Some ( "failed" ), None , None , false ),
test_registry_entry (
& root ,
"deleting.pdf" ,
Some ( "doc-deleting" ),
Some ( "delete_submitted" ),
Some ( 2 ),
Some ( 3 ),
true ,
),
test_registry_entry (
& root ,
"removed.pdf" ,
None ,
Some ( "delete_completed" ),
None ,
Some ( 3 ),
true ,
),
],
};
write_registry ( & root , & mut registry ). expect ( "write registry" );
let statuses = knowledge_rag_source_statuses ( & root , "ws" , & root_uri ). expect ( "statuses" );
assert! ( statuses . indexed_paths . contains ( "indexed.pdf" ));
assert! ( statuses . indexing_paths . contains ( "submitted.pdf" ));
assert! ( statuses . indexing_paths . contains ( "deleting.pdf" ));
assert! ( statuses . failed_paths . contains ( "failed.pdf" ));
assert! ( ! statuses . indexed_paths . contains ( "removed.pdf" ));
assert! ( ! statuses . indexing_paths . contains ( "removed.pdf" ));
assert! ( ! statuses . failed_paths . contains ( "removed.pdf" ));
let _ = fs ::remove_dir_all ( root );
}
#[test]
fn image_source_is_staged_as_markdown_wrapper_for_lightrag_scan () {
let root = temp_root ( "mnote-knowledge-rag-image-wrapper" );
let input_dir = root . join ( "inputs" );
fs ::create_dir_all ( & input_dir ). expect ( "input dir" );
let image_path = root . join ( "image copy 6.png" );
fs ::write ( & image_path , b "png" ). expect ( "image" );
let staged = stage_lightrag_source (
& image_path ,
"image copy 6.png" ,
None ,
& input_dir ,
& test_context (),
)
. expect ( "stage image" );
assert! ( staged . light_rag_file_path . ends_with ( ".png.md" ));
let wrapper = fs ::read_to_string ( & staged . staged_path ). expect ( "wrapper" );
assert! ( wrapper . contains ( "![image copy 6.png]" ));
assert! ( wrapper . contains ( ".png>)" ));
let asset_name = staged . light_rag_file_path . trim_end_matches ( ".md" );
assert! ( input_dir . join ( asset_name ). symlink_metadata (). is_ok ());
let _ = fs ::remove_dir_all ( root );
}
#[test]
fn registry_prune_predicate_keeps_inflight_delete_until_confirmed () {
let root = temp_root ( "mnote-knowledge-rag-prune-predicate" );
let active = test_registry_entry (
& root ,
"active.pdf" ,
Some ( "doc-active" ),
Some ( "processed" ),
Some ( 2 ),
None ,
false ,
);
let deleting = test_registry_entry (
& root ,
"deleting.pdf" ,
Some ( "doc-deleting" ),
Some ( "delete_submitted" ),
Some ( 2 ),
Some ( 3 ),
true ,
);
let removed = test_registry_entry (
& root ,
"removed.pdf" ,
None ,
Some ( "delete_completed" ),
None ,
Some ( 3 ),
true ,
);
let failed =
test_registry_entry ( & root , "failed.pdf" , None , Some ( "failed" ), None , None , false );
2026-06-07 10:35:21 +08:00
let retry = test_registry_entry (
& root ,
"retry.pdf" ,
Some ( "doc-retry" ),
Some ( "delete_retry_required" ),
Some ( 2 ),
Some ( 3 ),
true ,
);
let stale_with_doc = test_registry_entry (
& root ,
"stale.pdf" ,
Some ( "doc-stale" ),
Some ( "processed" ),
Some ( 2 ),
None ,
true ,
);
2026-06-07 01:10:31 +08:00
assert! ( ! knowledge_rag_registry_entry_prunable ( & active ));
assert! ( ! knowledge_rag_registry_entry_prunable ( & deleting ));
2026-06-07 10:35:21 +08:00
assert! ( ! knowledge_rag_registry_entry_prunable ( & retry ));
assert! ( ! knowledge_rag_registry_entry_prunable ( & stale_with_doc ));
2026-06-07 01:10:31 +08:00
assert! ( knowledge_rag_registry_entry_prunable ( & removed ));
assert! ( knowledge_rag_registry_entry_prunable ( & failed ));
let _ = fs ::remove_dir_all ( root );
}
#[test]
fn source_scope_filters_mapped_references_by_file_or_directory () {
let mut references = vec! [
json! ({ "sourceRootRelativePath" : "books/a.pdf" }),
json! ({ "sourceRootRelativePath" : "papers/one/b.pdf" }),
json! ({ "sourceRootRelativePath" : "papers/two/c.pdf" }),
];
let scope = normalize_source_scope ( Some ( & [
"books/a.pdf" . to_string (),
"papers/one" . to_string (),
"papers/one" . to_string (),
]));
assert_eq! ( scope , vec! [ "books/a.pdf" , "papers/one" ]);
filter_mapped_references_by_source_scope ( & mut references , & scope );
let paths = references
. iter ()
. filter_map ( | reference | reference [ "sourceRootRelativePath" ]. as_str ())
. collect ::< Vec < _ >> ();
assert_eq! ( paths , vec! [ "books/a.pdf" , "papers/one/b.pdf" ]);
}
#[test]
fn chunks_without_references_become_reference_candidates () {
let raw = json! ({
"data" : {
"chunks" : [
{ "file_path" : "a.md" , "reference_id" : "1" , "chunk_id" : "chunk-a" , "content" : "A" },
{ "file_path" : "b.md" , "reference_id" : "2" , "chunk_id" : "chunk-b" , "content" : "B" }
]
}
});
let references = reference_array ( & raw );
assert_eq! ( references . len (), 2 );
assert_eq! ( references [ 0 ][ "file_path" ]. as_str (), Some ( "a.md" ));
assert! ( references [ 0 ][ "chunks" ]
. as_array ()
. is_some_and ( | items | items . len () == 1 ));
}
#[test]
fn mapped_references_include_chunk_quote () {
let registry = KnowledgeRagSourceRegistry {
schema : REGISTRY_SCHEMA . to_string (),
workspace_id : "ws" . into (),
root_uri : "file:///tmp/root" . into (),
updated_at_ms : 1 ,
entries : vec ! [ KnowledgeRagSourceRegistryEntry {
source_id : "src1" . into (),
workspace_id : "ws" . into (),
root_uri : "file:///tmp/root" . into (),
source_path : "/tmp/root/papers/a.pdf" . into (),
source_root_relative_path : "papers/a.pdf" . into (),
source_hash : "mnote-fnv64:1" . into (),
light_rag_doc_id : Some ( "doc1" . into ()),
light_rag_status : Some ( "processed" . into ()),
light_rag_file_path : "mnote-hash-a.pdf" . into (),
symlink_path : "/tmp/input/mnote-hash-a.pdf" . into (),
parser_hint : None ,
indexed_at_ms : Some ( 2 ),
deleted_at_ms : None ,
stale : false ,
updated_at_ms : 2 ,
}],
};
let raw = json! ({
"data" : {
"references" : [{ "reference_id" :"1" , "file_path" :"mnote-hash-a.pdf" }],
"chunks" : [{
"reference_id" :"1" ,
"chunk_id" :"doc1-chunk-000" ,
"file_path" :"mnote-hash-a.pdf" ,
"content" :"chunk quote from LightRAG"
}]
}
});
let mapped = mapped_references ( & raw , & registry , "file:///tmp/root" , Path ::new ( "/tmp/root" ));
assert_eq! ( mapped [ 0 ][ "chunkId" ], "doc1-chunk-000" );
assert_eq! ( mapped [ 0 ][ "quote" ], "chunk quote from LightRAG" );
assert_eq! (
mapped [ 0 ][ "openAction" ][ "params" ][ "chunkId" ],
"doc1-chunk-000"
);
}
2026-06-07 10:35:21 +08:00
#[test]
fn mapped_references_filters_unmapped_provider_references () {
let registry = KnowledgeRagSourceRegistry {
schema : REGISTRY_SCHEMA . to_string (),
workspace_id : "ws" . into (),
root_uri : "file:///tmp/root" . into (),
updated_at_ms : 1 ,
entries : vec ! [],
};
let raw = json! ({
"data" : {
"references" : [{ "reference_id" :"1" , "file_path" :"orphan.pdf" }],
"chunks" : [{
"reference_id" :"1" ,
"chunk_id" :"orphan-chunk" ,
"file_path" :"orphan.pdf" ,
"content" :"orphan provider chunk"
}]
}
});
let mapped = mapped_references ( & raw , & registry , "file:///tmp/root" , Path ::new ( "/tmp/root" ));
assert! (
mapped . is_empty (),
"unmapped provider references must not become MNote citations"
);
let plan = map_reference_plan (
& json! ({ "file_path" :"orphan.pdf" , "chunk_id" :"orphan-chunk" }),
& registry ,
"file:///tmp/root" ,
Path ::new ( "/tmp/root" ),
);
assert_eq! ( plan [ "unmapped" ], true );
assert_eq! ( plan [ "locatorDegraded" ], true );
}
2026-06-07 01:10:31 +08:00
#[test]
fn query_ranking_prefers_exact_source_and_quote_match () {
let mut references = vec! [
json! ({
"sourceRootRelativePath" : "knowledge-rag-fixtures-7-50/paper-brotli-comparison.pdf" ,
"filePath" : "mnote-paper-brotli-comparison.pdf" ,
"quote" : "brotli compression table" ,
"citationMarkdown" : "[paper-brotli-comparison.pdf · p.3](/)"
}),
json! ({
"sourceRootRelativePath" : "knowledge-rag-fixtures-7-50/scan-image-start.pdf" ,
"filePath" : "mnote-scan-image-start.pdf" ,
"quote" : "# START: This is the first image in PDF" ,
"citationMarkdown" : "[scan-image-start.pdf · p.1](/)"
}),
];
rank_mapped_references_for_query ( & mut references , "scan image start" );
assert_eq! (
references [ 0 ][ "sourceRootRelativePath" ],
"knowledge-rag-fixtures-7-50/scan-image-start.pdf"
);
}
#[test]
fn query_mode_aliases_hybrid_to_mix_for_lightrag () {
assert_eq! ( normalize_lightrag_query_mode ( Some ( "hybrid" )), "mix" );
assert_eq! ( normalize_lightrag_query_mode ( Some ( "global" )), "global" );
assert_eq! ( normalize_lightrag_query_mode ( None ), "mix" );
}
#[test]
fn symlink_name_preserves_parser_hint_before_extension () {
let name =
lightrag_symlink_name ( Path ::new ( "/tmp/books/demo.pdf" ), "demo.pdf" , Some ( "mineru" ));
assert! ( name . ends_with ( "-demo.[mineru].pdf" ));
}
#[test]
fn symlink_name_does_not_duplicate_existing_parser_hint () {
let name = lightrag_symlink_name (
Path ::new ( "/tmp/books/demo.[native].pdf" ),
"demo.[native].pdf" ,
Some ( "native" ),
);
assert! ( name . ends_with ( "-demo.[native].pdf" ));
assert! ( ! name . contains ( ".[native].[native]." ));
}
#[test]
fn registry_file_path_matches_lightrag_hint_canonicalization () {
let entry = KnowledgeRagSourceRegistryEntry {
source_id : "src1" . into (),
workspace_id : "ws" . into (),
root_uri : "file:///tmp/root" . into (),
source_path : "/tmp/root/scan.[mineru].pdf" . into (),
source_root_relative_path : "scan.[mineru].pdf" . into (),
source_hash : "mnote-fnv64:1" . into (),
light_rag_doc_id : None ,
light_rag_status : None ,
light_rag_file_path : "mnote-hash-scan.[mineru].[mineru].pdf" . into (),
symlink_path : "/tmp/input/mnote-hash-scan.[mineru].[mineru].pdf" . into (),
parser_hint : Some ( "mineru" . into ()),
indexed_at_ms : None ,
deleted_at_ms : None ,
stale : false ,
updated_at_ms : 2 ,
};
assert! ( lightrag_file_path_matches (
& entry ,
"mnote-hash-scan.[mineru].pdf"
));
}
#[test]
fn locator_uses_stripped_parser_hint_sidecar_path () {
let _guard = env_lock (). lock (). expect ( "env lock" );
let root = temp_root ( "mnote-knowledge-rag-sidecar" );
let input_dir = root . join ( "inputs" );
let parsed_dir = input_dir
. join ( "__parsed__" )
. join ( "mnote-hash-scan.pdf.parsed" );
fs ::create_dir_all ( & parsed_dir ). expect ( "parsed dir" );
fs ::write (
parsed_dir . join ( "mnote-hash-scan.blocks.jsonl" ),
[
r #"{"type":"meta","blocks":1}"# ,
r ##"{"type":"content","blockid":"block1","content":"# START: This is the first image in PDF\nThis is text BEFORE the image.","positions":[{"type":"bbox","anchor":"1","range":[171.0,126.0,648.0,152.0]}]}"## ,
]
. join ( " \n " ),
)
. expect ( "blocks" );
std ::env ::set_var ( "MNOTE_LIGHTRAG_INPUT_DIR" , & input_dir );
let entry = KnowledgeRagSourceRegistryEntry {
source_id : "src1" . into (),
workspace_id : "ws" . into (),
root_uri : "file:///tmp/root" . into (),
source_path : "/tmp/root/scan.pdf" . into (),
source_root_relative_path : "scan.pdf" . into (),
source_hash : "mnote-fnv64:1" . into (),
light_rag_doc_id : Some ( "doc1" . into ()),
light_rag_status : Some ( "processed" . into ()),
light_rag_file_path : "mnote-hash-scan.[mineru].pdf" . into (),
symlink_path : input_dir
. join ( "mnote-hash-scan.[mineru].pdf" )
. display ()
. to_string (),
parser_hint : Some ( "mineru" . into ()),
indexed_at_ms : Some ( 2 ),
deleted_at_ms : None ,
stale : false ,
updated_at_ms : 2 ,
};
let locator = lightrag_locator_for_reference (
Path ::new ( "/tmp/root" ),
"file:///tmp/root" ,
& entry ,
& json! ( "doc1-chunk-000" ),
"# START: This is the first image in PDF \n This is text BEFORE the image." ,
)
. expect ( "locator" );
assert_eq! ( locator . page , Some ( 1 ));
assert_eq! ( locator . bbox . as_ref (). map ( | bbox | bbox . x0 ), Some ( 171.0 ));
std ::env ::remove_var ( "MNOTE_LIGHTRAG_INPUT_DIR" );
2026-06-07 10:35:21 +08:00
let _ = fs ::remove_dir_all ( root );
}
#[test]
fn locator_degrades_when_sidecar_quote_does_not_match () {
let _guard = env_lock (). lock (). expect ( "env lock" );
let root = temp_root ( "mnote-knowledge-rag-sidecar-no-match" );
fs ::create_dir_all ( root . join ( "docs" )). expect ( "docs" );
fs ::write ( root . join ( "docs" ). join ( "Host.md" ), "# Host \n " ). expect ( "host" );
fs ::write ( root . join ( "docs" ). join ( "scan.pdf" ), b "pdf" ). expect ( "pdf" );
let input_dir = root . join ( "inputs" );
let parsed_dir = input_dir
. join ( "__parsed__" )
. join ( "mnote-hash-scan.pdf.parsed" );
fs ::create_dir_all ( & parsed_dir ). expect ( "parsed dir" );
fs ::write (
parsed_dir . join ( "mnote-hash-scan.blocks.jsonl" ),
[
r #"{"type":"meta","blocks":1}"# ,
r ##"{"type":"content","blockid":"block1","content":"This block is not the returned quote.","positions":[{"type":"bbox","anchor":"9","range":[1.0,2.0,3.0,4.0]}]}"## ,
]
. join ( " \n " ),
)
. expect ( "blocks" );
std ::env ::set_var ( "MNOTE_LIGHTRAG_INPUT_DIR" , & input_dir );
let registry = KnowledgeRagSourceRegistry {
schema : REGISTRY_SCHEMA . to_string (),
workspace_id : "ws" . into (),
root_uri : "file:///tmp/root" . into (),
updated_at_ms : 1 ,
entries : vec ! [ KnowledgeRagSourceRegistryEntry {
source_id : "src1" . into (),
workspace_id : "ws" . into (),
root_uri : "file:///tmp/root" . into (),
source_path : root . join ( "docs" ). join ( "scan.pdf" ). display (). to_string (),
source_root_relative_path : "docs/scan.pdf" . into (),
source_hash : "mnote-fnv64:1" . into (),
light_rag_doc_id : Some ( "doc1" . into ()),
light_rag_status : Some ( "processed" . into ()),
light_rag_file_path : "mnote-hash-scan.pdf" . into (),
symlink_path : input_dir . join ( "mnote-hash-scan.pdf" ). display (). to_string (),
parser_hint : None ,
indexed_at_ms : Some ( 2 ),
deleted_at_ms : None ,
stale : false ,
updated_at_ms : 2 ,
}],
};
let mapped = map_reference_plan (
& json! ({
"file_path" : "mnote-hash-scan.pdf" ,
"chunk_id" : "doc1-chunk-000" ,
"chunks" : [{ "chunk_id" : "doc1-chunk-000" , "content" : "A different quote should not get page or bbox." }]
}),
& registry ,
"file:///tmp/root" ,
& root ,
);
assert! ( mapped [ "locator" ]. is_null ());
assert_eq! ( mapped [ "locatorDegraded" ], true );
assert! ( mapped [ "citationUrl" ]
. as_str ()
. is_some_and ( | url | url . contains ( "resourceTab=" )));
assert! ( mapped [ "citationMarkdown" ]
. as_str ()
. unwrap ()
. contains ( "来源定位降级" ));
std ::env ::remove_var ( "MNOTE_LIGHTRAG_INPUT_DIR" );
2026-06-07 01:10:31 +08:00
let _ = fs ::remove_dir_all ( root );
}
#[test]
fn lightrag_paths_prefer_source_env_over_legacy_process_env () {
let _guard = env_lock (). lock (). expect ( "env lock" );
let root = temp_root ( "mnote-knowledge-rag-env" );
let source_input = root . join ( "source-inputs" );
let source_working = root . join ( "source-storage" );
let legacy_input = root . join ( "legacy-inputs" );
let legacy_working = root . join ( "legacy-storage" );
let env_file = root . join ( "lightrag.env" );
fs ::write (
& env_file ,
format! (
"INPUT_DIR= {} \n WORKING_DIR= {} \n " ,
source_input . display (),
source_working . display ()
),
)
. expect ( "env file" );
std ::env ::set_var ( "MNOTE_LIGHTRAG_ENV_FILE" , & env_file );
std ::env ::set_var ( "INPUT_DIR" , & legacy_input );
std ::env ::set_var ( "WORKING_DIR" , & legacy_working );
std ::env ::remove_var ( "MNOTE_LIGHTRAG_INPUT_DIR" );
std ::env ::remove_var ( "MNOTE_LIGHTRAG_WORKING_DIR" );
assert_eq! ( lightrag_input_dir (), source_input );
assert_eq! ( lightrag_working_dir (), source_working );
std ::env ::remove_var ( "MNOTE_LIGHTRAG_ENV_FILE" );
std ::env ::remove_var ( "INPUT_DIR" );
std ::env ::remove_var ( "WORKING_DIR" );
let _ = fs ::remove_dir_all ( root );
}
#[test]
fn directory_source_expands_supported_files_and_skips_internal_dirs () {
let root = temp_root ( "mnote-knowledge-rag-directory" );
fs ::create_dir_all ( root . join ( "docs" ). join ( "nested" )). expect ( "nested" );
fs ::create_dir_all ( root . join ( ".mnote" ). join ( "index" )). expect ( "mnote" );
fs ::write ( root . join ( "docs" ). join ( "a.pdf" ), b "%PDF" ). expect ( "pdf" );
fs ::write ( root . join ( "docs" ). join ( "nested" ). join ( "b.md" ), "# B" ). expect ( "md" );
fs ::write ( root . join ( "docs" ). join ( "ignored.tmp" ), "tmp" ). expect ( "tmp" );
fs ::write (
root . join ( ".mnote" ). join ( "index" ). join ( "hidden.pdf" ),
b "%PDF" ,
)
. expect ( "hidden" );
let sources =
resolve_knowledge_rag_sources ( & root , "docs" , & test_context ()). expect ( "sources" );
let rel = sources
. iter ()
. map ( | source | root_relative_path ( & root , & source . canonical_path ). expect ( "relative" ))
. collect ::< Vec < _ >> ();
assert_eq! ( rel , vec! [ "docs/a.pdf" , "docs/nested/b.md" ]);
assert! ( sources
. iter ()
. all ( | source | source . source_kind == "directory" ));
let _ = fs ::remove_dir_all ( root );
}
#[test]
fn file_source_rejects_unsupported_extension () {
let root = temp_root ( "mnote-knowledge-rag-unsupported" );
fs ::write ( root . join ( "notes.tmp" ), "tmp" ). expect ( "tmp" );
let error = resolve_knowledge_rag_sources ( & root , "notes.tmp" , & test_context ())
. expect_err ( "unsupported" );
assert_eq! ( error . code (), "knowledge_rag_source_unsupported" );
let _ = fs ::remove_dir_all ( root );
}
}