use crate::error::WebError; use crate::routes::local_folder_source::encode_local_id_segment; use crate::routes::local_markdown_parser::{ parse_markdown_attachment_link, parse_markdown_page, split_frontmatter, }; use crate::routes::local_ocr; use control_plane::{ControlPlaneStore, UpsertUserInput, UpsertUserUiPreferenceInput}; use core_protocol::{ EvidenceLocator, EvidenceSearchMatchInfo, EvidenceSearchResult, ParsedResourceArtifact, ResourceSourceMap, SourceMapBlock, PARSED_RESOURCE_ARTIFACT_SCHEMA, }; use rusqlite::{params, Connection, OptionalExtension}; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; use std::collections::BTreeSet; use std::fs; use std::path::{Component, Path, PathBuf}; use std::time::{SystemTime, UNIX_EPOCH}; use time::{Date, Duration, Month, OffsetDateTime, Time}; const LOCAL_SEARCH_INDEX_VERSION: u32 = 1; const LOCAL_INDEX_SETTINGS_SCHEMA: &str = "mnote.local_index.settings.v1"; const LOCAL_INDEX_SETTINGS_KEY: &str = "localIndex.settings"; const LOCAL_INDEX_SETTINGS_SCOPE_KIND: &str = "localIndex"; const LOCAL_INDEX_SOURCE_KIND: &str = "local_folder"; const DEFAULT_INDEX_SCHEDULE_MODE: &str = "daily"; const DEFAULT_INDEX_SCHEDULE_TIME: &str = "02:00"; const EVIDENCE_SQLITE_SCHEMA_VERSION: u32 = 1; fn local_index_scope_id(root_path: &Path) -> String { root_path.to_string_lossy().replace('\\', "/") } #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] struct LocalSearchIndex { version: u32, built_at: u128, root_uri: String, workspace_id: String, #[serde(default = "default_indexed_paths")] indexed_paths: Vec, documents: Vec, #[serde(default)] resources: Vec, } #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub(crate) struct LocalIndexSettings { pub schema: String, pub include_paths: Vec, #[serde(default = "default_index_schedule_mode")] pub schedule_mode: String, #[serde(default = "default_index_schedule_time")] pub schedule_time: String, #[serde(default)] pub schedule_date: Option, #[serde(default = "default_index_run_on_change")] pub run_on_change: bool, pub updated_at: u128, } #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] struct LocalSearchDocument { document_id: String, title: String, path: String, raw_text: String, tags: Vec, backlinks: Vec, resource_refs: Vec, updated_at: u128, } #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] struct LocalSearchResource { resource_id: String, resource_type: String, title: String, path: String, updated_at: u128, } pub(crate) fn query_local_search_index( root_path: &Path, root_uri: &str, workspace_id: &str, query: &str, page_id: Option<&str>, limit: u32, title_only: bool, exact: bool, include_ocr: bool, ) -> Result { let settings = read_local_index_settings_or_default(root_path)?; query_local_search_index_with_settings( root_path, root_uri, workspace_id, &settings, &settings, query, page_id, limit, title_only, exact, include_ocr, ) } pub(crate) fn query_local_search_index_with_settings( root_path: &Path, root_uri: &str, workspace_id: &str, cache_settings: &LocalIndexSettings, result_settings: &LocalIndexSettings, query: &str, page_id: Option<&str>, limit: u32, title_only: bool, exact: bool, include_ocr: bool, ) -> Result { let index = load_or_rebuild_local_search_index_with_settings( root_path, root_uri, workspace_id, cache_settings, )?; let normalized_query = normalize_search_text(query); let page_id = page_id.map(str::trim).filter(|value| !value.is_empty()); let page_resource_path = page_id.and_then(local_resource_path_from_document_id); let markdown_page_id = if page_resource_path.is_some() { None } else { page_id }; let recent_changes = local_recent_changes_projection(&index.documents, root_uri); let mut results = Vec::new(); for document in index.documents.iter() { if !index_relative_path_is_included(&document.path, &result_settings.include_paths) { continue; } if let Some(page_id) = markdown_page_id { if document.document_id != page_id { continue; } } if !local_search_document_matches(document, &normalized_query, title_only, exact) { continue; } results.push(local_search_document_projection( document, root_uri, &normalized_query, )); if results.len() >= limit.max(1) as usize { break; } } if markdown_page_id.is_none() && results.len() < limit.max(1) as usize { for resource in index.resources.iter() { if !index_relative_path_is_included(&resource.path, &result_settings.include_paths) { continue; } if let Some(resource_path) = page_resource_path.as_deref() { if resource.path != resource_path { continue; } } if !local_search_resource_matches(resource, &normalized_query, title_only, exact) { continue; } results.push(local_search_resource_projection(resource, root_uri)); if results.len() >= limit.max(1) as usize { break; } } } let _ = include_ocr; Ok(json!({ "enqueueAssetIds": [], "projectionOwner": "rust-kernel", "sourceKind": "local_folder", "index": { "version": index.version, "rootUri": index.root_uri, "workspaceId": index.workspace_id, "indexedPaths": index.indexed_paths, "resultIndexedPaths": result_settings.include_paths, "builtAt": index.built_at, "documentCount": index.documents.len(), "resourceCount": index.resources.len(), "includeOcr": include_ocr }, "recentChanges": recent_changes, "results": results })) } #[cfg(test)] pub(crate) fn write_local_index_settings( root_path: &Path, include_paths: &[String], schedule_mode: Option<&str>, schedule_time: Option<&str>, schedule_date: Option<&str>, run_on_change: Option, ) -> Result { let include_paths = normalize_index_include_paths(root_path, include_paths)?; let existing = read_local_index_settings_or_default(root_path) .unwrap_or_else(|_| default_local_index_settings()); let schedule_mode = normalize_index_schedule_mode(schedule_mode.unwrap_or(&existing.schedule_mode))?; let schedule_time = normalize_index_schedule_time(schedule_time.unwrap_or(&existing.schedule_time))?; let schedule_date = normalize_index_schedule_date(schedule_date.or(existing.schedule_date.as_deref()))?; let settings = LocalIndexSettings { schema: LOCAL_INDEX_SETTINGS_SCHEMA.to_string(), include_paths, schedule_mode, schedule_time, schedule_date, run_on_change: run_on_change.unwrap_or(existing.run_on_change), updated_at: now_ms(), }; write_local_index_settings_cache(root_path, &settings)?; Ok(settings) } pub(crate) fn read_user_local_index_settings( store: &dyn ControlPlaneStore, user_id: &str, workspace_id: &str, root_path: &Path, ) -> Result { let user_id = user_id.trim(); let scope_id = local_index_scope_id(root_path); if user_id.is_empty() || user_id == "anonymous" { return Ok(read_local_index_settings_or_default(root_path) .unwrap_or_else(|_| default_local_index_settings())); } let root_preferences = store .list_user_ui_preferences_for_scope( Some(workspace_id), Some(LOCAL_INDEX_SOURCE_KIND), LOCAL_INDEX_SETTINGS_SCOPE_KIND, &scope_id, LOCAL_INDEX_SETTINGS_KEY, ) .map_err(|error| WebError::internal(format!("SQLite 本地索引设置读取失败: {error}")))?; let preferences = store .list_user_ui_preferences(user_id, Some(workspace_id), Some(LOCAL_INDEX_SOURCE_KIND)) .map_err(|error| WebError::internal(format!("SQLite 本地索引设置读取失败: {error}")))?; let Some(record) = preferences.iter().rev().find(|preference| { preference.scope_kind == LOCAL_INDEX_SETTINGS_SCOPE_KIND && preference.scope_id == scope_id && preference.key == LOCAL_INDEX_SETTINGS_KEY }) else { if root_preferences.is_empty() { return Ok(read_local_index_settings_or_default(root_path) .unwrap_or_else(|_| default_local_index_settings())); } return Ok(default_local_index_settings()); }; parse_local_index_settings_value(root_path, &record.value_json) } pub(crate) fn write_user_local_index_settings( store: &dyn ControlPlaneStore, user_id: &str, workspace_id: &str, root_path: &Path, include_paths: &[String], schedule_mode: Option<&str>, schedule_time: Option<&str>, schedule_date: Option<&str>, run_on_change: Option, ) -> Result { let user_id = user_id.trim(); let scope_id = local_index_scope_id(root_path); if user_id.is_empty() || user_id == "anonymous" { return Err(WebError::bad_request_code( "local_index_settings_auth_required", "本地索引设置需要登录用户", )); } store .upsert_user(UpsertUserInput { id: Some(user_id.to_string()), email: None, username: user_id.to_string(), display_name: user_id.to_string(), role: None, password_hash: None, }) .map_err(|error| WebError::internal(format!("SQLite 用户初始化失败: {error}")))?; let existing = read_user_local_index_settings(store, user_id, workspace_id, root_path)?; let include_paths = normalize_index_include_paths(root_path, include_paths)?; let schedule_mode = normalize_index_schedule_mode(schedule_mode.unwrap_or(&existing.schedule_mode))?; let schedule_time = normalize_index_schedule_time(schedule_time.unwrap_or(&existing.schedule_time))?; let schedule_date = normalize_index_schedule_date(schedule_date.or(existing.schedule_date.as_deref()))?; let settings = LocalIndexSettings { schema: LOCAL_INDEX_SETTINGS_SCHEMA.to_string(), include_paths, schedule_mode, schedule_time, schedule_date, run_on_change: run_on_change.unwrap_or(existing.run_on_change), updated_at: now_ms(), }; let value_json = serde_json::to_string(&settings) .map_err(|error| WebError::internal(format!("本地索引设置序列化失败: {error}")))?; store .upsert_user_ui_preference(UpsertUserUiPreferenceInput { id: None, user_id: user_id.to_string(), workspace_id: Some(workspace_id.to_string()), source_kind: Some(LOCAL_INDEX_SOURCE_KIND.to_string()), scope_kind: LOCAL_INDEX_SETTINGS_SCOPE_KIND.to_string(), scope_id, key: LOCAL_INDEX_SETTINGS_KEY.to_string(), value_json, }) .map_err(|error| WebError::internal(format!("SQLite 本地索引设置写入失败: {error}")))?; let effective = effective_local_index_settings_for_root(store, workspace_id, root_path)?; write_local_index_settings_cache(root_path, &effective)?; Ok(settings) } pub(crate) fn preview_user_local_index_settings( store: &dyn ControlPlaneStore, user_id: &str, workspace_id: &str, root_path: &Path, include_paths: &[String], schedule_mode: Option<&str>, schedule_time: Option<&str>, schedule_date: Option<&str>, run_on_change: Option, ) -> Result { let user_id = user_id.trim(); if user_id.is_empty() || user_id == "anonymous" { return Err(WebError::bad_request_code( "local_index_settings_auth_required", "本地索引设置需要登录用户", )); } let existing = read_user_local_index_settings(store, user_id, workspace_id, root_path)?; let include_paths = normalize_index_include_paths(root_path, include_paths)?; let schedule_mode = normalize_index_schedule_mode(schedule_mode.unwrap_or(&existing.schedule_mode))?; let schedule_time = normalize_index_schedule_time(schedule_time.unwrap_or(&existing.schedule_time))?; let schedule_date = normalize_index_schedule_date(schedule_date.or(existing.schedule_date.as_deref()))?; Ok(LocalIndexSettings { schema: LOCAL_INDEX_SETTINGS_SCHEMA.to_string(), include_paths, schedule_mode, schedule_time, schedule_date, run_on_change: run_on_change.unwrap_or(existing.run_on_change), updated_at: now_ms(), }) } pub(crate) fn effective_local_index_settings_for_root( store: &dyn ControlPlaneStore, workspace_id: &str, root_path: &Path, ) -> Result { let scope_id = local_index_scope_id(root_path); let records = store .list_user_ui_preferences_for_scope( Some(workspace_id), Some(LOCAL_INDEX_SOURCE_KIND), LOCAL_INDEX_SETTINGS_SCOPE_KIND, &scope_id, LOCAL_INDEX_SETTINGS_KEY, ) .map_err(|error| WebError::internal(format!("SQLite 本地索引设置读取失败: {error}")))?; let mut effective = read_local_index_settings_or_default(root_path) .unwrap_or_else(|_| default_local_index_settings()); if records.is_empty() { return Ok(effective); } let mut include_paths = Vec::new(); let mut run_on_change = false; for record in records { let settings = parse_local_index_settings_value(root_path, &record.value_json)?; include_paths.extend(settings.include_paths); run_on_change |= settings.run_on_change; } effective.include_paths = normalize_index_include_paths(root_path, &include_paths)?; effective.run_on_change = run_on_change; effective.updated_at = now_ms(); Ok(effective) } pub(crate) fn local_index_any_schedule_due_for_root( store: &dyn ControlPlaneStore, workspace_id: &str, root_path: &Path, built_at_ms: u128, ) -> Result { let scope_id = local_index_scope_id(root_path); let records = store .list_user_ui_preferences_for_scope( Some(workspace_id), Some(LOCAL_INDEX_SOURCE_KIND), LOCAL_INDEX_SETTINGS_SCOPE_KIND, &scope_id, LOCAL_INDEX_SETTINGS_KEY, ) .map_err(|error| WebError::internal(format!("SQLite 本地索引设置读取失败: {error}")))?; if records.is_empty() { let settings = read_local_index_settings_or_default(root_path) .unwrap_or_else(|_| default_local_index_settings()); return Ok(local_index_schedule_is_due(&settings, built_at_ms)); } for record in records { let settings = parse_local_index_settings_value(root_path, &record.value_json)?; if local_index_schedule_is_due(&settings, built_at_ms) { return Ok(true); } } Ok(false) } #[cfg(test)] pub(crate) fn local_index_status( root_path: &Path, root_uri: &str, workspace_id: &str, ) -> Result { let settings = read_local_index_settings_or_default(root_path)?; local_index_status_for_settings(root_path, root_uri, workspace_id, &settings) } fn local_index_status_for_settings( root_path: &Path, root_uri: &str, workspace_id: &str, settings: &LocalIndexSettings, ) -> Result { let index = read_local_search_index(root_path)?; let index_path = root_path .join(".mnote") .join("index") .join("search-index.json"); let evidence_path = evidence_sqlite_path(root_path); let mut document_count = 0usize; let mut resource_count = 0usize; let mut built_at = Value::Null; let cache_matches_settings; let scheduled_due = if let Some(index) = index.as_ref() { document_count = index.documents.len(); resource_count = index.resources.len(); built_at = json!(index.built_at); cache_matches_settings = index.version == LOCAL_SEARCH_INDEX_VERSION && index.root_uri == root_uri && index.workspace_id == workspace_id && normalized_indexed_paths(&index.indexed_paths) == settings.include_paths; local_index_schedule_is_due(&settings, index.built_at) } else { cache_matches_settings = settings.include_paths.is_empty(); local_index_schedule_is_due(&settings, 0) }; let evidence_block_count = if evidence_path.exists() { count_evidence_blocks(&evidence_path).unwrap_or(0) } else { 0 }; Ok(json!({ "schema": "mnote.local_index.status.v1", "rootUri": root_uri, "workspaceId": workspace_id, "settings": settings, "indexPath": ".mnote/index/search-index.json", "evidenceIndexPath": ".mnote/index/evidence.sqlite", "settingsPath": ".mnote/index/local-index-settings.json", "indexExists": index_path.exists(), "evidenceIndexExists": evidence_path.exists(), "cacheMatchesSettings": cache_matches_settings, "scheduledDue": scheduled_due, "builtAt": built_at, "indexedPaths": index .as_ref() .map(|index| index.indexed_paths.clone()) .unwrap_or_default(), "documentCount": document_count, "resourceCount": resource_count, "evidenceBlockCount": evidence_block_count, })) } pub(crate) fn local_index_status_with_settings( root_path: &Path, root_uri: &str, workspace_id: &str, user_settings: &LocalIndexSettings, effective_settings: &LocalIndexSettings, ) -> Result { let mut status = local_index_status_for_settings(root_path, root_uri, workspace_id, effective_settings)?; if let Some(object) = status.as_object_mut() { object.insert("settings".to_string(), json!(user_settings)); object.insert("effectiveSettings".to_string(), json!(effective_settings)); } Ok(status) } #[cfg(test)] #[allow(dead_code)] pub(crate) fn local_index_should_refresh_on_change(root_path: &Path) -> bool { read_local_index_settings_or_default(root_path) .map(|settings| settings.run_on_change) .unwrap_or(false) } #[cfg(test)] #[allow(dead_code)] pub(crate) fn refresh_local_search_index_for_change( root_path: &Path, root_uri: &str, workspace_id: &str, ) -> Result, WebError> { if !local_index_should_refresh_on_change(root_path) { return Ok(None); } refresh_local_search_index(root_path, root_uri, workspace_id).map(Some) } pub(crate) fn refresh_local_search_index_for_change_with_store( store: &dyn ControlPlaneStore, root_path: &Path, root_uri: &str, workspace_id: &str, ) -> Result, WebError> { let effective_settings = effective_local_index_settings_for_root(store, workspace_id, root_path)?; if !effective_settings.run_on_change { return Ok(None); } refresh_local_search_index_with_settings(root_path, root_uri, workspace_id, &effective_settings) .map(Some) } #[cfg(test)] #[allow(dead_code)] pub(crate) fn refresh_local_search_index_for_change_path( root_path: &Path, root_uri: &str, workspace_id: &str, relative_path: &str, ) -> Result, WebError> { if !local_index_should_refresh_on_change(root_path) { return Ok(None); } refresh_local_search_index_for_path(root_path, root_uri, workspace_id, relative_path).map(Some) } #[cfg(test)] #[allow(dead_code)] pub(crate) fn refresh_local_search_index_if_scheduled_due( root_path: &Path, root_uri: &str, workspace_id: &str, ) -> Result, WebError> { let settings = read_local_index_settings_or_default(root_path)?; let Some(index) = read_local_search_index(root_path)? else { if local_index_schedule_is_due(&settings, 0) { return refresh_local_search_index(root_path, root_uri, workspace_id).map(Some); } return Ok(None); }; if !local_index_schedule_is_due(&settings, index.built_at) { return Ok(None); } refresh_local_search_index(root_path, root_uri, workspace_id).map(Some) } pub(crate) fn refresh_local_search_index_for_change_path_with_store( store: &dyn ControlPlaneStore, root_path: &Path, root_uri: &str, workspace_id: &str, relative_path: &str, ) -> Result, WebError> { let effective_settings = effective_local_index_settings_for_root(store, workspace_id, root_path)?; if !effective_settings.run_on_change { return Ok(None); } refresh_local_search_index_for_path_with_settings( root_path, root_uri, workspace_id, &effective_settings, relative_path, ) .map(Some) } pub(crate) fn refresh_local_search_index_if_scheduled_due_with_store( store: &dyn ControlPlaneStore, root_path: &Path, root_uri: &str, workspace_id: &str, ) -> Result, WebError> { let effective_settings = effective_local_index_settings_for_root(store, workspace_id, root_path)?; let built_at_ms = current_local_search_index_built_at(root_path)?.unwrap_or(0); if !local_index_any_schedule_due_for_root(store, workspace_id, root_path, built_at_ms)? { return Ok(None); } refresh_local_search_index_with_settings(root_path, root_uri, workspace_id, &effective_settings) .map(Some) } #[cfg(test)] pub(crate) fn query_evidence_sqlite_results( root_path: &Path, query: &str, owner_document_id: Option<&str>, limit: u32, ) -> Result>, WebError> { query_evidence_sqlite_results_with_mode(root_path, query, owner_document_id, limit, true) } pub(crate) fn query_evidence_sqlite_results_with_mode( root_path: &Path, query: &str, owner_document_id: Option<&str>, limit: u32, exact: bool, ) -> Result>, WebError> { let path = evidence_sqlite_path(root_path); if !path.exists() { return Ok(None); } let normalized_query = query.trim(); if normalized_query.is_empty() { return Ok(Some(Vec::new())); } let owner_document_id = owner_document_id .map(str::trim) .filter(|value| !value.is_empty()); let resource_path = owner_document_id.and_then(local_resource_path_from_document_id); let owner_document_id = if resource_path.is_some() { None } else { owner_document_id }; let connection = Connection::open(&path).map_err(|error| { WebError::bad_request_code( "evidence_index_open_failed", format!("无法打开 evidence 索引 {}: {error}", path.display()), ) })?; if !exact { return Ok(Some(query_evidence_sqlite_fuzzy( &connection, normalized_query, owner_document_id, resource_path.as_deref(), limit, )?)); } let fts_query = evidence_fts_phrase(normalized_query); let results = match query_evidence_sqlite_fts( &connection, &fts_query, normalized_query, owner_document_id, resource_path.as_deref(), limit, ) { Ok(results) if results.is_empty() => query_evidence_sqlite_like( &connection, normalized_query, owner_document_id, resource_path.as_deref(), limit, )?, Ok(results) => results, Err(_) => query_evidence_sqlite_like( &connection, normalized_query, owner_document_id, resource_path.as_deref(), limit, )?, }; Ok(Some(results)) } pub(crate) fn read_evidence_sqlite_context( root_path: &Path, locator: &EvidenceLocator, before_blocks: u32, after_blocks: u32, ) -> Result>, WebError> { let path = evidence_sqlite_path(root_path); if !path.exists() { return Ok(None); } let connection = Connection::open(&path).map_err(|error| { WebError::bad_request_code( "evidence_index_open_failed", format!("无法打开 evidence 索引 {}: {error}", path.display()), ) })?; let mut statement = connection .prepare( "SELECT b.block_id, b.text, b.locator_json \ FROM evidence_block b \ JOIN evidence_resource r ON r.resource_id = b.resource_id \ WHERE r.owner_document_id = ?1 \ ORDER BY b.rowid", ) .map_err(sqlite_error)?; let rows = statement .query_map(params![locator.owner_document_id], |row| { let block_id: String = row.get(0)?; let text: String = row.get(1)?; let locator_json: String = row.get(2)?; let source = serde_json::from_str::(&locator_json).map_err(|error| { rusqlite::Error::FromSqlConversionFailure( 2, rusqlite::types::Type::Text, Box::new(error), ) })?; Ok((block_id, text, source)) }) .map_err(sqlite_error)? .collect::, _>>() .map_err(sqlite_error)?; if rows.is_empty() { return Ok(Some(Vec::new())); } let Some(anchor_index) = rows .iter() .position(|(block_id, _, source)| evidence_locator_matches(block_id, source, locator)) else { return Ok(Some(Vec::new())); }; let start = anchor_index.saturating_sub(before_blocks as usize); let end = (anchor_index + after_blocks as usize + 1).min(rows.len()); let results = rows[start..end] .iter() .map(|(block_id, text, source)| EvidenceSearchResult { evidence_id: block_id.clone(), quote: text.clone(), score: if evidence_locator_matches(block_id, source, locator) { 1.0 } else { 0.8 }, source: source.clone(), match_info: None, citation_url: None, citation_label: None, citation_markdown: None, }) .collect::>(); Ok(Some(results)) } pub(crate) fn query_evidence_graph_results( root_path: &Path, query: &str, owner_document_id: Option<&str>, limit: u32, ) -> Result>, WebError> { let path = evidence_sqlite_path(root_path); if !path.exists() { return Ok(None); } let normalized_query = query.trim().to_ascii_lowercase(); if normalized_query.is_empty() { return Ok(Some(Vec::new())); } let connection = Connection::open(&path).map_err(|error| { WebError::bad_request_code( "evidence_index_open_failed", format!("无法打开 evidence 索引 {}: {error}", path.display()), ) })?; let like_query = format!("%{}%", normalized_query.replace('%', "\\%")); let mut statement = if owner_document_id.is_some() { connection.prepare( "SELECT e.edge_id, e.edge_type, e.from_id, e.to_id, e.source_block_id, b.text, b.locator_json \ FROM evidence_edge e \ JOIN evidence_block b ON b.block_id = e.source_block_id \ JOIN evidence_resource r ON r.resource_id = b.resource_id \ WHERE r.owner_document_id = ?2 \ AND (lower(e.edge_type) LIKE ?1 OR lower(e.from_id) LIKE ?1 OR lower(e.to_id) LIKE ?1) \ ORDER BY e.edge_type, e.from_id, e.to_id \ LIMIT ?3", ) } else { connection.prepare( "SELECT e.edge_id, e.edge_type, e.from_id, e.to_id, e.source_block_id, b.text, b.locator_json \ FROM evidence_edge e \ JOIN evidence_block b ON b.block_id = e.source_block_id \ WHERE lower(e.edge_type) LIKE ?1 OR lower(e.from_id) LIKE ?1 OR lower(e.to_id) LIKE ?1 \ ORDER BY e.edge_type, e.from_id, e.to_id \ LIMIT ?2", ) } .map_err(sqlite_error)?; let results = if let Some(owner_document_id) = owner_document_id { statement .query_map( params![like_query, owner_document_id, limit], graph_edge_row, ) .map_err(sqlite_error)? .collect::, _>>() .map_err(sqlite_error)? } else { statement .query_map(params![like_query, limit], graph_edge_row) .map_err(sqlite_error)? .collect::, _>>() .map_err(sqlite_error)? }; Ok(Some(results)) } fn graph_edge_row(row: &rusqlite::Row<'_>) -> rusqlite::Result { let edge_id: String = row.get(0)?; let edge_type: String = row.get(1)?; let from_id: String = row.get(2)?; let to_id: String = row.get(3)?; let source_block_id: String = row.get(4)?; let text: String = row.get(5)?; let locator_json: String = row.get(6)?; let source = serde_json::from_str::(&locator_json).map_err(|error| { rusqlite::Error::FromSqlConversionFailure(6, rusqlite::types::Type::Text, Box::new(error)) })?; Ok(EvidenceSearchResult { evidence_id: edge_id, quote: format!("{edge_type}: {from_id} -> {to_id}\n{text}"), score: 1.0, source: EvidenceLocator { block_id: source.block_id.or(Some(source_block_id)), ..source }, match_info: None, citation_url: None, citation_label: None, citation_markdown: None, }) } fn evidence_locator_matches( block_id: &str, source: &EvidenceLocator, locator: &EvidenceLocator, ) -> bool { locator.block_id.as_deref().is_some_and(|locator_block_id| { locator_block_id == block_id || source.block_id.as_deref() == Some(locator_block_id) }) || source.owner_document_id == locator.owner_document_id && source.resource_path == locator.resource_path && source.source_map_path == locator.source_map_path } fn query_evidence_sqlite_fts( connection: &Connection, fts_query: &str, display_query: &str, owner_document_id: Option<&str>, resource_path: Option<&str>, limit: u32, ) -> Result, WebError> { let mut sql = String::from( "SELECT b.block_id, b.text, b.locator_json, bm25(evidence_fts) AS rank \ FROM evidence_fts \ JOIN evidence_block b ON b.block_id = evidence_fts.block_id \ JOIN evidence_resource r ON r.resource_id = b.resource_id \ WHERE evidence_fts MATCH ?1", ); if owner_document_id.is_some() { sql.push_str(" AND r.owner_document_id = ?2"); } else if resource_path.is_some() { sql.push_str(" AND r.source_root_relative_path = ?2"); } sql.push_str(if owner_document_id.is_some() || resource_path.is_some() { " ORDER BY rank LIMIT ?3" } else { " ORDER BY rank LIMIT ?2" }); let limit = i64::from(limit.max(1)); let mut statement = connection.prepare(&sql).map_err(sqlite_error)?; let rows = if let Some(owner_document_id) = owner_document_id { statement .query_map(params![fts_query, owner_document_id, limit], |row| { evidence_result_from_sqlite_row(row, display_query) }) .map_err(sqlite_error)? .collect::, _>>() } else if let Some(resource_path) = resource_path { statement .query_map(params![fts_query, resource_path, limit], |row| { evidence_result_from_sqlite_row(row, display_query) }) .map_err(sqlite_error)? .collect::, _>>() } else { statement .query_map(params![fts_query, limit], |row| { evidence_result_from_sqlite_row(row, display_query) }) .map_err(sqlite_error)? .collect::, _>>() }; rows.map_err(sqlite_error) } fn query_evidence_sqlite_like( connection: &Connection, query: &str, owner_document_id: Option<&str>, resource_path: Option<&str>, limit: u32, ) -> Result, WebError> { let mut sql = String::from( "SELECT b.block_id, b.text, b.locator_json, 0.0 AS rank \ FROM evidence_block b \ JOIN evidence_resource r ON r.resource_id = b.resource_id \ WHERE b.text LIKE ?1 ESCAPE '\\'", ); if owner_document_id.is_some() { sql.push_str(" AND r.owner_document_id = ?2"); } else if resource_path.is_some() { sql.push_str(" AND r.source_root_relative_path = ?2"); } sql.push_str(if owner_document_id.is_some() || resource_path.is_some() { " LIMIT ?3" } else { " LIMIT ?2" }); let like_query = format!("%{}%", query.replace('%', "\\%").replace('_', "\\_")); let limit = i64::from(limit.max(1)); let mut statement = connection.prepare(&sql).map_err(sqlite_error)?; let rows = if let Some(owner_document_id) = owner_document_id { statement .query_map(params![like_query, owner_document_id, limit], |row| { evidence_result_from_sqlite_row(row, query) }) .map_err(sqlite_error)? .collect::, _>>() } else if let Some(resource_path) = resource_path { statement .query_map(params![like_query, resource_path, limit], |row| { evidence_result_from_sqlite_row(row, query) }) .map_err(sqlite_error)? .collect::, _>>() } else { statement .query_map(params![like_query, limit], |row| { evidence_result_from_sqlite_row(row, query) }) .map_err(sqlite_error)? .collect::, _>>() }; rows.map_err(sqlite_error) } fn query_evidence_sqlite_fuzzy( connection: &Connection, query: &str, owner_document_id: Option<&str>, resource_path: Option<&str>, limit: u32, ) -> Result, WebError> { let mut sql = String::from( "SELECT b.block_id, b.text, b.locator_json, 0.0 AS rank \ FROM evidence_block b \ JOIN evidence_resource r ON r.resource_id = b.resource_id", ); let first_char_like = query .chars() .find(|ch| !ch.is_whitespace()) .map(|ch| format!("%{}%", ch)); match ( owner_document_id.is_some() || resource_path.is_some(), first_char_like.is_some(), resource_path.is_some(), ) { (true, true, true) => { sql.push_str(" WHERE r.source_root_relative_path = ?1 AND b.text LIKE ?2 LIMIT ?3") } (true, true, false) => { sql.push_str(" WHERE r.owner_document_id = ?1 AND b.text LIKE ?2 LIMIT ?3") } (true, false, true) => sql.push_str(" WHERE r.source_root_relative_path = ?1 LIMIT ?2"), (true, false, false) => sql.push_str(" WHERE r.owner_document_id = ?1 LIMIT ?2"), (false, true, _) => sql.push_str(" WHERE b.text LIKE ?1 LIMIT ?2"), (false, false, _) => sql.push_str(" LIMIT ?1"), } let mut statement = connection.prepare(&sql).map_err(sqlite_error)?; let scan_limit = i64::from(limit.max(1)) * 200; let scope_value = owner_document_id.or(resource_path); let rows = match (scope_value, first_char_like.as_deref()) { (Some(scope_value), Some(first_char_like)) => statement .query_map( params![scope_value, first_char_like, scan_limit], evidence_sqlite_row_parts, ) .map_err(sqlite_error)? .collect::, _>>(), (Some(scope_value), None) => statement .query_map(params![scope_value, scan_limit], evidence_sqlite_row_parts) .map_err(sqlite_error)? .collect::, _>>(), (None, Some(first_char_like)) => statement .query_map( params![first_char_like, scan_limit], evidence_sqlite_row_parts, ) .map_err(sqlite_error)? .collect::, _>>(), (None, None) => statement .query_map(params![scan_limit], evidence_sqlite_row_parts) .map_err(sqlite_error)? .collect::, _>>(), } .map_err(sqlite_error)?; let mut scored = rows .into_iter() .filter_map(|(block_id, text, source)| { let text_match = score_evidence_text_match(&text, query)?; Some(EvidenceSearchResult { evidence_id: block_id, quote: ocr_search_snippet_for_terms(&text, query, &text_match.matched_terms), score: text_match.score, source, match_info: Some(text_match.into_match_info(None)), citation_url: None, citation_label: None, citation_markdown: None, }) }) .collect::>(); scored.sort_by(|left, right| { right .score .partial_cmp(&left.score) .unwrap_or(std::cmp::Ordering::Equal) }); Ok(scored .into_iter() .enumerate() .map(|(index, mut result)| { if let Some(match_info) = result.match_info.as_mut() { match_info.rank = Some((index + 1) as u32); } result }) .take(limit.max(1) as usize) .collect()) } fn evidence_sqlite_row_parts( row: &rusqlite::Row<'_>, ) -> rusqlite::Result<(String, String, EvidenceLocator)> { let block_id: String = row.get(0)?; let text: String = row.get(1)?; let locator_json: String = row.get(2)?; let source = serde_json::from_str::(&locator_json).map_err(|error| { rusqlite::Error::FromSqlConversionFailure(2, rusqlite::types::Type::Text, Box::new(error)) })?; Ok((block_id, text, source)) } fn evidence_result_from_sqlite_row( row: &rusqlite::Row<'_>, query: &str, ) -> rusqlite::Result { let block_id: String = row.get(0)?; let text: String = row.get(1)?; let locator_json: String = row.get(2)?; let rank: f64 = row.get(3)?; let source = serde_json::from_str::(&locator_json).map_err(|error| { rusqlite::Error::FromSqlConversionFailure(2, rusqlite::types::Type::Text, Box::new(error)) })?; let text_match = score_evidence_text_match(&text, query); Ok(EvidenceSearchResult { evidence_id: block_id, quote: text_match .as_ref() .map(|text_match| ocr_search_snippet_for_terms(&text, query, &text_match.matched_terms)) .unwrap_or_else(|| ocr_search_snippet(&text, query)), score: if rank == 0.0 { 1.0 } else { 1.0 / (1.0 + rank.abs()) }, source, match_info: text_match.map(|text_match| text_match.into_match_info(None)), citation_url: None, citation_label: None, citation_markdown: None, }) } fn evidence_fts_phrase(query: &str) -> String { format!("\"{}\"", query.replace('"', "\"\"")) } #[cfg(test)] pub(crate) fn refresh_local_search_index( root_path: &Path, root_uri: &str, workspace_id: &str, ) -> Result { let settings = read_local_index_settings_or_default(root_path)?; refresh_local_search_index_with_settings(root_path, root_uri, workspace_id, &settings) } pub(crate) fn refresh_local_search_index_with_settings( root_path: &Path, root_uri: &str, workspace_id: &str, settings: &LocalIndexSettings, ) -> Result { if settings.include_paths.is_empty() { clear_local_search_index(root_path)?; return Ok(json!({ "version": LOCAL_SEARCH_INDEX_VERSION, "rootUri": root_uri, "workspaceId": workspace_id, "indexedPaths": [], "builtAt": now_ms(), "documentCount": 0, "resourceCount": 0 })); } let index = rebuild_local_search_index_with_settings(root_path, root_uri, workspace_id, settings)?; Ok(json!({ "version": index.version, "rootUri": index.root_uri, "workspaceId": index.workspace_id, "indexedPaths": index.indexed_paths, "builtAt": index.built_at, "documentCount": index.documents.len(), "resourceCount": index.resources.len() })) } #[cfg(test)] pub(crate) fn refresh_local_search_index_for_path( root_path: &Path, root_uri: &str, workspace_id: &str, relative_path: &str, ) -> Result { let settings = read_local_index_settings_or_default(root_path)?; refresh_local_search_index_for_path_with_settings( root_path, root_uri, workspace_id, &settings, relative_path, ) } pub(crate) fn refresh_local_search_index_for_path_with_settings( root_path: &Path, root_uri: &str, workspace_id: &str, settings: &LocalIndexSettings, relative_path: &str, ) -> Result { let relative_path = normalize_index_relative_path(relative_path)?; let mut index = match read_local_search_index(root_path) { Ok(Some(index)) if index.version == LOCAL_SEARCH_INDEX_VERSION && index.root_uri == root_uri && index.workspace_id == workspace_id && normalized_indexed_paths(&index.indexed_paths) == settings.include_paths => { index } Ok(_) | Err(_) => { return refresh_local_search_index_with_settings( root_path, root_uri, workspace_id, settings, ); } }; index .documents .retain(|document| document.path != relative_path); index .resources .retain(|resource| resource.path != relative_path); let included = index_relative_path_is_included(&relative_path, &settings.include_paths); if local_ocr::is_local_ocr_sidecar_relative_path(&relative_path) { index.built_at = now_ms(); write_local_search_index_json(root_path, &index)?; let _ = included; remove_evidence_sqlite_index_for_path(root_path, &index, &relative_path)?; return Ok(json!({ "version": index.version, "rootUri": index.root_uri, "workspaceId": index.workspace_id, "indexedPaths": index.indexed_paths, "builtAt": index.built_at, "documentCount": index.documents.len(), "resourceCount": index.resources.len() })); } if is_local_evidence_sidecar_relative_path(&relative_path) { index.built_at = now_ms(); write_local_search_index_json(root_path, &index)?; return Ok(json!({ "version": index.version, "rootUri": index.root_uri, "workspaceId": index.workspace_id, "indexedPaths": index.indexed_paths, "builtAt": index.built_at, "documentCount": index.documents.len(), "resourceCount": index.resources.len() })); } let absolute_path = root_path.join(&relative_path); if included && absolute_path.exists() && absolute_path.is_file() && is_markdown_path(&absolute_path) { index .documents .push(index_markdown_file(root_path, &absolute_path)?); } else if included && absolute_path.exists() && absolute_path.is_file() && resource_type_from_path(&absolute_path).is_some() { index .resources .push(index_resource_file(root_path, &absolute_path)?); } index .documents .sort_by(|left, right| left.path.cmp(&right.path)); index .resources .sort_by(|left, right| left.path.cmp(&right.path)); index.built_at = now_ms(); write_local_search_index_json(root_path, &index)?; if included { refresh_evidence_sqlite_index_for_path(root_path, &index, &relative_path)?; } else { remove_evidence_sqlite_index_for_path(root_path, &index, &relative_path)?; } Ok(json!({ "version": index.version, "rootUri": index.root_uri, "workspaceId": index.workspace_id, "indexedPaths": index.indexed_paths, "builtAt": index.built_at, "documentCount": index.documents.len(), "resourceCount": index.resources.len() })) } #[cfg(test)] #[allow(dead_code)] pub(crate) fn query_local_backlinks( root_path: &Path, root_uri: &str, workspace_id: &str, document_id: &str, ) -> Result { let settings = read_local_index_settings_or_default(root_path)?; query_local_backlinks_with_settings( root_path, root_uri, workspace_id, &settings, &settings, document_id, ) } pub(crate) fn query_local_backlinks_with_settings( root_path: &Path, root_uri: &str, workspace_id: &str, cache_settings: &LocalIndexSettings, result_settings: &LocalIndexSettings, document_id: &str, ) -> Result { let index = load_or_rebuild_local_search_index_with_settings( root_path, root_uri, workspace_id, cache_settings, )?; let Some(target) = index.documents.iter().find(|document| { document.document_id == document_id && index_relative_path_is_included(&document.path, &result_settings.include_paths) }) else { return Ok(json!({ "documentId": document_id, "rootUri": root_uri, "workspaceId": workspace_id, "backlinks": [] })); }; let target_keys = normalized_document_targets(target); let backlinks = index .documents .iter() .filter(|document| { index_relative_path_is_included(&document.path, &result_settings.include_paths) }) .filter(|document| document.document_id != target.document_id) .filter_map(|document| { let matched_targets = document .backlinks .iter() .filter(|link| target_keys.contains(&normalize_search_text(link))) .cloned() .collect::>(); if matched_targets.is_empty() { return None; } Some(json!({ "documentId": document.document_id, "title": document.title, "path": document.path, "resourceType": "markdown", "matchedTargets": matched_targets, "snippet": search_snippet(document, &normalize_search_text(&target.title)) })) }) .collect::>(); Ok(json!({ "documentId": target.document_id, "title": target.title, "path": target.path, "rootUri": root_uri, "workspaceId": workspace_id, "backlinks": backlinks })) } #[cfg(test)] #[allow(dead_code)] pub(crate) fn query_local_tags( root_path: &Path, root_uri: &str, workspace_id: &str, ) -> Result { let settings = read_local_index_settings_or_default(root_path)?; query_local_tags_with_settings(root_path, root_uri, workspace_id, &settings, &settings) } pub(crate) fn query_local_tags_with_settings( root_path: &Path, root_uri: &str, workspace_id: &str, cache_settings: &LocalIndexSettings, result_settings: &LocalIndexSettings, ) -> Result { let index = load_or_rebuild_local_search_index_with_settings( root_path, root_uri, workspace_id, cache_settings, )?; let mut tags = std::collections::BTreeMap::>::new(); for document in &index.documents { if !index_relative_path_is_included(&document.path, &result_settings.include_paths) { continue; } for tag in &document.tags { tags.entry(tag.clone()).or_default().push(document); } } Ok(json!({ "rootUri": root_uri, "workspaceId": workspace_id, "tags": tags .into_iter() .map(|(tag, documents)| { json!({ "tag": tag, "count": documents.len(), "documents": documents .into_iter() .map(|document| { json!({ "documentId": document.document_id, "title": document.title, "path": document.path, "resourceType": "markdown" }) }) .collect::>() }) }) .collect::>() })) } #[cfg(test)] #[allow(dead_code)] fn rebuild_local_search_index( root_path: &Path, root_uri: &str, workspace_id: &str, ) -> Result { let settings = read_local_index_settings_or_default(root_path)?; rebuild_local_search_index_with_settings(root_path, root_uri, workspace_id, &settings) } fn rebuild_local_search_index_with_settings( root_path: &Path, root_uri: &str, workspace_id: &str, settings: &LocalIndexSettings, ) -> Result { let mut documents = Vec::new(); let mut resources = Vec::new(); for include_path in &settings.include_paths { let base_path = if include_path == "." { root_path.to_path_buf() } else { root_path.join(include_path) }; if base_path.exists() { collect_markdown_documents(root_path, &base_path, &mut documents, &mut resources)?; } } documents.sort_by(|left, right| left.path.cmp(&right.path)); documents.dedup_by(|left, right| left.path == right.path); resources.sort_by(|left, right| left.path.cmp(&right.path)); resources.dedup_by(|left, right| left.path == right.path); let index = LocalSearchIndex { version: LOCAL_SEARCH_INDEX_VERSION, built_at: now_ms(), root_uri: root_uri.to_string(), workspace_id: workspace_id.to_string(), indexed_paths: settings.include_paths.clone(), documents, resources, }; write_local_search_index(root_path, &index)?; Ok(index) } #[cfg(test)] #[allow(dead_code)] fn load_or_rebuild_local_search_index( root_path: &Path, root_uri: &str, workspace_id: &str, ) -> Result { let settings = read_local_index_settings_or_default(root_path)?; load_or_rebuild_local_search_index_with_settings(root_path, root_uri, workspace_id, &settings) } fn load_or_rebuild_local_search_index_with_settings( root_path: &Path, root_uri: &str, workspace_id: &str, settings: &LocalIndexSettings, ) -> Result { match read_local_search_index(root_path) { Ok(Some(index)) if index.version == LOCAL_SEARCH_INDEX_VERSION && index.root_uri == root_uri && index.workspace_id == workspace_id && normalized_indexed_paths(&index.indexed_paths) == settings.include_paths => { if local_index_schedule_is_due(&settings, index.built_at) { return rebuild_local_search_index_with_settings( root_path, root_uri, workspace_id, settings, ); } ensure_evidence_sqlite_index(root_path, &index)?; Ok(index) } Ok(_) | Err(_) => { rebuild_local_search_index_with_settings(root_path, root_uri, workspace_id, settings) } } } fn read_local_search_index(root_path: &Path) -> Result, WebError> { let index_path = root_path .join(".mnote") .join("index") .join("search-index.json"); if !index_path.exists() { return Ok(None); } let content = fs::read_to_string(&index_path).map_err(|error| { WebError::bad_request_code( "local_search_index_read_failed", format!("无法读取本地搜索索引 {}: {error}", index_path.display()), ) })?; serde_json::from_str::(&content) .map(Some) .map_err(|error| { WebError::bad_request_code( "local_search_index_invalid", format!("本地搜索索引格式非法 {}: {error}", index_path.display()), ) }) } pub(crate) fn current_local_search_index_built_at( root_path: &Path, ) -> Result, WebError> { Ok(read_local_search_index(root_path)?.map(|index| index.built_at)) } fn default_indexed_paths() -> Vec { vec![".".to_string()] } fn local_index_settings_path(root_path: &Path) -> PathBuf { root_path .join(".mnote") .join("index") .join("local-index-settings.json") } pub(crate) fn read_local_index_settings_or_default( root_path: &Path, ) -> Result { let path = local_index_settings_path(root_path); if !path.exists() { return Ok(default_local_index_settings()); } let content = fs::read_to_string(&path).map_err(|error| { WebError::bad_request_code( "local_index_settings_read_failed", format!("无法读取本地索引设置 {}: {error}", path.display()), ) })?; parse_local_index_settings_value(root_path, &content).map_err(|error| { if error.code() == "local_index_settings_invalid" { WebError::bad_request_code( "local_index_settings_invalid", format!( "本地索引设置格式非法 {}: {}", path.display(), error.message() ), ) } else { error } }) } fn write_local_index_settings_cache( root_path: &Path, settings: &LocalIndexSettings, ) -> Result<(), WebError> { let path = local_index_settings_path(root_path); if let Some(parent) = path.parent() { fs::create_dir_all(parent).map_err(|error| { WebError::bad_request_code( "local_index_settings_create_failed", format!("无法创建本地索引设置目录 {}: {error}", parent.display()), ) })?; } let payload = serde_json::to_string_pretty(settings) .map_err(|error| WebError::internal(format!("本地索引设置序列化失败: {error}")))?; fs::write(&path, format!("{payload}\n")).map_err(|error| { WebError::bad_request_code( "local_index_settings_write_failed", format!("无法写入本地索引设置 {}: {error}", path.display()), ) }) } fn parse_local_index_settings_value( root_path: &Path, value_json: &str, ) -> Result { let raw = serde_json::from_str::(value_json).map_err(|error| { WebError::bad_request_code( "local_index_settings_invalid", format!("本地索引设置 JSON 非法: {error}"), ) })?; Ok(LocalIndexSettings { schema: LOCAL_INDEX_SETTINGS_SCHEMA.to_string(), include_paths: normalize_index_include_paths(root_path, &raw.include_paths)?, schedule_mode: normalize_index_schedule_mode(&raw.schedule_mode)?, schedule_time: normalize_index_schedule_time(&raw.schedule_time)?, schedule_date: normalize_index_schedule_date(raw.schedule_date.as_deref())?, run_on_change: raw.run_on_change, updated_at: raw.updated_at, }) } fn default_local_index_settings() -> LocalIndexSettings { LocalIndexSettings { schema: LOCAL_INDEX_SETTINGS_SCHEMA.to_string(), include_paths: Vec::new(), schedule_mode: default_index_schedule_mode(), schedule_time: default_index_schedule_time(), schedule_date: None, run_on_change: default_index_run_on_change(), updated_at: 0, } } fn default_index_schedule_mode() -> String { DEFAULT_INDEX_SCHEDULE_MODE.to_string() } fn default_index_schedule_time() -> String { DEFAULT_INDEX_SCHEDULE_TIME.to_string() } fn default_index_run_on_change() -> bool { false } fn normalize_index_schedule_mode(value: &str) -> Result { let mode = value.trim(); if mode.is_empty() { return Ok(default_index_schedule_mode()); } match mode { "daily" | "once" | "manual" => Ok(mode.to_string()), _ => Err(WebError::bad_request_code( "local_index_schedule_mode_invalid", "本地索引调度模式只能是 daily、once 或 manual", )), } } fn normalize_index_schedule_time(value: &str) -> Result { let value = value.trim(); let (hour, minute) = parse_hh_mm(value)?; Ok(format!("{hour:02}:{minute:02}")) } fn normalize_index_schedule_date(value: Option<&str>) -> Result, WebError> { let Some(value) = value.map(str::trim).filter(|value| !value.is_empty()) else { return Ok(None); }; let _ = parse_yyyy_mm_dd(value)?; Ok(Some(value.to_string())) } fn local_index_schedule_is_due(settings: &LocalIndexSettings, built_at_ms: u128) -> bool { let now = OffsetDateTime::now_local().unwrap_or_else(|_| OffsetDateTime::now_utc()); let Ok((hour, minute)) = parse_hh_mm(&settings.schedule_time) else { return false; }; let Ok(schedule_time) = Time::from_hms(hour, minute, 0) else { return false; }; let target = match settings.schedule_mode.as_str() { "manual" => return false, "once" => { let Some(date) = settings .schedule_date .as_deref() .and_then(|value| parse_yyyy_mm_dd(value).ok()) else { return false; }; date.with_time(schedule_time).assume_offset(now.offset()) } _ => now .date() .with_time(schedule_time) .assume_offset(now.offset()), }; if now < target { return false; } let built_at = unix_ms_to_local_datetime(built_at_ms); built_at.map(|value| value < target).unwrap_or(true) } fn unix_ms_to_local_datetime(value: u128) -> Option { let seconds = i64::try_from(value / 1000).ok()?; let nanos = u32::try_from((value % 1000) * 1_000_000).ok()?; OffsetDateTime::from_unix_timestamp(seconds) .ok() .map(|datetime| datetime + Duration::nanoseconds(nanos as i64)) .map(|datetime| { datetime.to_offset( OffsetDateTime::now_local() .unwrap_or_else(|_| OffsetDateTime::now_utc()) .offset(), ) }) } fn parse_hh_mm(value: &str) -> Result<(u8, u8), WebError> { let parts = value.trim().split(':').collect::>(); if parts.len() != 2 { return Err(WebError::bad_request_code( "local_index_schedule_time_invalid", "本地索引时间必须是 HH:MM", )); } let hour = parts[0].parse::().map_err(|_| { WebError::bad_request_code( "local_index_schedule_time_invalid", "本地索引时间必须是 HH:MM", ) })?; let minute = parts[1].parse::().map_err(|_| { WebError::bad_request_code( "local_index_schedule_time_invalid", "本地索引时间必须是 HH:MM", ) })?; if hour > 23 || minute > 59 { return Err(WebError::bad_request_code( "local_index_schedule_time_invalid", "本地索引时间必须是 00:00 到 23:59", )); } Ok((hour, minute)) } fn parse_yyyy_mm_dd(value: &str) -> Result { let parts = value.trim().split('-').collect::>(); if parts.len() != 3 { return Err(WebError::bad_request_code( "local_index_schedule_date_invalid", "本地索引日期必须是 YYYY-MM-DD", )); } let year = parts[0].parse::().map_err(|_| { WebError::bad_request_code( "local_index_schedule_date_invalid", "本地索引日期必须是 YYYY-MM-DD", ) })?; let month = parts[1].parse::().map_err(|_| { WebError::bad_request_code( "local_index_schedule_date_invalid", "本地索引日期必须是 YYYY-MM-DD", ) })?; let day = parts[2].parse::().map_err(|_| { WebError::bad_request_code( "local_index_schedule_date_invalid", "本地索引日期必须是 YYYY-MM-DD", ) })?; let month = Month::try_from(month).map_err(|_| { WebError::bad_request_code( "local_index_schedule_date_invalid", "本地索引日期必须是 YYYY-MM-DD", ) })?; Date::from_calendar_date(year, month, day).map_err(|_| { WebError::bad_request_code( "local_index_schedule_date_invalid", "本地索引日期必须是 YYYY-MM-DD", ) }) } fn normalize_index_include_paths( root_path: &Path, include_paths: &[String], ) -> Result, WebError> { let mut output = Vec::new(); let root_canonical = root_path .canonicalize() .unwrap_or_else(|_| root_path.to_path_buf()); for value in include_paths { let trimmed = value.trim(); if trimmed.is_empty() { continue; } let raw_path = PathBuf::from(trimmed); let normalized = if trimmed == "." || trimmed == "/" { ".".to_string() } else if raw_path.is_absolute() { let canonical = raw_path.canonicalize().map_err(|error| { WebError::bad_request_code( "local_index_scope_not_found", format!("本地索引范围不存在 {}: {error}", raw_path.display()), ) })?; if !canonical.starts_with(&root_canonical) { return Err(WebError::bad_request_code( "local_index_scope_escape", "本地索引目录必须位于当前授权 root 内", )); } if !canonical.is_dir() { return Err(WebError::bad_request_code( "local_index_scope_not_directory", "本地索引范围必须是目录", )); } canonical .strip_prefix(&root_canonical) .map_err(|_| { WebError::bad_request_code( "local_index_scope_escape", "本地索引目录必须位于当前授权 root 内", ) })? .to_string_lossy() .replace('\\', "/") } else { normalize_index_relative_path(trimmed.trim_start_matches("./"))? }; let normalized = if normalized.is_empty() { ".".to_string() } else { normalized }; if normalized.split('/').any(|part| part == ".mnote") { return Err(WebError::bad_request_code( "local_index_scope_reserved", "本地索引目录不能指向 .mnote 内部缓存", )); } let absolute = if normalized == "." { root_path.to_path_buf() } else { root_path.join(&normalized) }; if let Ok(canonical) = absolute.canonicalize() { if !canonical.starts_with(&root_canonical) { return Err(WebError::bad_request_code( "local_index_scope_escape", "本地索引目录必须位于当前授权 root 内", )); } if !canonical.is_dir() { return Err(WebError::bad_request_code( "local_index_scope_not_directory", "本地索引范围必须是目录", )); } } if output.iter().any(|existing| existing == &normalized) { continue; } output.push(normalized); } Ok(normalized_indexed_paths(&output)) } fn normalized_indexed_paths(paths: &[String]) -> Vec { let mut values = paths .iter() .map(|value| value.trim().replace('\\', "/")) .filter(|value| !value.is_empty()) .map(|value| { if value == "/" || value == "./" { ".".to_string() } else { value .trim_start_matches("./") .trim_end_matches('/') .to_string() } }) .map(|value| { if value.is_empty() { ".".to_string() } else { value } }) .collect::>(); values.sort(); values.dedup(); values } fn normalize_index_relative_path(relative_path: &str) -> Result { let normalized = relative_path.trim().replace('\\', "/"); if normalized.is_empty() { return Err(WebError::bad_request_code( "local_search_index_path_required", "本地搜索索引更新缺少相对路径", )); } let path = PathBuf::from(&normalized); if path.is_absolute() || path.components().any(|component| { matches!( component, Component::ParentDir | Component::RootDir | Component::Prefix(_) ) }) { return Err(WebError::bad_request_code( "local_search_index_path_escape", "本地搜索索引相对路径不能越过 root", )); } Ok(normalized) } fn index_relative_path_is_included(relative_path: &str, include_paths: &[String]) -> bool { let relative_path = relative_path.trim().replace('\\', "/"); include_paths.iter().any(|include| { include == "." || relative_path == *include || relative_path .strip_prefix(include) .map(|rest| rest.starts_with('/')) .unwrap_or(false) }) } fn collect_markdown_documents( root_path: &Path, current: &Path, documents: &mut Vec, resources: &mut Vec, ) -> Result<(), WebError> { let entries = match fs::read_dir(current) { Ok(entries) => entries, Err(error) => { return Err(WebError::bad_request_code( "local_search_index_read_dir_failed", format!("无法读取本地搜索索引目录 {}: {error}", current.display()), )); } }; for entry in entries { let entry = entry.map_err(|error| { WebError::bad_request_code( "local_search_index_read_entry_failed", format!("无法读取本地搜索索引条目: {error}"), ) })?; let path = entry.path(); let file_name = path .file_name() .and_then(|value| value.to_str()) .unwrap_or_default(); if file_name == ".mnote" { continue; } let file_type = entry.file_type().map_err(|error| { WebError::bad_request_code( "local_search_index_stat_failed", format!("无法读取本地搜索索引文件状态 {}: {error}", path.display()), ) })?; if file_type.is_dir() { collect_markdown_documents(root_path, &path, documents, resources)?; continue; } if !file_type.is_file() { continue; } let relative_path = path .strip_prefix(root_path) .unwrap_or(&path) .to_string_lossy() .replace('\\', "/"); if is_local_evidence_sidecar_relative_path(&relative_path) { continue; } if is_markdown_path(&path) { if local_ocr::is_local_ocr_sidecar_relative_path(&relative_path) { continue; } documents.push(index_markdown_file(root_path, &path)?); } else if resource_type_from_path(&path).is_some() { resources.push(index_resource_file(root_path, &path)?); } } Ok(()) } fn index_markdown_file(root_path: &Path, path: &Path) -> Result { let markdown = fs::read_to_string(path).map_err(|error| { WebError::bad_request_code( "local_search_index_read_markdown_failed", format!("无法读取本地 Markdown 索引文件 {}: {error}", path.display()), ) })?; let file_name = path .file_name() .and_then(|value| value.to_str()) .unwrap_or(""); let parsed = parse_markdown_page(&markdown, file_name); let relative_path = path .strip_prefix(root_path) .unwrap_or(path) .to_string_lossy() .replace('\\', "/"); let document_id = format!("local-md:{}", encode_local_id_segment(&relative_path)); let metadata = fs::metadata(path).map_err(|error| { WebError::bad_request_code( "local_search_index_stat_failed", format!( "无法读取本地 Markdown 索引文件状态 {}: {error}", path.display() ), ) })?; Ok(LocalSearchDocument { document_id, title: parsed.title, path: relative_path, raw_text: parsed.body.clone(), tags: extract_tags(&markdown), backlinks: extract_backlinks(&parsed.body), resource_refs: extract_resource_refs(&parsed.body), updated_at: metadata .modified() .ok() .and_then(|value| value.duration_since(UNIX_EPOCH).ok()) .map(|value| value.as_millis()) .unwrap_or_default(), }) } fn index_resource_file(root_path: &Path, path: &Path) -> Result { let resource_type = resource_type_from_path(path).ok_or_else(|| { WebError::bad_request_code( "local_search_index_resource_type_unsupported", format!("不支持索引该资源文件 {}", path.display()), ) })?; let relative_path = path .strip_prefix(root_path) .unwrap_or(path) .to_string_lossy() .replace('\\', "/"); let title = path .file_stem() .and_then(|value| value.to_str()) .unwrap_or("resource") .trim() .trim_end_matches(".mindmap") .to_string(); let metadata = fs::metadata(path).map_err(|error| { WebError::bad_request_code( "local_search_index_stat_failed", format!("无法读取本地资源索引文件状态 {}: {error}", path.display()), ) })?; Ok(LocalSearchResource { resource_id: format!("local-resource:{}", relative_path.replace('/', "~2F")), resource_type: resource_type.to_string(), title, path: relative_path, updated_at: metadata .modified() .ok() .and_then(|value| value.duration_since(UNIX_EPOCH).ok()) .map(|value| value.as_millis()) .unwrap_or_default(), }) } fn write_local_search_index(root_path: &Path, index: &LocalSearchIndex) -> Result<(), WebError> { write_local_search_index_json(root_path, index)?; write_evidence_sqlite_index(root_path, index) } fn write_local_search_index_json( root_path: &Path, index: &LocalSearchIndex, ) -> Result<(), WebError> { let index_dir = root_path.join(".mnote").join("index"); fs::create_dir_all(&index_dir).map_err(|error| { WebError::bad_request_code( "local_search_index_create_failed", format!("无法创建本地搜索索引目录 {}: {error}", index_dir.display()), ) })?; let index_path = index_dir.join("search-index.json"); let payload = serde_json::to_string_pretty(index) .map_err(|error| WebError::internal(format!("本地搜索索引序列化失败: {error}")))?; fs::write(&index_path, format!("{payload}\n")).map_err(|error| { WebError::bad_request_code( "local_search_index_write_failed", format!("无法写入本地搜索索引 {}: {error}", index_path.display()), ) }) } fn clear_local_search_index(root_path: &Path) -> Result<(), WebError> { let index_path = root_path .join(".mnote") .join("index") .join("search-index.json"); let evidence_path = evidence_sqlite_path(root_path); for path in [index_path, evidence_path] { match fs::remove_file(&path) { Ok(()) => {} Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} Err(error) => { return Err(WebError::bad_request_code( "local_search_index_delete_failed", format!("无法删除本地搜索索引文件 {}: {error}", path.display()), )); } } } Ok(()) } fn ensure_evidence_sqlite_index( root_path: &Path, index: &LocalSearchIndex, ) -> Result<(), WebError> { let path = evidence_sqlite_path(root_path); if path.exists() { return Ok(()); } write_evidence_sqlite_index(root_path, index) } fn write_evidence_sqlite_index(root_path: &Path, index: &LocalSearchIndex) -> Result<(), WebError> { let Some(parent) = evidence_sqlite_path(root_path) .parent() .map(Path::to_path_buf) else { return Ok(()); }; fs::create_dir_all(&parent).map_err(|error| { WebError::bad_request_code( "evidence_index_create_failed", format!("无法创建 evidence 索引目录 {}: {error}", parent.display()), ) })?; let path = evidence_sqlite_path(root_path); let mut connection = Connection::open(&path).map_err(|error| { WebError::bad_request_code( "evidence_index_open_failed", format!("无法打开 evidence 索引 {}: {error}", path.display()), ) })?; let tx = connection.transaction().map_err(sqlite_error)?; create_evidence_schema(&tx)?; tx.execute("DELETE FROM evidence_fts", []) .map_err(sqlite_error)?; for table in [ "evidence_edge", "evidence_section", "evidence_block", "evidence_resource", ] { tx.execute(&format!("DELETE FROM {table}"), []) .map_err(sqlite_error)?; } for document in &index.documents { insert_document_evidence(&tx, index, document)?; } for resource in &index.resources { insert_resource_evidence(&tx, root_path, index, resource)?; } for document in &index.documents { insert_document_graph_edges(&tx, document)?; } tx.execute( "INSERT OR REPLACE INTO evidence_meta(key, value) VALUES('schema_version', ?1)", params![EVIDENCE_SQLITE_SCHEMA_VERSION.to_string()], ) .map_err(sqlite_error)?; tx.execute( "INSERT OR REPLACE INTO evidence_meta(key, value) VALUES('workspace_id', ?1)", params![index.workspace_id], ) .map_err(sqlite_error)?; tx.commit().map_err(sqlite_error) } fn create_evidence_schema(connection: &Connection) -> Result<(), WebError> { connection .execute_batch( r#" CREATE TABLE IF NOT EXISTS evidence_meta( key TEXT PRIMARY KEY, value TEXT NOT NULL ); CREATE TABLE IF NOT EXISTS evidence_resource( resource_id TEXT PRIMARY KEY, owner_document_id TEXT NOT NULL, owner_document_path TEXT NOT NULL, source_root_relative_path TEXT NOT NULL, provider TEXT NOT NULL, source_hash TEXT NOT NULL, artifact_root_relative_path TEXT NOT NULL, source_map_root_relative_path TEXT NOT NULL, updated_at_ms INTEGER NOT NULL ); CREATE TABLE IF NOT EXISTS evidence_block( block_id TEXT PRIMARY KEY, resource_id TEXT NOT NULL, text TEXT NOT NULL, section_path_json TEXT NOT NULL, page_start INTEGER, page_end INTEGER, bbox_json TEXT, char_range_json TEXT, line_range_json TEXT, locator_json TEXT NOT NULL ); CREATE VIRTUAL TABLE IF NOT EXISTS evidence_fts USING fts5( block_id UNINDEXED, text ); CREATE TABLE IF NOT EXISTS evidence_section( section_id TEXT PRIMARY KEY, resource_id TEXT NOT NULL, title TEXT NOT NULL, path_json TEXT NOT NULL, summary TEXT, page_start INTEGER, page_end INTEGER, parent_section_id TEXT ); CREATE TABLE IF NOT EXISTS evidence_edge( edge_id TEXT PRIMARY KEY, from_id TEXT NOT NULL, to_id TEXT NOT NULL, edge_type TEXT NOT NULL, source_block_id TEXT, confidence REAL NOT NULL, created_by TEXT NOT NULL ); "#, ) .map_err(sqlite_error) } fn refresh_evidence_sqlite_index_for_path( root_path: &Path, index: &LocalSearchIndex, relative_path: &str, ) -> Result<(), WebError> { if let Some(parent) = evidence_sqlite_path(root_path) .parent() .map(Path::to_path_buf) { fs::create_dir_all(&parent).map_err(|error| { WebError::bad_request_code( "evidence_index_create_failed", format!("无法创建 evidence 索引目录 {}: {error}", parent.display()), ) })?; } let mut connection = Connection::open(evidence_sqlite_path(root_path)).map_err(|error| { WebError::bad_request_code( "evidence_index_open_failed", format!("无法打开 evidence 索引: {error}"), ) })?; let tx = connection.transaction().map_err(sqlite_error)?; create_evidence_schema(&tx)?; let mut affected_resource_ids = affected_evidence_resource_ids(index, relative_path); affected_resource_ids.extend(existing_evidence_resource_ids_for_path(&tx, relative_path)?); affected_resource_ids.sort(); affected_resource_ids.dedup(); for resource_id in &affected_resource_ids { delete_evidence_resource_projection(&tx, resource_id)?; } if let Some(document) = index .documents .iter() .find(|document| document.path == relative_path) { insert_document_evidence(&tx, index, document)?; insert_document_graph_edges(&tx, document)?; } if let Some(resource) = index .resources .iter() .find(|resource| resource.path == relative_path) { insert_resource_evidence(&tx, root_path, index, resource)?; } tx.execute( "INSERT OR REPLACE INTO evidence_meta(key, value) VALUES('schema_version', ?1)", params![EVIDENCE_SQLITE_SCHEMA_VERSION.to_string()], ) .map_err(sqlite_error)?; tx.execute( "INSERT OR REPLACE INTO evidence_meta(key, value) VALUES('workspace_id', ?1)", params![index.workspace_id], ) .map_err(sqlite_error)?; tx.commit().map_err(sqlite_error) } fn remove_evidence_sqlite_index_for_path( root_path: &Path, index: &LocalSearchIndex, relative_path: &str, ) -> Result<(), WebError> { let path = evidence_sqlite_path(root_path); if !path.exists() { return Ok(()); } let mut connection = Connection::open(&path).map_err(|error| { WebError::bad_request_code( "evidence_index_open_failed", format!("无法打开 evidence 索引 {}: {error}", path.display()), ) })?; let tx = connection.transaction().map_err(sqlite_error)?; create_evidence_schema(&tx)?; let mut affected_resource_ids = affected_evidence_resource_ids(index, relative_path); affected_resource_ids.extend(existing_evidence_resource_ids_for_path(&tx, relative_path)?); affected_resource_ids.sort(); affected_resource_ids.dedup(); for resource_id in &affected_resource_ids { delete_evidence_resource_projection(&tx, resource_id)?; } tx.commit().map_err(sqlite_error) } fn affected_evidence_resource_ids(index: &LocalSearchIndex, relative_path: &str) -> Vec { let mut ids = Vec::new(); let document_id = format!("local-md:{}", encode_local_id_segment(relative_path)); ids.push(document_id.clone()); ids.push(format!( "local-resource:{}", relative_path.replace('/', "~2F") )); ids.extend( index .documents .iter() .filter(|document| document.path == relative_path) .map(|document| document.document_id.clone()), ); ids.extend( index .resources .iter() .filter(|resource| resource.path == relative_path) .map(|resource| resource.resource_id.clone()), ); if let Some(stripped) = relative_path.strip_suffix(".ocr.md") { ids.push(format!("{document_id}#ocr:{stripped}")); } ids } fn existing_evidence_resource_ids_for_path( connection: &Connection, relative_path: &str, ) -> Result, WebError> { let mut statement = connection .prepare( "SELECT resource_id FROM evidence_resource \ WHERE source_root_relative_path = ?1 \ OR artifact_root_relative_path = ?1 \ OR source_map_root_relative_path = ?1", ) .map_err(sqlite_error)?; let resource_ids = statement .query_map(params![relative_path], |row| row.get::<_, String>(0)) .map_err(sqlite_error)? .collect::, _>>() .map_err(sqlite_error)?; Ok(resource_ids) } fn delete_evidence_resource_projection( connection: &Connection, resource_id: &str, ) -> Result<(), WebError> { let mut statement = connection .prepare("SELECT block_id FROM evidence_block WHERE resource_id = ?1") .map_err(sqlite_error)?; let block_ids = statement .query_map(params![resource_id], |row| row.get::<_, String>(0)) .map_err(sqlite_error)? .collect::, _>>() .map_err(sqlite_error)?; for block_id in &block_ids { connection .execute( "DELETE FROM evidence_fts WHERE block_id = ?1", params![block_id], ) .map_err(sqlite_error)?; connection .execute( "DELETE FROM evidence_edge WHERE source_block_id = ?1", params![block_id], ) .map_err(sqlite_error)?; } connection .execute( "DELETE FROM evidence_edge WHERE from_id = ?1", params![resource_id], ) .map_err(sqlite_error)?; connection .execute( "DELETE FROM evidence_section WHERE resource_id = ?1", params![resource_id], ) .map_err(sqlite_error)?; connection .execute( "DELETE FROM evidence_block WHERE resource_id = ?1", params![resource_id], ) .map_err(sqlite_error)?; connection .execute( "DELETE FROM evidence_resource WHERE resource_id = ?1", params![resource_id], ) .map(|_| ()) .map_err(sqlite_error) } fn insert_document_evidence( connection: &Connection, index: &LocalSearchIndex, document: &LocalSearchDocument, ) -> Result<(), WebError> { let resource_id = document.document_id.as_str(); let artifact = markdown_parsed_artifact(document); insert_evidence_resource_from_artifact(connection, resource_id, &artifact)?; for section in markdown_evidence_sections(document) { insert_markdown_evidence_section(connection, resource_id, §ion)?; } let blocks = markdown_evidence_blocks(document); if blocks.is_empty() { let block_id = format!("{resource_id}#line1"); let line_range = json!({"start": 1, "end": 1}); let locator = markdown_locator( index, &document.document_id, &document.path, &block_id, Vec::new(), Some(line_range.clone()), ); return insert_evidence_block_with_metadata( connection, &block_id, resource_id, &document.raw_text, Vec::new(), Some(line_range), locator, ); } for block in blocks { let locator = markdown_locator( index, &document.document_id, &document.path, &block.block_id, block.section_path.clone(), Some(block.line_range.clone()), ); insert_evidence_block_with_metadata( connection, &block.block_id, resource_id, &block.text, block.section_path, Some(block.line_range), locator, )?; } Ok(()) } fn markdown_parsed_artifact(document: &LocalSearchDocument) -> ParsedResourceArtifact { ParsedResourceArtifact { schema: PARSED_RESOURCE_ARTIFACT_SCHEMA.into(), provider: "markdown".into(), model_version: None, owner_document_id: document.document_id.clone(), owner_document_path: document.path.clone(), source_root_relative_path: document.path.clone(), source_hash: format!("updated:{}", document.updated_at), artifact_root_relative_path: document.path.clone(), source_map_root_relative_path: format!("{}.source-map.json", document.path), updated_at_ms: document.updated_at as u64, } } fn insert_evidence_resource_from_artifact( connection: &Connection, resource_id: &str, artifact: &ParsedResourceArtifact, ) -> Result<(), WebError> { insert_evidence_resource( connection, resource_id, &artifact.owner_document_id, &artifact.owner_document_path, &artifact.source_root_relative_path, &artifact.provider, &artifact.source_hash, &artifact.artifact_root_relative_path, &artifact.source_map_root_relative_path, artifact.updated_at_ms as u128, ) } fn insert_resource_evidence( connection: &Connection, root_path: &Path, index: &LocalSearchIndex, resource: &LocalSearchResource, ) -> Result<(), WebError> { insert_evidence_resource( connection, &resource.resource_id, &resource.resource_id, &resource.path, &resource.path, "resource", &format!("updated:{}", resource.updated_at), &resource.path, "", resource.updated_at, )?; let locator = json!({ "schema": "mnote.evidence_locator.v1", "rootUri": index.root_uri, "ownerDocumentId": resource.resource_id, "ownerDocumentPath": resource.path, "resourcePath": resource.path, "resourceKind": evidence_resource_kind_for_type(&resource.resource_type), "openAction": { "actionType": "mnote.open_resource_locator", "url": format!("/?treeView=filetree&sourceKind=local_folder&rootUri={}", encode_query_component(&index.root_uri)), "params": {"resourcePath": resource.path} } }); insert_evidence_block( connection, &format!("{}#resource", resource.resource_id), &resource.resource_id, &format!("{}\n{}", resource.title, resource.path), locator, )?; if let Some(output) = parse_resource_evidence_artifact(root_path, index, resource)? { insert_source_map_artifact_evidence( connection, index, &parsed_resource_id(resource), &output.artifact, &output.source_map, &output.markdown, )?; } Ok(()) } fn parse_resource_evidence_artifact( _root_path: &Path, _index: &LocalSearchIndex, _resource: &LocalSearchResource, ) -> Result, WebError> { Ok(None) } fn parsed_resource_id(resource: &LocalSearchResource) -> String { format!("{}#parse", resource.resource_id) } fn insert_source_map_artifact_evidence( connection: &Connection, index: &LocalSearchIndex, resource_id: &str, artifact: &ParsedResourceArtifact, source_map: &ResourceSourceMap, fallback_text: &str, ) -> Result<(), WebError> { insert_evidence_resource_from_artifact(connection, resource_id, artifact)?; for section in &source_map.sections { insert_source_map_evidence_section(connection, resource_id, section)?; } let mut inserted = 0_usize; for page in &source_map.pages { for block in &page.blocks { if block.text.trim().is_empty() { continue; } let block_id = format!("{resource_id}#{}", block.id); let section_path = source_map_section_path_for_block(source_map, &block.id); let locator = source_map_block_locator( index, artifact, &block.id, Some(page.page), block, section_path.clone(), ); insert_evidence_block_with_metadata( connection, &block_id, resource_id, &block.text, section_path, None, locator, )?; inserted += 1; } } if inserted == 0 { let block_id = format!("{resource_id}#artifact"); let fallback_block = empty_source_map_block(fallback_text); let locator = source_map_block_locator( index, artifact, &fallback_block.id, None, &fallback_block, Vec::new(), ); insert_evidence_block_with_metadata( connection, &block_id, resource_id, fallback_text, Vec::new(), None, locator, )?; } Ok(()) } fn insert_source_map_evidence_section( connection: &Connection, resource_id: &str, section: &core_protocol::SourceMapSection, ) -> Result<(), WebError> { let path_json = serde_json::to_string(§ion.path).map_err(|error| { WebError::bad_request_code( "evidence_index_section_serialize_failed", format!("source-map section path 序列化失败: {error}"), ) })?; connection .execute( "INSERT OR REPLACE INTO evidence_section(section_id, resource_id, title, path_json, summary, page_start, page_end, parent_section_id) VALUES(?1, ?2, ?3, ?4, NULL, ?5, ?6, NULL)", params![ format!("{resource_id}#{}", section.id), resource_id, section.title, path_json, section.page_start, section.page_end, ], ) .map(|_| ()) .map_err(sqlite_error) } fn source_map_section_path_for_block( source_map: &ResourceSourceMap, block_id: &str, ) -> Vec { source_map .sections .iter() .find(|section| section.block_ids.iter().any(|id| id == block_id)) .map(|section| section.path.clone()) .unwrap_or_default() } fn source_map_block_locator( index: &LocalSearchIndex, artifact: &ParsedResourceArtifact, source_map_block_id: &str, page: Option, block: &SourceMapBlock, section_path: Vec, ) -> Value { let mut params = json!({ "resourcePath": artifact.source_root_relative_path, "sourceMapPath": artifact.source_map_root_relative_path, "blockId": source_map_block_id, }); if let Some(params_object) = params.as_object_mut() { if let Some(page) = page { params_object.insert("page".into(), json!(page)); } if let Some(bbox) = &block.bbox { params_object.insert("bbox".into(), json!(bbox)); } if let Some(char_range) = &block.char_range { params_object.insert("charRange".into(), json!(char_range)); } } json!({ "schema": "mnote.evidence_locator.v1", "rootUri": index.root_uri, "ownerDocumentId": artifact.owner_document_id, "ownerDocumentPath": artifact.owner_document_path, "resourcePath": artifact.source_root_relative_path, "resourceKind": evidence_resource_kind_for_path(&artifact.source_root_relative_path), "page": page, "bbox": block.bbox, "sectionPath": section_path, "charRange": block.char_range, "blockId": source_map_block_id, "sourceMapPath": artifact.source_map_root_relative_path, "openAction": { "actionType": "mnote.open_resource_locator", "url": format!("/documents/{}?sourceKind=local_folder&rootUri={}", artifact.owner_document_id, encode_query_component(&index.root_uri)), "params": params } }) } fn empty_source_map_block(text: &str) -> SourceMapBlock { SourceMapBlock { id: "artifact".into(), block_type: core_protocol::SourceMapBlockKind::Text, text: text.to_string(), bbox: None, char_range: None, } } fn insert_evidence_resource( connection: &Connection, resource_id: &str, owner_document_id: &str, owner_document_path: &str, source_root_relative_path: &str, provider: &str, source_hash: &str, artifact_root_relative_path: &str, source_map_root_relative_path: &str, updated_at_ms: u128, ) -> Result<(), WebError> { connection .execute( "INSERT OR REPLACE INTO evidence_resource(resource_id, owner_document_id, owner_document_path, source_root_relative_path, provider, source_hash, artifact_root_relative_path, source_map_root_relative_path, updated_at_ms) VALUES(?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)", params![ resource_id, owner_document_id, owner_document_path, source_root_relative_path, provider, source_hash, artifact_root_relative_path, source_map_root_relative_path, updated_at_ms.to_string(), ], ) .map(|_| ()) .map_err(sqlite_error) } fn insert_evidence_block( connection: &Connection, block_id: &str, resource_id: &str, text: &str, locator: Value, ) -> Result<(), WebError> { insert_evidence_block_with_metadata( connection, block_id, resource_id, text, Vec::new(), None, locator, ) } fn insert_evidence_block_with_metadata( connection: &Connection, block_id: &str, resource_id: &str, text: &str, section_path: Vec, line_range: Option, locator: Value, ) -> Result<(), WebError> { let locator_json = serde_json::to_string(&locator).map_err(|error| { WebError::bad_request_code( "evidence_index_locator_serialize_failed", format!("locator 序列化失败: {error}"), ) })?; let section_path_json = serde_json::to_string(§ion_path).map_err(|error| { WebError::bad_request_code( "evidence_index_section_serialize_failed", format!("sectionPath 序列化失败: {error}"), ) })?; let line_range_json = line_range .map(|value| serde_json::to_string(&value)) .transpose() .map_err(|error| { WebError::bad_request_code( "evidence_index_line_range_serialize_failed", format!("lineRange 序列化失败: {error}"), ) })?; connection .execute( "INSERT OR REPLACE INTO evidence_block(block_id, resource_id, text, section_path_json, page_start, page_end, bbox_json, char_range_json, line_range_json, locator_json) VALUES(?1, ?2, ?3, ?4, NULL, NULL, NULL, NULL, ?5, ?6)", params![block_id, resource_id, text, section_path_json, line_range_json, locator_json], ) .map_err(sqlite_error)?; connection .execute( "INSERT INTO evidence_fts(block_id, text) VALUES(?1, ?2)", params![block_id, text], ) .map(|_| ()) .map_err(sqlite_error) } #[derive(Debug, Clone)] struct MarkdownEvidenceBlock { block_id: String, text: String, section_path: Vec, line_range: Value, } #[derive(Debug, Clone)] struct MarkdownEvidenceSection { section_id: String, title: String, path: Vec, line_start: u64, parent_section_id: Option, } fn markdown_evidence_blocks(document: &LocalSearchDocument) -> Vec { let mut headings: Vec = Vec::new(); let mut blocks = Vec::new(); for (index, line) in document.raw_text.lines().enumerate() { let line_number = (index + 1) as u64; let trimmed = line.trim(); if trimmed.is_empty() { continue; } if let Some((level, title)) = markdown_heading(trimmed) { headings.truncate(level.saturating_sub(1)); headings.push(title); } blocks.push(MarkdownEvidenceBlock { block_id: format!("{}#line{}", document.document_id, line_number), text: trimmed.to_string(), section_path: headings.clone(), line_range: json!({"start": line_number, "end": line_number}), }); } blocks } fn markdown_evidence_sections(document: &LocalSearchDocument) -> Vec { let mut headings: Vec = Vec::new(); let mut sections = Vec::new(); for (index, line) in document.raw_text.lines().enumerate() { let line_number = (index + 1) as u64; let Some((level, title)) = markdown_heading(line.trim()) else { continue; }; headings.truncate(level.saturating_sub(1)); headings.push(title.clone()); let path = headings.clone(); let parent_section_id = if path.len() > 1 { Some(format!( "{}#section:{}", document.document_id, encode_local_id_segment(&path[..path.len() - 1].join("/")) )) } else { None }; sections.push(MarkdownEvidenceSection { section_id: format!( "{}#section:{}", document.document_id, encode_local_id_segment(&path.join("/")) ), title, path, line_start: line_number, parent_section_id, }); } sections } fn markdown_heading(line: &str) -> Option<(usize, String)> { let trimmed = line.trim_start(); let marker_count = trimmed.chars().take_while(|value| *value == '#').count(); if marker_count == 0 || marker_count > 6 { return None; } let rest = trimmed.get(marker_count..)?.trim(); if rest.is_empty() { return None; } Some((marker_count, rest.trim_matches('#').trim().to_string())) } fn insert_markdown_evidence_section( connection: &Connection, resource_id: &str, section: &MarkdownEvidenceSection, ) -> Result<(), WebError> { let path_json = serde_json::to_string(§ion.path).map_err(|error| { WebError::bad_request_code( "evidence_index_section_serialize_failed", format!("section path 序列化失败: {error}"), ) })?; connection .execute( "INSERT OR REPLACE INTO evidence_section(section_id, resource_id, title, path_json, summary, page_start, page_end, parent_section_id) VALUES(?1, ?2, ?3, ?4, NULL, ?5, ?5, ?6)", params![ section.section_id, resource_id, section.title, path_json, section.line_start, section.parent_section_id, ], ) .map(|_| ()) .map_err(sqlite_error) } fn insert_document_graph_edges( connection: &Connection, document: &LocalSearchDocument, ) -> Result<(), WebError> { let source_block_id = first_markdown_evidence_block_id(document); for target in &document.backlinks { let to_id = markdown_reference_target_id(&document.path, target); insert_evidence_edge( connection, &document.document_id, &to_id, "markdown_links_to", &source_block_id, )?; } for target in &document.resource_refs { let to_id = resource_reference_target_id(&document.path, target); insert_evidence_edge( connection, &document.document_id, &to_id, "resource_refers_to", &source_block_id, )?; insert_evidence_edge( connection, &document.document_id, &to_id, "document_contains_resource", &source_block_id, )?; } for mention in markdown_mention_edges(document) { insert_evidence_edge( connection, &document.document_id, &mention.target_id, "mentions", &mention.source_block_id, )?; } Ok(()) } fn first_markdown_evidence_block_id(document: &LocalSearchDocument) -> String { document .raw_text .lines() .enumerate() .find(|(_, line)| !line.trim().is_empty()) .map(|(index, _)| format!("{}#line{}", document.document_id, index + 1)) .unwrap_or_else(|| format!("{}#line1", document.document_id)) } fn insert_evidence_edge( connection: &Connection, from_id: &str, to_id: &str, edge_type: &str, source_block_id: &str, ) -> Result<(), WebError> { let has_source_locator = connection .query_row( "SELECT 1 FROM evidence_block WHERE block_id = ?1 AND locator_json <> '' LIMIT 1", params![source_block_id], |_| Ok(true), ) .optional() .map_err(sqlite_error)? .unwrap_or(false); if !has_source_locator { return Ok(()); } let edge_id = format!( "{}:{}:{}", edge_type, encode_local_id_segment(from_id), encode_local_id_segment(to_id) ); connection .execute( "INSERT OR REPLACE INTO evidence_edge(edge_id, from_id, to_id, edge_type, source_block_id, confidence, created_by) VALUES(?1, ?2, ?3, ?4, ?5, 1.0, 'deterministic_markdown')", params![edge_id, from_id, to_id, edge_type, source_block_id], ) .map(|_| ()) .map_err(sqlite_error) } fn markdown_reference_target_id(owner_path: &str, target: &str) -> String { let target = target.trim(); if target.starts_with("local-md:") { return target.to_string(); } if is_markdown_reference(target) { return format!( "local-md:{}", encode_local_id_segment(&normalize_local_reference_path(owner_path, target)) ); } format!("markdown-title:{}", encode_local_id_segment(target)) } fn resource_reference_target_id(owner_path: &str, target: &str) -> String { format!( "local-resource:{}", encode_local_id_segment(&normalize_local_reference_path(owner_path, target)) ) } #[derive(Debug, Clone)] struct MarkdownMentionEdge { target_id: String, source_block_id: String, } fn markdown_mention_edges(document: &LocalSearchDocument) -> Vec { let mut seen = std::collections::BTreeSet::new(); let mut mentions = Vec::new(); for (line_index, line) in document.raw_text.lines().enumerate() { let source_block_id = format!("{}#line{}", document.document_id, line_index + 1); for mention in extract_line_mentions(line) { let target_id = format!("entity:{}", encode_local_id_segment(&mention)); if seen.insert((target_id.clone(), source_block_id.clone())) { mentions.push(MarkdownMentionEdge { target_id, source_block_id: source_block_id.clone(), }); } } } mentions } fn extract_line_mentions(line: &str) -> Vec { let chars = line.chars().collect::>(); let mut mentions = Vec::new(); let mut index = 0; while index < chars.len() { if chars[index] != '@' { index += 1; continue; } if index > 0 && chars[index - 1].is_alphanumeric() { index += 1; continue; } let mut end = index + 1; while end < chars.len() && (chars[end].is_alphanumeric() || chars[end] == '_' || chars[end] == '-' || chars[end] == '.') { end += 1; } if end > index + 1 { let value = chars[index + 1..end].iter().collect::(); mentions.push(value); } index = end.max(index + 1); } mentions.sort(); mentions.dedup(); mentions } fn markdown_locator( index: &LocalSearchIndex, document_id: &str, path: &str, block_id: &str, section_path: Vec, line_range: Option, ) -> Value { json!({ "schema": "mnote.evidence_locator.v1", "rootUri": index.root_uri, "ownerDocumentId": document_id, "ownerDocumentPath": path, "resourcePath": path, "resourceKind": "markdown", "sectionPath": section_path, "lineRange": line_range, "blockId": block_id, "openAction": { "actionType": "mnote.open_resource_locator", "url": format!("/documents/{document_id}?sourceKind=local_folder&rootUri={}", encode_query_component(&index.root_uri)), "params": {"resourcePath": path, "blockId": block_id, "lineRange": line_range} } }) } fn evidence_sqlite_path(root_path: &Path) -> PathBuf { root_path .join(".mnote") .join("index") .join("evidence.sqlite") } fn count_evidence_blocks(path: &Path) -> Result { let connection = Connection::open(path).map_err(|error| { WebError::bad_request_code( "evidence_index_open_failed", format!("无法打开 evidence 索引 {}: {error}", path.display()), ) })?; connection .query_row("SELECT COUNT(*) FROM evidence_block", [], |row| { row.get::<_, u64>(0) }) .map_err(sqlite_error) } #[allow(dead_code)] #[derive(Debug, Clone, Default)] pub(crate) struct LocalEvidenceSourceStatuses { pub(crate) indexed_paths: BTreeSet, pub(crate) failed_paths: BTreeSet, } #[allow(dead_code)] pub(crate) fn local_evidence_source_statuses( root_path: &Path, ) -> Result { let mut statuses = LocalEvidenceSourceStatuses::default(); let evidence_path = evidence_sqlite_path(root_path); if evidence_path.exists() { let connection = Connection::open(&evidence_path).map_err(|error| { WebError::bad_request_code( "evidence_index_open_failed", format!( "无法打开 evidence 索引 {}: {error}", evidence_path.display() ), ) })?; let mut statement = connection .prepare( "SELECT DISTINCT source_root_relative_path \ FROM evidence_resource \ WHERE provider NOT IN ('resource', 'markdown')", ) .map_err(sqlite_error)?; let rows = statement .query_map([], |row| row.get::<_, String>(0)) .map_err(sqlite_error)?; for row in rows { if let Ok(path) = row { if !path.trim().is_empty() { statuses.indexed_paths.insert(path); } } } } if let Some(index) = read_local_search_index(root_path)? { for resource in index.resources { if matches!(resource.resource_type.as_str(), "pdf" | "office") && !statuses.indexed_paths.contains(&resource.path) { statuses.failed_paths.insert(resource.path); } } } Ok(statuses) } fn evidence_resource_kind_for_type(resource_type: &str) -> &'static str { match resource_type { "mindmap" => "mindmap", "office" => "office", "pdf" => "pdf", "image" => "image", "markdown" => "markdown", _ => "raw_file", } } fn evidence_resource_kind_for_path(path: &str) -> &'static str { match Path::new(path) .extension() .and_then(|value| value.to_str()) .unwrap_or_default() .to_ascii_lowercase() .as_str() { "pdf" => "pdf", "png" | "jpg" | "jpeg" | "webp" => "image", "doc" | "docx" | "xls" | "xlsx" | "ppt" | "pptx" => "office", "md" => "markdown", "mm" | "mindmap" => "mindmap", _ => "raw_file", } } fn local_resource_path_from_document_id(document_id: &str) -> Option { document_id .trim() .strip_prefix("local-resource:") .map(|value| value.replace("~2F", "/").replace("~2f", "/")) .filter(|value| !value.trim().is_empty()) } #[allow(dead_code)] fn path_source_map_path(path: &str) -> Option { if let Some(stripped) = path.strip_suffix(".ocr.md") { return Some(format!("{stripped}.source-map.json")); } if let Some(stripped) = path.strip_suffix(".parse.md") { return Some(format!("{stripped}.source-map.json")); } None } fn is_local_evidence_sidecar_relative_path(relative_path: &str) -> bool { let normalized = relative_path.trim().replace('\\', "/"); let path = Path::new(&normalized); let Some(file_name) = path.file_name().and_then(|value| value.to_str()) else { return false; }; let Some(parent_name) = path .parent() .and_then(|value| value.file_name()) .and_then(|value| value.to_str()) else { return false; }; parent_name.ends_with(".ocr") && (file_name.ends_with(".parse.md") || file_name.ends_with(".source-map.json")) } fn normalize_local_reference_path(owner_path: &str, target: &str) -> String { let target = target .split('#') .next() .unwrap_or(target) .split('?') .next() .unwrap_or(target) .trim(); if target.starts_with('/') { return target.trim_start_matches('/').replace('\\', "/"); } let owner_parent = Path::new(owner_path) .parent() .unwrap_or_else(|| Path::new("")); normalize_path_components(owner_parent.join(target)) } fn normalize_path_components(path: PathBuf) -> String { let mut parts = Vec::new(); for component in path.components() { match component { Component::Normal(value) => parts.push(value.to_string_lossy().to_string()), Component::ParentDir => { parts.pop(); } Component::CurDir => {} _ => {} } } parts.join("/") } fn sqlite_error(error: rusqlite::Error) -> WebError { WebError::bad_request_code( "evidence_index_sqlite_error", format!("evidence SQLite 错误: {error}"), ) } fn local_search_document_matches( document: &LocalSearchDocument, query: &str, title_only: bool, exact: bool, ) -> bool { if query.is_empty() { return false; } let haystack = if title_only { normalize_search_text(&document.title) } else { normalize_search_text(&format!( "{}\n{}\n{}\n{}", document.title, document.raw_text, document.tags.join(" "), document.resource_refs.join(" ") )) }; if exact { haystack.contains(query) } else { token_search_match(&haystack, query) } } fn local_search_resource_matches( resource: &LocalSearchResource, query: &str, title_only: bool, exact: bool, ) -> bool { if query.is_empty() { return false; } let haystack = if title_only { normalize_search_text(&resource.title) } else { normalize_search_text(&format!( "{}\n{}\n{}", resource.title, resource.path, resource.resource_type )) }; if exact { haystack.contains(query) } else { token_search_match(&haystack, query) } } fn local_search_document_projection( document: &LocalSearchDocument, root_uri: &str, query: &str, ) -> Value { let hit = search_document_hit(document, query); json!({ "id": document.document_id, "documentId": document.document_id, "title": document.title, "path": document.path, "resourceType": "markdown", "sourceKind": "local_folder", "rootUri": root_uri, "snippet": hit.snippet, "blockId": hit.block_id, "lineRange": { "start": hit.line_number, "end": hit.line_number, }, "tags": document.tags, "backlinks": document.backlinks, "resourceRefs": document.resource_refs, "updatedAt": document.updated_at, "publicPath": format!( "/documents/{}?sourceKind=local_folder&rootUri={}", document.document_id, encode_query_component(root_uri), ) }) } fn local_search_resource_projection(resource: &LocalSearchResource, root_uri: &str) -> Value { json!({ "id": resource.resource_id, "documentId": resource.resource_id, "title": resource.title, "path": resource.path, "resourceType": resource.resource_type, "sourceKind": "local_folder", "rootUri": root_uri, "updatedAt": resource.updated_at, "publicPath": format!( "/?treeView=filetree&sourceKind=local_folder&rootUri={}", encode_query_component(root_uri), ) }) } fn encode_query_component(value: &str) -> String { let mut encoded = String::with_capacity(value.len()); for byte in value.as_bytes() { if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~') { encoded.push(*byte as char); } else { encoded.push_str(&format!("%{byte:02X}")); } } encoded } fn local_recent_changes_projection(documents: &[LocalSearchDocument], root_uri: &str) -> Value { let mut sorted = documents.iter().collect::>(); sorted.sort_by(|left, right| right.updated_at.cmp(&left.updated_at)); Value::Array( sorted .into_iter() .take(20) .map(|document| { json!({ "id": document.document_id, "documentId": document.document_id, "title": document.title, "path": document.path, "resourceType": "markdown", "sourceKind": "local_folder", "rootUri": root_uri, "updatedAt": document.updated_at }) }) .collect(), ) } fn normalized_document_targets( document: &LocalSearchDocument, ) -> std::collections::BTreeSet { let mut values = std::collections::BTreeSet::new(); values.insert(normalize_search_text(&document.document_id)); values.insert(normalize_search_text(&document.path)); values.insert(normalize_search_text(&document.title)); if let Some(encoded) = document.document_id.strip_prefix("local-md:") { values.insert(normalize_search_text(&encoded.replace("~2F", "/"))); } values } fn extract_tags(markdown: &str) -> Vec { let Some(frontmatter) = split_frontmatter(markdown).0 else { return Vec::new(); }; for line in frontmatter.lines() { let trimmed = line.trim(); let Some(value) = trimmed .strip_prefix("tags:") .or_else(|| trimmed.strip_prefix("tag:")) else { continue; }; return value .trim() .trim_matches(['[', ']']) .split([',', ' ']) .map(|item| item.trim().trim_matches(['"', '\''])) .filter(|item| !item.is_empty()) .map(ToOwned::to_owned) .collect(); } Vec::new() } fn extract_backlinks(markdown: &str) -> Vec { let mut values = extract_markdown_links(markdown) .into_iter() .filter(|target| is_markdown_reference(target)) .collect::>(); let mut rest = markdown; while let Some(start) = rest.find("[[") { let after_start = &rest[start + 2..]; let Some(end) = after_start.find("]]") else { break; }; let link = after_start[..end].trim(); if !link.is_empty() { values.push(link.to_string()); } rest = &after_start[end + 2..]; } values.sort(); values.dedup(); values } fn extract_resource_refs(markdown: &str) -> Vec { let mut values = markdown .lines() .filter_map(|line| parse_markdown_attachment_link(line.trim())) .map(|(_, target)| target) .chain( extract_markdown_links(markdown) .into_iter() .filter(|target| { !is_markdown_reference(target) && is_local_resource_reference(target) }), ) .collect::>(); values.sort(); values.dedup(); values } fn extract_markdown_links(markdown: &str) -> Vec { let mut links = Vec::new(); let mut rest = markdown; while let Some(label_end) = rest.find("](") { let after = &rest[label_end + 2..]; let Some(target_end) = after.find(')') else { break; }; let target = after[..target_end].trim(); if !target.is_empty() { links.push(target.to_string()); } rest = &after[target_end + 1..]; } links } fn is_markdown_reference(target: &str) -> bool { let lower = target.to_lowercase(); lower.ends_with(".md") || lower.ends_with(".markdown") || lower.starts_with("local-md:") } fn is_local_resource_reference(target: &str) -> bool { let lower = target.to_lowercase(); !lower.starts_with("http://") && !lower.starts_with("https://") && !lower.starts_with("mailto:") && !lower.starts_with('#') && Path::new(target).extension().is_some() } fn search_snippet(document: &LocalSearchDocument, query: &str) -> String { search_document_hit(document, query).snippet } #[derive(Debug, Clone)] struct SearchDocumentHit { snippet: String, line_number: usize, block_id: String, } fn search_document_hit(document: &LocalSearchDocument, query: &str) -> SearchDocumentHit { for (line_index, line) in document.raw_text.lines().enumerate() { let trimmed = line.trim(); if trimmed.is_empty() { continue; } if normalize_search_text(trimmed).contains(query) { let line_number = line_index + 1; return SearchDocumentHit { snippet: trimmed.chars().take(180).collect(), line_number, block_id: format!("{}#line{}", document.document_id, line_number), }; } } let line_number = document .raw_text .lines() .enumerate() .find(|(_, line)| !line.trim().is_empty()) .map(|(line_index, _)| line_index + 1) .unwrap_or(1); SearchDocumentHit { snippet: document.raw_text.chars().take(180).collect(), line_number, block_id: format!("{}#line{}", document.document_id, line_number), } } #[derive(Debug, Clone)] struct EvidenceQueryTerm { display: String, normalized: String, alternatives: Vec, } #[derive(Debug, Clone)] struct EvidenceTextMatch { score: f64, matched_terms: Vec, missing_terms: Vec, match_mode: String, strong: bool, } impl EvidenceTextMatch { fn into_match_info(self, rank: Option) -> EvidenceSearchMatchInfo { EvidenceSearchMatchInfo { rank, matched_terms: self.matched_terms, missing_terms: self.missing_terms, match_mode: self.match_mode, match_scope: "block".into(), strong: self.strong, } } } fn score_evidence_text_match(text: &str, query: &str) -> Option { let normalized_text = normalize_search_text(text); let normalized_query = normalize_search_text(query); if normalized_query.is_empty() { return None; } let terms = evidence_query_terms(&normalized_query); if terms.is_empty() { return None; } let phrase_exact = normalized_text.contains(&normalized_query); let mut matched_terms = Vec::new(); let mut missing_terms = Vec::new(); let mut exact_count = 0usize; let mut partial_count = 0usize; let mut positions = Vec::new(); for term in &terms { if let Some(index) = normalized_text.find(&term.normalized) { exact_count += 1; positions.push(index); push_unique(&mut matched_terms, term.display.clone()); continue; } let partial_matches = term .alternatives .iter() .filter_map(|alternative| { normalized_text .find(alternative) .map(|index| (alternative.clone(), index)) }) .collect::>(); let partial_allowed = !partial_matches.is_empty() && if terms.len() > 1 { true } else { partial_matches.len() >= 2 }; if partial_allowed { partial_count += 1; if let Some((_, index)) = partial_matches.first() { positions.push(*index); } for (alternative, _) in partial_matches { push_unique(&mut matched_terms, alternative); } push_unique(&mut missing_terms, term.display.clone()); } else { return None; } } let term_count = terms.len().max(1) as f64; let coverage = (exact_count as f64 + partial_count as f64 * 0.45) / term_count; let proximity = if positions.len() > 1 { let min = positions.iter().min().copied().unwrap_or(0); let max = positions.iter().max().copied().unwrap_or(min); let span = max.saturating_sub(min); if span <= 80 { 0.1 } else if span <= 240 { 0.05 } else { 0.0 } } else { 0.0 }; let score = if phrase_exact { 1.0 } else { (coverage * 0.85 + proximity).min(0.95) }; let match_mode = if phrase_exact { "exact_phrase" } else if partial_count == 0 { "all_terms" } else { "cjk_partial" }; Some(EvidenceTextMatch { score, matched_terms, missing_terms, match_mode: match_mode.into(), strong: phrase_exact || partial_count == 0 && exact_count == terms.len(), }) } fn evidence_query_terms(query: &str) -> Vec { split_search_query_tokens(query) .into_iter() .filter(|token| !token.is_empty()) .map(|token| { let alternatives = cjk_token_bigrams(&token) .into_iter() .filter(|alternative| alternative != &token) .collect::>(); EvidenceQueryTerm { display: token.clone(), normalized: token, alternatives, } }) .collect() } fn split_search_query_tokens(query: &str) -> Vec { let mut tokens = Vec::new(); let mut current = String::new(); for ch in query.chars() { if ch.is_whitespace() || is_search_separator(ch) { if !current.is_empty() { tokens.push(std::mem::take(&mut current)); } } else { current.push(ch); } } if !current.is_empty() { tokens.push(current); } tokens } fn is_search_separator(ch: char) -> bool { matches!( ch, ',' | ';' | ':' | ',' | ';' | ':' | '。' | '、' | '(' | ')' | '[' | ']' | '{' | '}' ) } fn cjk_token_bigrams(token: &str) -> Vec { let chars = token.chars().collect::>(); if chars.len() < 3 || !chars.iter().any(|ch| is_cjk_char(*ch)) { return Vec::new(); } chars .windows(2) .filter_map(|window| { if window.iter().any(|ch| is_cjk_stop_char(*ch)) { return None; } Some(window.iter().collect::()) }) .collect() } fn is_cjk_char(ch: char) -> bool { matches!( ch as u32, 0x3400..=0x4DBF | 0x4E00..=0x9FFF | 0xF900..=0xFAFF ) } fn is_cjk_stop_char(ch: char) -> bool { matches!( ch, '的' | '了' | '和' | '与' | '及' | '在' | '是' | '为' | '对' | '中' ) } fn push_unique(values: &mut Vec, value: String) { if !value.is_empty() && !values.iter().any(|existing| existing == &value) { values.push(value); } } fn ocr_search_snippet(body: &str, query: &str) -> String { let normalized_body = body.replace('\n', " "); let normalized_query = query.trim(); if normalized_query.is_empty() { return normalized_body.chars().take(160).collect(); } let lower = normalized_body.to_ascii_lowercase(); let lower_query = normalized_query.to_ascii_lowercase(); if let Some(byte_index) = lower.find(&lower_query) { let start = normalized_body[..byte_index] .char_indices() .rev() .nth(40) .map(|(idx, _)| idx) .unwrap_or(0); normalized_body[start..].chars().take(160).collect() } else if let Some(byte_index) = fuzzy_search_start_byte(&normalized_body, normalized_query) { let start = normalized_body[..byte_index] .char_indices() .rev() .nth(40) .map(|(idx, _)| idx) .unwrap_or(0); normalized_body[start..].chars().take(180).collect() } else { normalized_body.chars().take(160).collect() } } fn ocr_search_snippet_for_terms(body: &str, query: &str, terms: &[String]) -> String { let normalized_body = body.replace('\n', " "); let normalized_query = query.trim(); if !normalized_query.is_empty() { let lower = normalized_body.to_lowercase(); let lower_query = normalized_query.to_lowercase(); if let Some(byte_index) = lower.find(&lower_query) { return snippet_from_byte_index(&normalized_body, byte_index, 180); } } let lower = normalized_body.to_lowercase(); for term in terms { let term = term.trim().to_lowercase(); if term.is_empty() { continue; } if let Some(byte_index) = lower.find(&term) { return snippet_from_byte_index(&normalized_body, byte_index, 180); } } ocr_search_snippet(body, query) } fn snippet_from_byte_index(body: &str, byte_index: usize, len: usize) -> String { let start = body[..byte_index] .char_indices() .rev() .nth(40) .map(|(idx, _)| idx) .unwrap_or(0); body[start..].chars().take(len).collect() } fn fuzzy_search_start_byte(haystack: &str, query: &str) -> Option { if query.is_empty() { return None; } let query_chars = query .chars() .filter(|ch| !ch.is_whitespace()) .collect::>(); if query_chars.is_empty() { return None; } let haystack_chars = haystack.char_indices().collect::>(); for (start_index, (byte_index, ch)) in haystack_chars.iter().enumerate() { if ch != &query_chars[0] { continue; } let mut query_index = 1usize; for (_, next_ch) in haystack_chars.iter().skip(start_index + 1) { if next_ch.is_whitespace() { continue; } if query_index < query_chars.len() && next_ch == &query_chars[query_index] { query_index += 1; if query_index >= query_chars.len() { return Some(*byte_index); } } } } None } fn token_search_match(haystack: &str, query: &str) -> bool { score_evidence_text_match(haystack, query).is_some() } fn is_markdown_path(path: &Path) -> bool { path.extension() .and_then(|value| value.to_str()) .map(|value| value.eq_ignore_ascii_case("md") || value.eq_ignore_ascii_case("markdown")) .unwrap_or(false) } fn resource_type_from_path(path: &Path) -> Option<&'static str> { let file_name = path .file_name() .and_then(|value| value.to_str()) .unwrap_or_default() .to_lowercase(); if file_name.ends_with(".mindmap.json") { return Some("mindmap"); } let extension = path .extension() .and_then(|value| value.to_str()) .unwrap_or_default() .to_lowercase(); match extension.as_str() { "png" | "jpg" | "jpeg" | "webp" | "bmp" | "gif" | "tif" | "tiff" | "svg" => Some("image"), "pdf" => Some("pdf"), "doc" | "docx" | "odt" | "ppt" | "pptx" | "odp" | "xls" | "xlsx" | "ods" => Some("office"), _ => None, } } fn normalize_search_text(value: &str) -> String { value.trim().to_lowercase() } fn now_ms() -> u128 { SystemTime::now() .duration_since(UNIX_EPOCH) .map(|value| value.as_millis()) .unwrap_or_default() } #[cfg(test)] mod tests { use super::*; use control_plane::SqliteControlPlaneStore; use std::path::PathBuf; use std::sync::{Mutex, OnceLock}; fn env_lock() -> &'static Mutex<()> { static LOCK: OnceLock> = OnceLock::new(); LOCK.get_or_init(|| Mutex::new(())) } fn temp_root(name: &str) -> PathBuf { let root = std::env::temp_dir().join(format!("{name}-{}", std::process::id())); let _ = fs::remove_dir_all(&root); fs::create_dir_all(root.join("docs")).expect("create root"); root } #[test] fn local_search_index_extracts_backlinks_resource_refs_and_tags() { let root = temp_root("mnote-local-search-index"); fs::write( root.join("README.md"), "---\ntitle: Home\ntags: [alpha, beta]\n---\n# Home\nSee [[Daily]] and [Child](docs/child.md).\n[Spec](assets/spec.pdf)\n[Map](maps/idea.mindmap.json)\n[Sheet](office/report.xlsx)\n![Diagram](assets/diagram.png)\n", ) .expect("write readme"); fs::write( root.join("docs").join("child.md"), "---\nmnote_id: child-page\ntitle: Child\n---\nChild body with alpha and office.xlsx\n", ) .expect("write child"); fs::create_dir_all(root.join("maps")).expect("create maps"); fs::create_dir_all(root.join("office")).expect("create office"); fs::write( root.join("maps").join("idea.mindmap.json"), r#"{"title":"Idea Map","nodes":[]}"#, ) .expect("write mindmap"); fs::create_dir_all(root.join("assets")).expect("create assets"); fs::write(root.join("assets").join("spec.pdf"), b"%PDF-1.4\n").expect("write pdf"); fs::write(root.join("assets").join("diagram.png"), b"png").expect("write image"); fs::write(root.join("office").join("report.xlsx"), b"office bytes").expect("write office"); write_local_index_settings(&root, &[String::from(".")], None, None, None, None) .expect("settings"); let projection = query_local_search_index( &root, &format!("file://{}", root.display()), "local-ws-test", "alpha", None, 10, false, false, false, ) .expect("projection"); let results = projection["results"].as_array().expect("results"); let home = results .iter() .find(|item| item["documentId"].as_str() == Some("local-md:README.md")) .expect("home result"); assert!(home["tags"] .as_array() .unwrap() .iter() .any(|tag| tag.as_str() == Some("alpha"))); assert!(home["backlinks"] .as_array() .unwrap() .iter() .any(|link| link.as_str() == Some("Daily"))); assert!(home["backlinks"] .as_array() .unwrap() .iter() .any(|link| link.as_str() == Some("docs/child.md"))); assert!(home["resourceRefs"] .as_array() .unwrap() .iter() .any(|reference| reference.as_str() == Some("assets/spec.pdf"))); assert!(home["resourceRefs"] .as_array() .unwrap() .iter() .any(|reference| reference.as_str() == Some("maps/idea.mindmap.json"))); assert!(home["resourceRefs"] .as_array() .unwrap() .iter() .any(|reference| reference.as_str() == Some("office/report.xlsx"))); assert!(home["publicPath"] .as_str() .is_some_and(|path| path.starts_with( "/documents/local-md:README.md?sourceKind=local_folder&rootUri=file%3A%2F%2F" ))); let evidence_db = root.join(".mnote").join("index").join("evidence.sqlite"); let connection = Connection::open(&evidence_db).expect("open evidence sqlite"); let markdown_edge_count: i64 = connection .query_row( "SELECT COUNT(*) FROM evidence_edge WHERE edge_type = 'markdown_links_to'", [], |row| row.get(0), ) .expect("markdown edge count"); let resource_edge_count: i64 = connection .query_row( "SELECT COUNT(*) FROM evidence_edge WHERE edge_type = 'resource_refers_to'", [], |row| row.get(0), ) .expect("resource edge count"); let contains_edge_count: i64 = connection .query_row( "SELECT COUNT(*) FROM evidence_edge WHERE edge_type = 'document_contains_resource'", [], |row| row.get(0), ) .expect("contains edge count"); assert!(markdown_edge_count >= 2); assert!(resource_edge_count >= 4); assert!(contains_edge_count >= 4); // 含 mnote_id 的 Markdown 仍返回路径型 documentId,不应出现 local-mdid: let child_search = query_local_search_index( &root, &format!("file://{}", root.display()), "local-ws-test", "office.xlsx", None, 10, false, false, false, ) .expect("child search"); let child_result = child_search["results"] .as_array() .unwrap() .iter() .find(|item| item["path"].as_str() == Some("docs/child.md")) .expect("child in search results"); assert_eq!( child_result["documentId"].as_str(), Some("local-md:docs~2Fchild.md") ); assert_ne!( child_result["documentId"].as_str(), Some("local-mdid:child-page") ); assert!(root .join(".mnote") .join("index") .join("search-index.json") .exists()); let mindmap_projection = query_local_search_index( &root, &format!("file://{}", root.display()), "local-ws-test", "idea.mindmap", None, 10, false, false, false, ) .expect("mindmap projection"); assert!(mindmap_projection["results"] .as_array() .unwrap() .iter() .any(|item| item["resourceType"].as_str() == Some("mindmap") && item["path"].as_str() == Some("maps/idea.mindmap.json") && item["publicPath"] .as_str() .is_some_and(|path| path .starts_with("/?treeView=filetree&sourceKind=local_folder&rootUri=")) && item["publicPath"] .as_str() .is_some_and(|path| !path.starts_with("/tree?")))); let office_projection = query_local_search_index( &root, &format!("file://{}", root.display()), "local-ws-test", "report.xlsx", None, 10, false, false, false, ) .expect("office projection"); assert!(office_projection["results"] .as_array() .unwrap() .iter() .any(|item| item["resourceType"].as_str() == Some("office") && item["path"].as_str() == Some("office/report.xlsx"))); let pdf_projection = query_local_search_index( &root, &format!("file://{}", root.display()), "local-ws-test", "spec.pdf", None, 10, false, false, false, ) .expect("pdf projection"); assert!(pdf_projection["results"] .as_array() .unwrap() .iter() .any(|item| item["resourceType"].as_str() == Some("pdf") && item["path"].as_str() == Some("assets/spec.pdf"))); let image_projection = query_local_search_index( &root, &format!("file://{}", root.display()), "local-ws-test", "diagram.png", None, 10, false, false, false, ) .expect("image projection"); assert!(image_projection["results"] .as_array() .unwrap() .iter() .any(|item| item["resourceType"].as_str() == Some("image") && item["path"].as_str() == Some("assets/diagram.png"))); let _ = fs::remove_dir_all(&root); } #[test] fn local_search_two_char_cjk_query_requires_real_match() { assert!(score_evidence_text_match("13. N -甲基吗啉 N -氧化物", "吗啉").is_some()); assert!(score_evidence_text_match("Scope Alpha SOURCE SCOPE RAG", "吗啉").is_none()); } #[test] fn local_index_settings_restricts_search_and_evidence_scope() { let root = temp_root("mnote-local-index-settings-scope"); fs::create_dir_all(root.join("docs").join("included")).expect("create included"); fs::create_dir_all(root.join("docs").join("excluded")).expect("create excluded"); fs::write( root.join("docs").join("included").join("keep.md"), "# Keep\nScopedToken\n", ) .expect("write included"); fs::write( root.join("docs").join("excluded").join("skip.md"), "# Skip\nExcludedToken\n", ) .expect("write excluded"); let root_uri = format!("file://{}", root.display()); write_local_index_settings( &root, &[String::from("docs/included")], None, None, None, None, ) .expect("settings"); refresh_local_search_index(&root, &root_uri, "local-ws-scope").expect("refresh"); let included = query_local_search_index( &root, &root_uri, "local-ws-scope", "ScopedToken", None, 10, false, false, false, ) .expect("included query"); assert_eq!(included["results"].as_array().unwrap().len(), 1); let excluded = query_local_search_index( &root, &root_uri, "local-ws-scope", "ExcludedToken", None, 10, false, false, false, ) .expect("excluded query"); assert_eq!(excluded["results"].as_array().unwrap().len(), 0); let evidence = query_evidence_sqlite_results(&root, "ScopedToken", None, 10) .expect("evidence query") .expect("evidence index"); assert_eq!(evidence.len(), 1); let status = local_index_status(&root, &root_uri, "local-ws-scope").expect("status"); assert_eq!( status["settings"]["includePaths"].as_array().unwrap()[0].as_str(), Some("docs/included") ); assert_eq!(status["documentCount"].as_u64(), Some(1)); let _ = fs::remove_dir_all(&root); } #[test] fn local_index_settings_accepts_absolute_path_under_root_as_frozen_scope() { let root = temp_root("mnote-local-index-settings-absolute"); fs::create_dir_all(root.join("docs").join("absolute")).expect("create absolute dir"); let absolute_scope = root.join("docs").join("absolute"); let settings = write_local_index_settings( &root, &[absolute_scope.to_string_lossy().to_string()], None, None, None, None, ) .expect("absolute scope under root"); assert_eq!(settings.include_paths, vec![String::from("docs/absolute")]); let _ = fs::remove_dir_all(&root); } #[test] fn user_local_index_empty_scope_deletes_index_files() { let root = temp_root("mnote-local-index-empty-delete"); fs::write( root.join("docs").join("keep.md"), "# Keep\nDeleteIndexToken\n", ) .expect("write doc"); let root_uri = format!("file://{}", root.display()); let workspace_id = "local-ws-empty-delete"; let store = SqliteControlPlaneStore::in_memory().expect("init control plane"); write_user_local_index_settings( &store, "alice", workspace_id, &root, &[String::from("docs")], Some("manual"), Some("02:00"), None, Some(false), ) .expect("write indexed scope"); let effective = effective_local_index_settings_for_root(&store, workspace_id, &root) .expect("effective indexed"); refresh_local_search_index_with_settings(&root, &root_uri, workspace_id, &effective) .expect("refresh indexed"); assert!(root.join(".mnote/index/search-index.json").exists()); assert!(root.join(".mnote/index/evidence.sqlite").exists()); write_user_local_index_settings( &store, "alice", workspace_id, &root, &[], Some("manual"), Some("02:00"), None, Some(false), ) .expect("delete indexed scopes"); let empty_effective = effective_local_index_settings_for_root(&store, workspace_id, &root) .expect("effective empty"); assert!(empty_effective.include_paths.is_empty()); refresh_local_search_index_with_settings(&root, &root_uri, workspace_id, &empty_effective) .expect("clear index files"); assert!(!root.join(".mnote/index/search-index.json").exists()); assert!(!root.join(".mnote/index/evidence.sqlite").exists()); let status = local_index_status_with_settings( &root, &root_uri, workspace_id, &empty_effective, &empty_effective, ) .expect("status"); assert_eq!(status["cacheMatchesSettings"].as_bool(), Some(true)); let _ = fs::remove_dir_all(&root); } #[test] fn local_index_settings_rejects_escape_and_internal_cache_scope() { let root = temp_root("mnote-local-index-settings-escape"); let escape = write_local_index_settings( &root, &[String::from("../outside")], None, None, None, None, ) .expect_err("escape should fail"); assert_eq!(escape.code(), "local_search_index_path_escape"); let internal = write_local_index_settings( &root, &[String::from(".mnote/index")], None, None, None, None, ) .expect_err("internal cache should fail"); assert_eq!(internal.code(), "local_index_scope_reserved"); let _ = fs::remove_dir_all(&root); } #[test] fn local_index_settings_defaults_and_schedule_are_persisted() { let root = temp_root("mnote-local-index-settings-schedule"); let root_uri = format!("file://{}", root.display()); let initial_status = local_index_status(&root, &root_uri, "local-ws-schedule").expect("initial status"); assert_eq!( initial_status["settings"]["scheduleMode"].as_str(), Some("daily") ); assert_eq!( initial_status["settings"]["scheduleTime"].as_str(), Some("02:00") ); assert_eq!( initial_status["settings"]["runOnChange"].as_bool(), Some(false) ); assert_eq!( initial_status["settings"]["includePaths"] .as_array() .map(Vec::len), Some(0), "默认不应索引工作区根目录;用户新增范围后才开始索引" ); assert_eq!(initial_status["cacheMatchesSettings"].as_bool(), Some(true)); let settings = write_local_index_settings( &root, &[String::from("docs")], Some("once"), Some("08:30"), Some("2026-06-04"), Some(true), ) .expect("write schedule settings"); assert_eq!(settings.include_paths, vec![String::from("docs")]); assert_eq!(settings.schedule_mode, "once"); assert_eq!(settings.schedule_time, "08:30"); assert_eq!(settings.schedule_date.as_deref(), Some("2026-06-04")); assert!(settings.run_on_change); let status = local_index_status(&root, &root_uri, "local-ws-schedule").expect("status"); assert_eq!(status["settings"]["scheduleMode"].as_str(), Some("once")); assert_eq!(status["settings"]["scheduleTime"].as_str(), Some("08:30")); assert_eq!( status["settings"]["scheduleDate"].as_str(), Some("2026-06-04") ); assert_eq!(status["settings"]["runOnChange"].as_bool(), Some(true)); let _ = fs::remove_dir_all(&root); } #[test] fn local_index_settings_rejects_invalid_schedule() { let root = temp_root("mnote-local-index-settings-invalid-schedule"); let invalid_mode = write_local_index_settings( &root, &[String::from(".")], Some("weekly"), None, None, None, ) .expect_err("invalid mode should fail"); assert_eq!(invalid_mode.code(), "local_index_schedule_mode_invalid"); let invalid_time = write_local_index_settings( &root, &[String::from(".")], Some("daily"), Some("24:00"), None, None, ) .expect_err("invalid time should fail"); assert_eq!(invalid_time.code(), "local_index_schedule_time_invalid"); let invalid_date = write_local_index_settings( &root, &[String::from(".")], Some("once"), Some("02:00"), Some("2026-13-01"), None, ) .expect_err("invalid date should fail"); assert_eq!(invalid_date.code(), "local_index_schedule_date_invalid"); let _ = fs::remove_dir_all(&root); } #[test] fn local_index_schedule_due_respects_modes() { let today = OffsetDateTime::now_local() .unwrap_or_else(|_| OffsetDateTime::now_utc()) .date(); let yesterday = today - Duration::days(1); let tomorrow = today + Duration::days(1); let yesterday = format!( "{:04}-{:02}-{:02}", yesterday.year(), u8::from(yesterday.month()), yesterday.day() ); let tomorrow = format!( "{:04}-{:02}-{:02}", tomorrow.year(), u8::from(tomorrow.month()), tomorrow.day() ); let mut settings = default_local_index_settings(); settings.schedule_mode = "manual".to_string(); assert!(!local_index_schedule_is_due(&settings, 0)); settings.schedule_mode = "once".to_string(); settings.schedule_time = "00:00".to_string(); settings.schedule_date = Some(yesterday); assert!(local_index_schedule_is_due(&settings, 0)); settings.schedule_date = Some(tomorrow); assert!(!local_index_schedule_is_due(&settings, 0)); settings.schedule_mode = "daily".to_string(); settings.schedule_date = None; settings.schedule_time = "00:00".to_string(); assert!(local_index_schedule_is_due(&settings, 0)); assert!(!local_index_schedule_is_due(&settings, now_ms())); } #[test] fn multi_user_local_index_settings_share_one_cache_and_filter_results() { let root = temp_root("mnote-local-index-multi-user"); fs::create_dir_all(root.join("docs").join("alice")).expect("create alice dir"); fs::create_dir_all(root.join("docs").join("bob")).expect("create bob dir"); fs::write( root.join("docs").join("alice").join("keep.md"), "# Alice\nAliceScopedToken\n", ) .expect("write alice doc"); fs::write( root.join("docs").join("bob").join("keep.md"), "# Bob\nBobScopedToken\n", ) .expect("write bob doc"); let root_uri = format!("file://{}", root.display()); let workspace_id = "local-ws-multi-user"; let store = SqliteControlPlaneStore::in_memory().expect("init control plane"); write_user_local_index_settings( &store, "alice", workspace_id, &root, &[String::from("docs/alice")], Some("manual"), Some("02:00"), None, Some(false), ) .expect("write alice settings"); write_user_local_index_settings( &store, "bob", workspace_id, &root, &[String::from("docs/bob")], Some("manual"), Some("02:00"), None, Some(true), ) .expect("write bob settings"); let sqlite_records = store .list_user_ui_preferences_for_scope( Some(workspace_id), Some(LOCAL_INDEX_SOURCE_KIND), LOCAL_INDEX_SETTINGS_SCOPE_KIND, &local_index_scope_id(&root), LOCAL_INDEX_SETTINGS_KEY, ) .expect("read raw sqlite preference rows"); assert_eq!(sqlite_records.len(), 2); let alice_sqlite_record = sqlite_records .iter() .find(|record| record.user_id == "alice") .expect("alice sqlite preference row"); let alice_sqlite_value: serde_json::Value = serde_json::from_str(&alice_sqlite_record.value_json).expect("alice sqlite json"); assert_eq!( alice_sqlite_value["includePaths"], serde_json::json!(["docs/alice"]), "索引范围必须写入 SQLite user_ui_preferences.value_json" ); let alice_settings = read_user_local_index_settings(&store, "alice", workspace_id, &root).expect("alice"); let bob_settings = read_user_local_index_settings(&store, "bob", workspace_id, &root).expect("bob"); let effective = effective_local_index_settings_for_root(&store, workspace_id, &root) .expect("effective"); assert_eq!( effective.include_paths, vec![String::from("docs/alice"), String::from("docs/bob")] ); assert!(effective.run_on_change); refresh_local_search_index_with_settings(&root, &root_uri, workspace_id, &effective) .expect("refresh shared index"); let alice_results = query_local_search_index_with_settings( &root, &root_uri, workspace_id, &effective, &alice_settings, "AliceScopedToken", None, 10, false, false, false, ) .expect("alice query"); assert_eq!(alice_results["results"].as_array().map(Vec::len), Some(1)); let alice_hidden = query_local_search_index_with_settings( &root, &root_uri, workspace_id, &effective, &alice_settings, "BobScopedToken", None, 10, false, false, false, ) .expect("alice hidden query"); assert_eq!(alice_hidden["results"].as_array().map(Vec::len), Some(0)); let bob_results = query_local_search_index_with_settings( &root, &root_uri, workspace_id, &effective, &bob_settings, "BobScopedToken", None, 10, false, false, false, ) .expect("bob query"); assert_eq!(bob_results["results"].as_array().map(Vec::len), Some(1)); let status = local_index_status_with_settings( &root, &root_uri, workspace_id, &alice_settings, &effective, ) .expect("status"); assert_eq!( status["settings"]["includePaths"].as_array().map(Vec::len), Some(1) ); assert_eq!( status["effectiveSettings"]["includePaths"] .as_array() .map(Vec::len), Some(2) ); let _ = fs::remove_dir_all(&root); } #[test] fn local_index_schedule_due_uses_any_user_schedule() { let root = temp_root("mnote-local-index-schedule-any-user"); let workspace_id = "local-ws-schedule-any-user"; let store = SqliteControlPlaneStore::in_memory().expect("init control plane"); let today = OffsetDateTime::now_local() .unwrap_or_else(|_| OffsetDateTime::now_utc()) .date(); let yesterday = today - Duration::days(1); let yesterday = format!( "{:04}-{:02}-{:02}", yesterday.year(), u8::from(yesterday.month()), yesterday.day() ); write_user_local_index_settings( &store, "alice", workspace_id, &root, &[String::from(".")], Some("manual"), Some("02:00"), None, Some(false), ) .expect("write alice settings"); write_user_local_index_settings( &store, "bob", workspace_id, &root, &[String::from(".")], Some("once"), Some("00:00"), Some(&yesterday), Some(false), ) .expect("write bob settings"); assert!( local_index_any_schedule_due_for_root(&store, workspace_id, &root, 0) .expect("schedule due"), "任一用户到期就应该触发共享索引重建" ); let _ = fs::remove_dir_all(&root); } #[test] fn refresh_local_search_index_for_change_path_with_store_uses_sqlite_effective_settings() { let root = temp_root("mnote-local-index-change-with-store"); fs::create_dir_all(root.join("docs").join("included")).expect("create included"); fs::create_dir_all(root.join("docs").join("excluded")).expect("create excluded"); fs::write( root.join("docs").join("included").join("keep.md"), "# Keep\nKeepScopedToken\n", ) .expect("write included"); fs::write( root.join("docs").join("excluded").join("skip.md"), "# Skip\nSkipScopedToken\n", ) .expect("write excluded"); let root_uri = format!("file://{}", root.display()); let workspace_id = "local-ws-change-with-store"; let store = SqliteControlPlaneStore::in_memory().expect("init control plane"); write_local_index_settings( &root, &[String::from("docs/excluded")], Some("manual"), Some("02:00"), None, Some(false), ) .expect("write stale cache settings"); write_user_local_index_settings( &store, "alice", workspace_id, &root, &[String::from("docs/included")], Some("manual"), Some("02:00"), None, Some(true), ) .expect("write sqlite settings"); let refreshed = refresh_local_search_index_for_change_path_with_store( &store, &root, &root_uri, workspace_id, "docs/included/keep.md", ) .expect("refresh") .expect("refresh result"); assert_eq!(refreshed["documentCount"].as_u64(), Some(1)); let keep_results = query_local_search_index_with_settings( &root, &root_uri, workspace_id, &effective_local_index_settings_for_root(&store, workspace_id, &root) .expect("effective"), &read_user_local_index_settings(&store, "alice", workspace_id, &root).expect("alice"), "KeepScopedToken", None, 10, false, false, false, ) .expect("keep query"); assert_eq!(keep_results["results"].as_array().map(Vec::len), Some(1)); let skip_results = query_local_search_index_with_settings( &root, &root_uri, workspace_id, &effective_local_index_settings_for_root(&store, workspace_id, &root) .expect("effective"), &read_user_local_index_settings(&store, "alice", workspace_id, &root).expect("alice"), "SkipScopedToken", None, 10, false, false, false, ) .expect("skip query"); assert_eq!(skip_results["results"].as_array().map(Vec::len), Some(0)); let _ = fs::remove_dir_all(&root); } #[test] fn local_search_index_incrementally_updates_single_markdown_path() { let root = temp_root("mnote-local-search-index-incremental"); let root_uri = format!("file://{}", root.display()); let workspace_id = "local-ws-incremental"; fs::write(root.join("README.md"), "# Home\nRoot body.\n").expect("write home"); fs::write( root.join("docs").join("child.md"), "---\ntitle: Child\n---\n# Child\nOriginal body.\n", ) .expect("write child"); write_local_index_settings(&root, &[String::from(".")], None, None, None, None) .expect("settings"); refresh_local_search_index(&root, &root_uri, workspace_id).expect("initial refresh"); fs::write( root.join("docs").join("child.md"), "---\ntitle: Child Updated\n---\n# Child Updated\nChangedToken body.\n", ) .expect("update child"); refresh_local_search_index_for_path(&root, &root_uri, workspace_id, "docs/child.md") .expect("incremental update"); let index = read_local_search_index(&root) .expect("read index") .expect("index exists"); assert!(index .documents .iter() .any(|document| document.path == "README.md")); let child = index .documents .iter() .find(|document| document.path == "docs/child.md") .expect("child document"); // 本地 Markdown 的树标题统一来自文件名;frontmatter/H1 只进入正文和索引文本。 assert_eq!(child.title, "child"); assert!(child.raw_text.contains("ChangedToken")); let changed_evidence = query_evidence_sqlite_results(&root, "ChangedToken", None, 10) .expect("changed evidence") .expect("sqlite exists"); assert_eq!(changed_evidence.len(), 1); assert_eq!( changed_evidence[0].source.owner_document_path, "docs/child.md" ); let original_evidence = query_evidence_sqlite_results(&root, "Original body", None, 10) .expect("original evidence") .expect("sqlite exists"); assert!(original_evidence.is_empty()); fs::remove_file(root.join("docs").join("child.md")).expect("remove child"); refresh_local_search_index_for_path(&root, &root_uri, workspace_id, "docs/child.md") .expect("incremental remove"); let index = read_local_search_index(&root) .expect("read index") .expect("index exists"); assert!(!index .documents .iter() .any(|document| document.path == "docs/child.md")); assert!(index .documents .iter() .any(|document| document.path == "README.md")); let removed_evidence = query_evidence_sqlite_results(&root, "ChangedToken", None, 10) .expect("removed evidence") .expect("sqlite exists"); assert!(removed_evidence.is_empty()); let root_evidence = query_evidence_sqlite_results(&root, "Root body", None, 10) .expect("root evidence") .expect("sqlite exists"); assert_eq!(root_evidence.len(), 1); let _ = fs::remove_dir_all(&root); } #[test] fn local_search_ocr_sidecar_is_hidden_after_lightrag_retirement() { let root = temp_root("mnote-local-search-ocr"); let root_uri = format!("file://{}", root.display()); let workspace_id = "local-ws-ocr"; fs::write(root.join("docs").join("Page.md"), "# Page\n正文\n").expect("page"); fs::create_dir_all(root.join("docs").join("Page.assets")).expect("assets"); fs::write( root.join("docs").join("Page.assets").join("photo.png"), b"png", ) .expect("photo"); fs::create_dir_all(root.join("docs").join("Page.ocr")).expect("ocr dir"); fs::write( root.join("docs").join("Page.ocr").join("photo.png.ocr.md"), "---\nmnote_ocr_version: 1\nprovider: mock\nmodel_version: vlm\nowner_document: ../Page.md\nsource_path: ./Page.assets/photo.png\nsource_root_relative_path: docs/Page.assets/photo.png\nsource_size: 3\nsource_mtime_ms: 1\nstatus: done\ncreated_at: 1\nupdated_at: 1\n---\n\nOCR-only-token 识别正文\n", ) .expect("ocr markdown"); fs::write( root.join("docs").join("Page.ocr").join("photo.png-704905.ocr.md"), "---\nmnote_ocr_version: 1\nprovider: mock\nmodel_version: vlm\nowner_document: ../Page.md\nsource_path: ./Page.assets/photo.png\nsource_root_relative_path: docs/Page.assets/photo.png\nsource_size: 3\nsource_mtime_ms: 2\nstatus: done\ncreated_at: 2\nupdated_at: 2\n---\n\nOCR-hash-token 识别正文\n", ) .expect("hashed ocr markdown"); fs::create_dir_all(root.join(".mnote")).expect("mnote dir"); fs::write( root.join(".mnote").join("ocr-index.json"), serde_json::to_string_pretty(&json!({ "version": 1, "entries": { "docs/Page.assets/photo.png": { "jobId": "ocr_test", "ownerDocumentId": "local-md:docs~2FPage.md", "ownerDocumentPath": "docs/Page.md", "sourceRootRelativePath": "docs/Page.assets/photo.png", "ocrRootRelativePath": "docs/Page.ocr/photo.png.ocr.md", "provider": "mock", "modelVersion": "vlm", "status": "done", "sourceSize": 3, "sourceMtimeMs": 1, "createdAtMs": 1, "updatedAtMs": 1, "plainTextPreview": "OCR-only-token 识别正文" } } })) .expect("serialize index"), ) .expect("ocr index"); write_local_index_settings(&root, &[String::from(".")], None, None, None, None) .expect("settings"); let without_ocr = query_local_search_index( &root, &root_uri, workspace_id, "OCR-only-token", None, 10, false, false, false, ) .expect("without ocr"); assert_eq!(without_ocr["results"].as_array().map(Vec::len), Some(0)); let without_hashed_ocr = query_local_search_index( &root, &root_uri, workspace_id, "OCR-hash-token", None, 10, false, false, false, ) .expect("without hashed ocr"); assert_eq!( without_hashed_ocr["results"].as_array().map(Vec::len), Some(0) ); let with_ocr = query_local_search_index( &root, &root_uri, workspace_id, "OCR-only-token", None, 10, false, false, true, ) .expect("with ocr"); assert_eq!(with_ocr["results"].as_array().map(Vec::len), Some(0)); let index = read_local_search_index(&root) .expect("read search index") .expect("search index"); assert!(!index .documents .iter() .any(|document| document.path == "docs/Page.ocr/photo.png.ocr.md")); assert!(!index .documents .iter() .any(|document| document.path == "docs/Page.ocr/photo.png-704905.ocr.md")); refresh_local_search_index_for_path( &root, &root_uri, workspace_id, "docs/Page.ocr/photo.png-704905.ocr.md", ) .expect("refresh hashed ocr sidecar"); let refreshed_index = read_local_search_index(&root) .expect("read refreshed search index") .expect("refreshed search index"); assert!(!refreshed_index .documents .iter() .any(|document| document.path == "docs/Page.ocr/photo.png-704905.ocr.md")); let refreshed_evidence = query_evidence_sqlite_results(&root, "OCR-hash-token", None, 10) .expect("query refreshed evidence") .unwrap_or_default(); assert!(refreshed_evidence.is_empty()); let _ = fs::remove_dir_all(&root); } #[test] fn local_search_query_reads_existing_index_without_rebuilding() { let root = temp_root("mnote-local-search-query-cache"); let root_uri = format!("file://{}", root.display()); let workspace_id = "local-ws-query-cache"; let child_path = root.join("docs").join("child.md"); fs::write( &child_path, "---\ntitle: Child\n---\n# Child\nOriginalToken body.\n", ) .expect("write child"); write_local_index_settings(&root, &[String::from(".")], None, None, None, None) .expect("settings"); let first_projection = query_local_search_index( &root, &root_uri, workspace_id, "OriginalToken", None, 10, false, false, false, ) .expect("first query"); assert_eq!( first_projection["results"].as_array().map(Vec::len), Some(1) ); fs::write( &child_path, "---\ntitle: Child\n---\n# Child\nUnindexedToken body.\n", ) .expect("update child without refresh"); let stale_projection = query_local_search_index( &root, &root_uri, workspace_id, "UnindexedToken", None, 10, false, false, false, ) .expect("query existing index"); assert_eq!( stale_projection["results"].as_array().map(Vec::len), Some(0) ); refresh_local_search_index_for_path(&root, &root_uri, workspace_id, "docs/child.md") .expect("incremental refresh"); let refreshed_projection = query_local_search_index( &root, &root_uri, workspace_id, "UnindexedToken", None, 10, false, false, false, ) .expect("query refreshed index"); assert_eq!( refreshed_projection["results"].as_array().map(Vec::len), Some(1) ); let _ = fs::remove_dir_all(&root); } #[test] fn evidence_sqlite_query_returns_locator_results() { let root = temp_root("mnote-evidence-sqlite-query"); let root_uri = format!("file://{}", root.display()); let workspace_id = "local-ws-evidence-sqlite"; fs::write( root.join("README.md"), "# Home\nIntro body.\n## Evidence Section\nEvidenceToken root body.\nAsk @Reasonix for citation.\n", ) .expect("write home"); fs::write( root.join("docs").join("child.md"), "# Child\nEvidenceToken child body.\n", ) .expect("write child"); write_local_index_settings(&root, &[String::from(".")], None, None, None, None) .expect("settings"); refresh_local_search_index(&root, &root_uri, workspace_id).expect("refresh"); let results = query_evidence_sqlite_results(&root, "EvidenceToken", None, 10) .expect("sqlite query") .expect("sqlite exists"); assert!(results.len() >= 2); assert!(results .iter() .any(|result| result.source.owner_document_path == "README.md")); assert!(results .iter() .any(|result| result.source.owner_document_path == "docs/child.md")); assert!(results .iter() .all(|result| result.source.schema == "mnote.evidence_locator.v1")); let home_result = results .iter() .find(|result| result.source.owner_document_path == "README.md") .expect("home evidence result"); assert_eq!( home_result.source.section_path, vec!["Home".to_string(), "Evidence Section".to_string()] ); assert_eq!( home_result .source .line_range .as_ref() .map(|range| range.start), Some(4) ); assert_eq!( home_result.source.block_id.as_deref(), Some("local-md:README.md#line4") ); let scoped = query_evidence_sqlite_results( &root, "EvidenceToken", Some("local-md:docs~2Fchild.md"), 10, ) .expect("scoped sqlite query") .expect("sqlite exists"); assert_eq!(scoped.len(), 1); assert_eq!(scoped[0].source.owner_document_path, "docs/child.md"); let connection = Connection::open(root.join(".mnote").join("index").join("evidence.sqlite")) .expect("open evidence sqlite"); let section_count: i64 = connection .query_row("SELECT COUNT(*) FROM evidence_section", [], |row| { row.get(0) }) .expect("section count"); assert!(section_count >= 3); let mention_source_block: String = connection .query_row( "SELECT source_block_id FROM evidence_edge WHERE edge_type = 'mentions' AND to_id = 'entity:Reasonix'", [], |row| row.get(0), ) .expect("mention edge"); assert_eq!(mention_source_block, "local-md:README.md#line5"); let mention_locator: String = connection .query_row( "SELECT locator_json FROM evidence_block WHERE block_id = ?1", rusqlite::params![mention_source_block], |row| row.get(0), ) .expect("mention source locator"); assert!(mention_locator.contains("mnote.evidence_locator.v1")); let graph_results = query_evidence_graph_results(&root, "Reasonix", None, 10) .expect("graph query") .expect("sqlite exists"); let mention_result = graph_results .iter() .find(|result| result.evidence_id.contains("mentions")) .expect("mention graph result"); assert!(mention_result.quote.contains("mentions:")); assert_eq!( mention_result.source.block_id.as_deref(), Some("local-md:README.md#line5") ); connection .execute("DELETE FROM evidence_edge", []) .expect("delete graph projection"); let search_after_graph_delete = query_evidence_sqlite_results(&root, "EvidenceToken", None, 10) .expect("sqlite query after graph delete") .expect("sqlite exists"); assert!(!search_after_graph_delete.is_empty()); let _ = fs::remove_dir_all(&root); } #[test] fn evidence_sqlite_fuzzy_uses_cjk_terms_without_overrequiring_suffix() { let root = temp_root("mnote-evidence-sqlite-cjk-fuzzy"); let root_uri = format!("file://{}", root.display()); let workspace_id = "local-ws-evidence-cjk"; fs::write( root.join("README.md"), "# 有机合成\n羧酸需要被保护有许多原因,常见做法是形成稳定酯类后再脱保护。\n", ) .expect("write home"); write_local_index_settings(&root, &[String::from(".")], None, None, None, None) .expect("settings"); refresh_local_search_index(&root, &root_uri, workspace_id).expect("refresh"); let exact = query_evidence_sqlite_results_with_mode(&root, "羧酸 保护基", None, 10, true) .expect("exact sqlite query") .expect("sqlite exists"); assert!(exact.is_empty(), "精确短语不应伪造不存在的同串命中"); let fuzzy = query_evidence_sqlite_results_with_mode(&root, "羧酸 保护基", None, 10, false) .expect("fuzzy sqlite query") .expect("sqlite exists"); assert_eq!(fuzzy.len(), 1); let info = fuzzy[0].match_info.as_ref().expect("match info"); assert_eq!(info.match_mode, "cjk_partial"); assert!(!info.strong); assert!(info.matched_terms.iter().any(|term| term == "羧酸")); assert!(info.matched_terms.iter().any(|term| term == "保护")); assert!(info.missing_terms.iter().any(|term| term == "保护基")); assert!(fuzzy[0].quote.contains("羧酸")); let projection = query_local_search_index( &root, &root_uri, workspace_id, "羧酸 保护基", None, 10, false, false, false, ) .expect("local search projection"); assert_eq!(projection["results"].as_array().map(Vec::len), Some(1)); let _ = fs::remove_dir_all(&root); } #[test] fn evidence_sqlite_read_context_returns_anchor_block() { let root = temp_root("mnote-evidence-sqlite-read-context"); let root_uri = format!("file://{}", root.display()); let workspace_id = "local-ws-evidence-read"; fs::write( root.join("README.md"), "# Home\nReadContextToken root body.\n", ) .expect("write home"); write_local_index_settings(&root, &[String::from(".")], None, None, None, None) .expect("settings"); refresh_local_search_index(&root, &root_uri, workspace_id).expect("refresh"); let results = query_evidence_sqlite_results(&root, "ReadContextToken", None, 10) .expect("sqlite query") .expect("sqlite exists"); let locator = results .first() .map(|item| item.source.clone()) .expect("locator"); let context = read_evidence_sqlite_context(&root, &locator, 1, 1) .expect("read context") .expect("sqlite exists"); assert_eq!(context.len(), 2); assert!(context .iter() .all(|item| item.source.owner_document_path == "README.md")); assert!(context .iter() .any(|item| item.quote.contains("ReadContextToken"))); let _ = fs::remove_dir_all(&root); } #[test] #[cfg(unix)] fn evidence_index_does_not_parse_resource_body_with_retired_liteparse_sidecar() { use std::os::unix::fs::PermissionsExt; let _guard = env_lock().lock().expect("env lock"); let root = temp_root("mnote-evidence-resource-liteparse"); let root_uri = format!("file://{}", root.display()); let workspace_id = "local-ws-evidence-resource"; fs::create_dir_all(root.join("docs").join("Page.assets")).expect("assets"); fs::write( root.join("docs").join("Page.md"), "# Page\n[Spec](./Page.assets/spec.pdf)\n", ) .expect("write owner"); fs::write( root.join("docs").join("Page.assets").join("spec.pdf"), b"%PDF-1.4", ) .expect("write pdf"); let lit = root.join("fake-lit"); fs::write( &lit, r#"#!/usr/bin/env bash cat <<'JSON' { "version": "liteparse-index-test", "pages": [{ "page": 2, "width": 612, "height": 792, "blocks": [ {"id": "b1", "type": "heading", "level": 1, "text": "配方要求", "bbox": [72, 90, 180, 120]}, {"id": "b2", "type": "text", "text": "ResourceBodyToken 来自 PDF 正文", "bbox": [72, 130, 330, 160]} ] }] } JSON "#, ) .expect("fake lit"); let mut permissions = fs::metadata(&lit).expect("fake lit metadata").permissions(); permissions.set_mode(0o755); fs::set_permissions(&lit, permissions).expect("chmod fake lit"); let old_bin = std::env::var("MNOTE_LITEPARSE_BIN").ok(); std::env::set_var("MNOTE_LITEPARSE_BIN", &lit); write_local_index_settings(&root, &[String::from(".")], None, None, None, None) .expect("settings"); refresh_local_search_index(&root, &root_uri, workspace_id).expect("refresh"); if let Some(value) = old_bin { std::env::set_var("MNOTE_LITEPARSE_BIN", value); } else { std::env::remove_var("MNOTE_LITEPARSE_BIN"); } let results = query_evidence_sqlite_results(&root, "ResourceBodyToken", None, 10) .expect("sqlite query") .expect("sqlite exists"); assert!( !results .iter() .any(|result| result.quote.contains("ResourceBodyToken")), "LiteParse resource body fallback is retired from active evidence indexing" ); let resource_scoped = query_evidence_sqlite_results_with_mode( &root, "ResourceBodyToken", Some("local-resource:docs~2FPage.assets~2Fspec.pdf"), 10, true, ) .expect("resource scoped sqlite query") .expect("sqlite exists"); assert!( resource_scoped.is_empty(), "retired LiteParse sidecar must not create resource-scoped evidence hits" ); assert!(!root .join("docs") .join("Page.ocr") .join("spec.pdf.parse.md") .exists()); assert!(!root .join("docs") .join("Page.ocr") .join("spec.pdf.source-map.json") .exists()); let _ = fs::remove_dir_all(&root); } }