Files
mnote/rust/crates/mnote-web/src/routes/local_search_index.rs
T

4867 lines
162 KiB
Rust
Raw Normal View History

use crate::error::WebError;
use crate::evidence_parse::{
liteparse_runtime_available, parse_liteparse_input_blocking, ParseInput, ParseProviderMode,
};
2026-05-20 10:43:38 +08:00
use crate::routes::local_folder_source::encode_local_id_segment;
use crate::routes::local_markdown_parser::{
parse_markdown_attachment_link, parse_markdown_page, split_frontmatter,
};
2026-06-01 09:29:12 +08:00
use crate::routes::local_ocr;
use control_plane::{ControlPlaneStore, UpsertUserInput, UpsertUserUiPreferenceInput};
use core_protocol::{
EvidenceLocator, EvidenceSearchResult, ParsedResourceArtifact, ResourceSourceMap,
SourceMapBlock, PARSED_RESOURCE_ARTIFACT_SCHEMA,
};
use rusqlite::{params, Connection, OptionalExtension};
use serde::{Deserialize, Serialize};
2026-05-29 11:13:05 +08:00
use serde_json::{json, Value};
use std::fs;
use std::path::{Component, Path, PathBuf};
use std::time::{SystemTime, UNIX_EPOCH};
use time::{Date, Duration, Month, OffsetDateTime, Time};
const LOCAL_SEARCH_INDEX_VERSION: u32 = 1;
const LOCAL_INDEX_SETTINGS_SCHEMA: &str = "mnote.local_index.settings.v1";
const LOCAL_INDEX_SETTINGS_KEY: &str = "localIndex.settings";
const LOCAL_INDEX_SETTINGS_SCOPE_KIND: &str = "localIndex";
const LOCAL_INDEX_SOURCE_KIND: &str = "local_folder";
const DEFAULT_INDEX_SCHEDULE_MODE: &str = "daily";
const DEFAULT_INDEX_SCHEDULE_TIME: &str = "02:00";
const EVIDENCE_SQLITE_SCHEMA_VERSION: u32 = 1;
fn local_index_scope_id(root_path: &Path) -> String {
root_path.to_string_lossy().replace('\\', "/")
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
struct LocalSearchIndex {
version: u32,
built_at: u128,
root_uri: String,
workspace_id: String,
#[serde(default = "default_indexed_paths")]
indexed_paths: Vec<String>,
documents: Vec<LocalSearchDocument>,
#[serde(default)]
resources: Vec<LocalSearchResource>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct LocalIndexSettings {
pub schema: String,
pub include_paths: Vec<String>,
#[serde(default = "default_index_schedule_mode")]
pub schedule_mode: String,
#[serde(default = "default_index_schedule_time")]
pub schedule_time: String,
#[serde(default)]
pub schedule_date: Option<String>,
#[serde(default = "default_index_run_on_change")]
pub run_on_change: bool,
pub updated_at: u128,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
struct LocalSearchDocument {
document_id: String,
title: String,
path: String,
raw_text: String,
tags: Vec<String>,
backlinks: Vec<String>,
resource_refs: Vec<String>,
updated_at: u128,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
struct LocalSearchResource {
resource_id: String,
resource_type: String,
title: String,
path: String,
updated_at: u128,
}
pub(crate) fn query_local_search_index(
root_path: &Path,
root_uri: &str,
workspace_id: &str,
query: &str,
page_id: Option<&str>,
limit: u32,
title_only: bool,
exact: bool,
2026-06-01 09:29:12 +08:00
include_ocr: bool,
) -> Result<Value, WebError> {
let settings = read_local_index_settings_or_default(root_path)?;
query_local_search_index_with_settings(
root_path,
root_uri,
workspace_id,
&settings,
&settings,
query,
page_id,
limit,
title_only,
exact,
include_ocr,
)
}
pub(crate) fn query_local_search_index_with_settings(
root_path: &Path,
root_uri: &str,
workspace_id: &str,
cache_settings: &LocalIndexSettings,
result_settings: &LocalIndexSettings,
query: &str,
page_id: Option<&str>,
limit: u32,
title_only: bool,
exact: bool,
include_ocr: bool,
) -> Result<Value, WebError> {
let index = load_or_rebuild_local_search_index_with_settings(
root_path,
root_uri,
workspace_id,
cache_settings,
)?;
let normalized_query = normalize_search_text(query);
let page_id = page_id.map(str::trim).filter(|value| !value.is_empty());
let recent_changes = local_recent_changes_projection(&index.documents, root_uri);
let mut results = Vec::new();
for document in index.documents.iter() {
if !index_relative_path_is_included(&document.path, &result_settings.include_paths) {
continue;
}
if let Some(page_id) = page_id {
if document.document_id != page_id {
continue;
}
}
if !local_search_document_matches(document, &normalized_query, title_only, exact) {
continue;
}
results.push(local_search_document_projection(
document,
root_uri,
&normalized_query,
));
if results.len() >= limit.max(1) as usize {
break;
}
}
if page_id.is_none() && results.len() < limit.max(1) as usize {
for resource in index.resources.iter() {
if !index_relative_path_is_included(&resource.path, &result_settings.include_paths) {
continue;
}
if !local_search_resource_matches(resource, &normalized_query, title_only, exact) {
continue;
}
results.push(local_search_resource_projection(resource, root_uri));
if results.len() >= limit.max(1) as usize {
break;
}
}
}
2026-06-01 09:29:12 +08:00
if include_ocr && results.len() < limit.max(1) as usize {
for entry in local_ocr::ocr_index_entries(root_path)? {
if !index_relative_path_is_included(
&entry.source_root_relative_path,
&result_settings.include_paths,
) {
continue;
}
2026-06-01 09:29:12 +08:00
if let Some(page_id) = page_id {
if entry.owner_document_id != page_id {
continue;
}
}
if entry.status != "done" {
continue;
}
let ocr_path = root_path.join(&entry.ocr_root_relative_path);
let markdown = match fs::read_to_string(&ocr_path) {
Ok(markdown) => markdown,
Err(_) => continue,
};
if local_ocr::parse_ocr_frontmatter(&markdown).is_none() {
continue;
}
let body = local_ocr::strip_ocr_frontmatter(&markdown);
if !local_search_ocr_matches(&entry, body, &normalized_query, title_only, exact) {
continue;
}
results.push(local_search_ocr_projection(
&entry,
body,
root_uri,
&normalized_query,
));
if results.len() >= limit.max(1) as usize {
break;
}
}
}
Ok(json!({
"enqueueAssetIds": [],
"projectionOwner": "rust-kernel",
"sourceKind": "local_folder",
"index": {
"version": index.version,
"rootUri": index.root_uri,
"workspaceId": index.workspace_id,
"indexedPaths": index.indexed_paths,
"resultIndexedPaths": result_settings.include_paths,
"builtAt": index.built_at,
"documentCount": index.documents.len(),
2026-06-01 09:29:12 +08:00
"resourceCount": index.resources.len(),
"includeOcr": include_ocr
},
"recentChanges": recent_changes,
"results": results
}))
}
#[cfg(test)]
pub(crate) fn write_local_index_settings(
root_path: &Path,
include_paths: &[String],
schedule_mode: Option<&str>,
schedule_time: Option<&str>,
schedule_date: Option<&str>,
run_on_change: Option<bool>,
) -> Result<LocalIndexSettings, WebError> {
let include_paths = normalize_index_include_paths(root_path, include_paths)?;
let existing = read_local_index_settings_or_default(root_path)
.unwrap_or_else(|_| default_local_index_settings());
let schedule_mode =
normalize_index_schedule_mode(schedule_mode.unwrap_or(&existing.schedule_mode))?;
let schedule_time =
normalize_index_schedule_time(schedule_time.unwrap_or(&existing.schedule_time))?;
let schedule_date =
normalize_index_schedule_date(schedule_date.or(existing.schedule_date.as_deref()))?;
let settings = LocalIndexSettings {
schema: LOCAL_INDEX_SETTINGS_SCHEMA.to_string(),
include_paths,
schedule_mode,
schedule_time,
schedule_date,
run_on_change: run_on_change.unwrap_or(existing.run_on_change),
updated_at: now_ms(),
};
write_local_index_settings_cache(root_path, &settings)?;
Ok(settings)
}
pub(crate) fn read_user_local_index_settings(
store: &dyn ControlPlaneStore,
user_id: &str,
workspace_id: &str,
root_path: &Path,
) -> Result<LocalIndexSettings, WebError> {
let user_id = user_id.trim();
let scope_id = local_index_scope_id(root_path);
if user_id.is_empty() || user_id == "anonymous" {
return Ok(read_local_index_settings_or_default(root_path)
.unwrap_or_else(|_| default_local_index_settings()));
}
let root_preferences = store
.list_user_ui_preferences_for_scope(
Some(workspace_id),
Some(LOCAL_INDEX_SOURCE_KIND),
LOCAL_INDEX_SETTINGS_SCOPE_KIND,
&scope_id,
LOCAL_INDEX_SETTINGS_KEY,
)
.map_err(|error| WebError::internal(format!("SQLite 本地索引设置读取失败: {error}")))?;
let preferences = store
.list_user_ui_preferences(user_id, Some(workspace_id), Some(LOCAL_INDEX_SOURCE_KIND))
.map_err(|error| WebError::internal(format!("SQLite 本地索引设置读取失败: {error}")))?;
let Some(record) = preferences.iter().rev().find(|preference| {
preference.scope_kind == LOCAL_INDEX_SETTINGS_SCOPE_KIND
&& preference.scope_id == scope_id
&& preference.key == LOCAL_INDEX_SETTINGS_KEY
}) else {
if root_preferences.is_empty() {
return Ok(read_local_index_settings_or_default(root_path)
.unwrap_or_else(|_| default_local_index_settings()));
}
return Ok(default_local_index_settings());
};
parse_local_index_settings_value(root_path, &record.value_json)
}
pub(crate) fn write_user_local_index_settings(
store: &dyn ControlPlaneStore,
user_id: &str,
workspace_id: &str,
root_path: &Path,
include_paths: &[String],
schedule_mode: Option<&str>,
schedule_time: Option<&str>,
schedule_date: Option<&str>,
run_on_change: Option<bool>,
) -> Result<LocalIndexSettings, WebError> {
let user_id = user_id.trim();
let scope_id = local_index_scope_id(root_path);
if user_id.is_empty() || user_id == "anonymous" {
return Err(WebError::bad_request_code(
"local_index_settings_auth_required",
"本地索引设置需要登录用户",
));
}
store
.upsert_user(UpsertUserInput {
id: Some(user_id.to_string()),
email: None,
username: user_id.to_string(),
display_name: user_id.to_string(),
role: None,
password_hash: None,
})
.map_err(|error| WebError::internal(format!("SQLite 用户初始化失败: {error}")))?;
let existing = read_user_local_index_settings(store, user_id, workspace_id, root_path)?;
let include_paths = normalize_index_include_paths(root_path, include_paths)?;
let schedule_mode =
normalize_index_schedule_mode(schedule_mode.unwrap_or(&existing.schedule_mode))?;
let schedule_time =
normalize_index_schedule_time(schedule_time.unwrap_or(&existing.schedule_time))?;
let schedule_date =
normalize_index_schedule_date(schedule_date.or(existing.schedule_date.as_deref()))?;
let settings = LocalIndexSettings {
schema: LOCAL_INDEX_SETTINGS_SCHEMA.to_string(),
include_paths,
schedule_mode,
schedule_time,
schedule_date,
run_on_change: run_on_change.unwrap_or(existing.run_on_change),
updated_at: now_ms(),
};
let value_json = serde_json::to_string(&settings)
.map_err(|error| WebError::internal(format!("本地索引设置序列化失败: {error}")))?;
store
.upsert_user_ui_preference(UpsertUserUiPreferenceInput {
id: None,
user_id: user_id.to_string(),
workspace_id: Some(workspace_id.to_string()),
source_kind: Some(LOCAL_INDEX_SOURCE_KIND.to_string()),
scope_kind: LOCAL_INDEX_SETTINGS_SCOPE_KIND.to_string(),
scope_id,
key: LOCAL_INDEX_SETTINGS_KEY.to_string(),
value_json,
})
.map_err(|error| WebError::internal(format!("SQLite 本地索引设置写入失败: {error}")))?;
let effective = effective_local_index_settings_for_root(store, workspace_id, root_path)?;
write_local_index_settings_cache(root_path, &effective)?;
Ok(settings)
}
pub(crate) fn effective_local_index_settings_for_root(
store: &dyn ControlPlaneStore,
workspace_id: &str,
root_path: &Path,
) -> Result<LocalIndexSettings, WebError> {
let scope_id = local_index_scope_id(root_path);
let records = store
.list_user_ui_preferences_for_scope(
Some(workspace_id),
Some(LOCAL_INDEX_SOURCE_KIND),
LOCAL_INDEX_SETTINGS_SCOPE_KIND,
&scope_id,
LOCAL_INDEX_SETTINGS_KEY,
)
.map_err(|error| WebError::internal(format!("SQLite 本地索引设置读取失败: {error}")))?;
let mut effective = read_local_index_settings_or_default(root_path)
.unwrap_or_else(|_| default_local_index_settings());
if records.is_empty() {
return Ok(effective);
}
let mut include_paths = Vec::new();
let mut run_on_change = false;
for record in records {
let settings = parse_local_index_settings_value(root_path, &record.value_json)?;
include_paths.extend(settings.include_paths);
run_on_change |= settings.run_on_change;
}
effective.include_paths = normalize_index_include_paths(root_path, &include_paths)?;
effective.run_on_change = run_on_change;
effective.updated_at = now_ms();
Ok(effective)
}
pub(crate) fn local_index_any_schedule_due_for_root(
store: &dyn ControlPlaneStore,
workspace_id: &str,
root_path: &Path,
built_at_ms: u128,
) -> Result<bool, WebError> {
let scope_id = local_index_scope_id(root_path);
let records = store
.list_user_ui_preferences_for_scope(
Some(workspace_id),
Some(LOCAL_INDEX_SOURCE_KIND),
LOCAL_INDEX_SETTINGS_SCOPE_KIND,
&scope_id,
LOCAL_INDEX_SETTINGS_KEY,
)
.map_err(|error| WebError::internal(format!("SQLite 本地索引设置读取失败: {error}")))?;
if records.is_empty() {
let settings = read_local_index_settings_or_default(root_path)
.unwrap_or_else(|_| default_local_index_settings());
return Ok(local_index_schedule_is_due(&settings, built_at_ms));
}
for record in records {
let settings = parse_local_index_settings_value(root_path, &record.value_json)?;
if local_index_schedule_is_due(&settings, built_at_ms) {
return Ok(true);
}
}
Ok(false)
}
#[cfg(test)]
pub(crate) fn local_index_status(
root_path: &Path,
root_uri: &str,
workspace_id: &str,
) -> Result<Value, WebError> {
let settings = read_local_index_settings_or_default(root_path)?;
local_index_status_for_settings(root_path, root_uri, workspace_id, &settings)
}
fn local_index_status_for_settings(
root_path: &Path,
root_uri: &str,
workspace_id: &str,
settings: &LocalIndexSettings,
) -> Result<Value, WebError> {
let index = read_local_search_index(root_path)?;
let index_path = root_path
.join(".mnote")
.join("index")
.join("search-index.json");
let evidence_path = evidence_sqlite_path(root_path);
let mut document_count = 0usize;
let mut resource_count = 0usize;
let mut built_at = Value::Null;
let mut cache_matches_settings = false;
let scheduled_due = if let Some(index) = index.as_ref() {
document_count = index.documents.len();
resource_count = index.resources.len();
built_at = json!(index.built_at);
cache_matches_settings = index.version == LOCAL_SEARCH_INDEX_VERSION
&& index.root_uri == root_uri
&& index.workspace_id == workspace_id
&& normalized_indexed_paths(&index.indexed_paths) == settings.include_paths;
local_index_schedule_is_due(&settings, index.built_at)
} else {
local_index_schedule_is_due(&settings, 0)
};
let evidence_block_count = if evidence_path.exists() {
count_evidence_blocks(&evidence_path).unwrap_or(0)
} else {
0
};
Ok(json!({
"schema": "mnote.local_index.status.v1",
"rootUri": root_uri,
"workspaceId": workspace_id,
"settings": settings,
"indexPath": ".mnote/index/search-index.json",
"evidenceIndexPath": ".mnote/index/evidence.sqlite",
"settingsPath": ".mnote/index/local-index-settings.json",
"indexExists": index_path.exists(),
"evidenceIndexExists": evidence_path.exists(),
"cacheMatchesSettings": cache_matches_settings,
"scheduledDue": scheduled_due,
"builtAt": built_at,
"indexedPaths": index
.as_ref()
.map(|index| index.indexed_paths.clone())
.unwrap_or_default(),
"documentCount": document_count,
"resourceCount": resource_count,
"evidenceBlockCount": evidence_block_count,
}))
}
pub(crate) fn local_index_status_with_settings(
root_path: &Path,
root_uri: &str,
workspace_id: &str,
user_settings: &LocalIndexSettings,
effective_settings: &LocalIndexSettings,
) -> Result<Value, WebError> {
let mut status =
local_index_status_for_settings(root_path, root_uri, workspace_id, effective_settings)?;
if let Some(object) = status.as_object_mut() {
object.insert("settings".to_string(), json!(user_settings));
object.insert("effectiveSettings".to_string(), json!(effective_settings));
}
Ok(status)
}
#[cfg(test)]
#[allow(dead_code)]
pub(crate) fn local_index_should_refresh_on_change(root_path: &Path) -> bool {
read_local_index_settings_or_default(root_path)
.map(|settings| settings.run_on_change)
.unwrap_or(false)
}
#[cfg(test)]
#[allow(dead_code)]
pub(crate) fn refresh_local_search_index_for_change(
root_path: &Path,
root_uri: &str,
workspace_id: &str,
) -> Result<Option<Value>, WebError> {
if !local_index_should_refresh_on_change(root_path) {
return Ok(None);
}
refresh_local_search_index(root_path, root_uri, workspace_id).map(Some)
}
pub(crate) fn refresh_local_search_index_for_change_with_store(
store: &dyn ControlPlaneStore,
root_path: &Path,
root_uri: &str,
workspace_id: &str,
) -> Result<Option<Value>, WebError> {
let effective_settings =
effective_local_index_settings_for_root(store, workspace_id, root_path)?;
if !effective_settings.run_on_change {
return Ok(None);
}
refresh_local_search_index_with_settings(root_path, root_uri, workspace_id, &effective_settings)
.map(Some)
}
#[cfg(test)]
#[allow(dead_code)]
pub(crate) fn refresh_local_search_index_for_change_path(
root_path: &Path,
root_uri: &str,
workspace_id: &str,
relative_path: &str,
) -> Result<Option<Value>, WebError> {
if !local_index_should_refresh_on_change(root_path) {
return Ok(None);
}
refresh_local_search_index_for_path(root_path, root_uri, workspace_id, relative_path).map(Some)
}
#[cfg(test)]
#[allow(dead_code)]
pub(crate) fn refresh_local_search_index_if_scheduled_due(
root_path: &Path,
root_uri: &str,
workspace_id: &str,
) -> Result<Option<Value>, WebError> {
let settings = read_local_index_settings_or_default(root_path)?;
let Some(index) = read_local_search_index(root_path)? else {
if local_index_schedule_is_due(&settings, 0) {
return refresh_local_search_index(root_path, root_uri, workspace_id).map(Some);
}
return Ok(None);
};
if !local_index_schedule_is_due(&settings, index.built_at) {
return Ok(None);
}
refresh_local_search_index(root_path, root_uri, workspace_id).map(Some)
}
pub(crate) fn refresh_local_search_index_for_change_path_with_store(
store: &dyn ControlPlaneStore,
root_path: &Path,
root_uri: &str,
workspace_id: &str,
relative_path: &str,
) -> Result<Option<Value>, WebError> {
let effective_settings =
effective_local_index_settings_for_root(store, workspace_id, root_path)?;
if !effective_settings.run_on_change {
return Ok(None);
}
refresh_local_search_index_for_path_with_settings(
root_path,
root_uri,
workspace_id,
&effective_settings,
relative_path,
)
.map(Some)
}
pub(crate) fn refresh_local_search_index_if_scheduled_due_with_store(
store: &dyn ControlPlaneStore,
root_path: &Path,
root_uri: &str,
workspace_id: &str,
) -> Result<Option<Value>, WebError> {
let effective_settings =
effective_local_index_settings_for_root(store, workspace_id, root_path)?;
let built_at_ms = current_local_search_index_built_at(root_path)?.unwrap_or(0);
if !local_index_any_schedule_due_for_root(store, workspace_id, root_path, built_at_ms)? {
return Ok(None);
}
refresh_local_search_index_with_settings(root_path, root_uri, workspace_id, &effective_settings)
.map(Some)
}
pub(crate) fn query_evidence_sqlite_results(
root_path: &Path,
query: &str,
owner_document_id: Option<&str>,
limit: u32,
) -> Result<Option<Vec<EvidenceSearchResult>>, WebError> {
let path = evidence_sqlite_path(root_path);
if !path.exists() {
return Ok(None);
}
let normalized_query = query.trim();
if normalized_query.is_empty() {
return Ok(Some(Vec::new()));
}
let connection = Connection::open(&path).map_err(|error| {
WebError::bad_request_code(
"evidence_index_open_failed",
format!("无法打开 evidence 索引 {}: {error}", path.display()),
)
})?;
let fts_query = evidence_fts_phrase(normalized_query);
let results = match query_evidence_sqlite_fts(
&connection,
&fts_query,
normalized_query,
owner_document_id,
limit,
) {
Ok(results) => results,
Err(_) => {
query_evidence_sqlite_like(&connection, normalized_query, owner_document_id, limit)?
}
};
Ok(Some(results))
}
pub(crate) fn read_evidence_sqlite_context(
root_path: &Path,
locator: &EvidenceLocator,
before_blocks: u32,
after_blocks: u32,
) -> Result<Option<Vec<EvidenceSearchResult>>, WebError> {
let path = evidence_sqlite_path(root_path);
if !path.exists() {
return Ok(None);
}
let connection = Connection::open(&path).map_err(|error| {
WebError::bad_request_code(
"evidence_index_open_failed",
format!("无法打开 evidence 索引 {}: {error}", path.display()),
)
})?;
let mut statement = connection
.prepare(
"SELECT b.block_id, b.text, b.locator_json \
FROM evidence_block b \
JOIN evidence_resource r ON r.resource_id = b.resource_id \
WHERE r.owner_document_id = ?1 \
ORDER BY b.rowid",
)
.map_err(sqlite_error)?;
let rows = statement
.query_map(params![locator.owner_document_id], |row| {
let block_id: String = row.get(0)?;
let text: String = row.get(1)?;
let locator_json: String = row.get(2)?;
let source =
serde_json::from_str::<EvidenceLocator>(&locator_json).map_err(|error| {
rusqlite::Error::FromSqlConversionFailure(
2,
rusqlite::types::Type::Text,
Box::new(error),
)
})?;
Ok((block_id, text, source))
})
.map_err(sqlite_error)?
.collect::<Result<Vec<_>, _>>()
.map_err(sqlite_error)?;
if rows.is_empty() {
return Ok(Some(Vec::new()));
}
let Some(anchor_index) = rows
.iter()
.position(|(block_id, _, source)| evidence_locator_matches(block_id, source, locator))
else {
return Ok(Some(Vec::new()));
};
let start = anchor_index.saturating_sub(before_blocks as usize);
let end = (anchor_index + after_blocks as usize + 1).min(rows.len());
let results = rows[start..end]
.iter()
.map(|(block_id, text, source)| EvidenceSearchResult {
evidence_id: block_id.clone(),
quote: text.clone(),
score: if evidence_locator_matches(block_id, source, locator) {
1.0
} else {
0.8
},
source: source.clone(),
})
.collect::<Vec<_>>();
Ok(Some(results))
}
pub(crate) fn query_evidence_graph_results(
root_path: &Path,
query: &str,
owner_document_id: Option<&str>,
limit: u32,
) -> Result<Option<Vec<EvidenceSearchResult>>, WebError> {
let path = evidence_sqlite_path(root_path);
if !path.exists() {
return Ok(None);
}
let normalized_query = query.trim().to_ascii_lowercase();
if normalized_query.is_empty() {
return Ok(Some(Vec::new()));
}
let connection = Connection::open(&path).map_err(|error| {
WebError::bad_request_code(
"evidence_index_open_failed",
format!("无法打开 evidence 索引 {}: {error}", path.display()),
)
})?;
let like_query = format!("%{}%", normalized_query.replace('%', "\\%"));
let mut statement = if owner_document_id.is_some() {
connection.prepare(
"SELECT e.edge_id, e.edge_type, e.from_id, e.to_id, e.source_block_id, b.text, b.locator_json \
FROM evidence_edge e \
JOIN evidence_block b ON b.block_id = e.source_block_id \
JOIN evidence_resource r ON r.resource_id = b.resource_id \
WHERE r.owner_document_id = ?2 \
AND (lower(e.edge_type) LIKE ?1 OR lower(e.from_id) LIKE ?1 OR lower(e.to_id) LIKE ?1) \
ORDER BY e.edge_type, e.from_id, e.to_id \
LIMIT ?3",
)
} else {
connection.prepare(
"SELECT e.edge_id, e.edge_type, e.from_id, e.to_id, e.source_block_id, b.text, b.locator_json \
FROM evidence_edge e \
JOIN evidence_block b ON b.block_id = e.source_block_id \
WHERE lower(e.edge_type) LIKE ?1 OR lower(e.from_id) LIKE ?1 OR lower(e.to_id) LIKE ?1 \
ORDER BY e.edge_type, e.from_id, e.to_id \
LIMIT ?2",
)
}
.map_err(sqlite_error)?;
let results = if let Some(owner_document_id) = owner_document_id {
statement
.query_map(
params![like_query, owner_document_id, limit],
graph_edge_row,
)
.map_err(sqlite_error)?
.collect::<Result<Vec<_>, _>>()
.map_err(sqlite_error)?
} else {
statement
.query_map(params![like_query, limit], graph_edge_row)
.map_err(sqlite_error)?
.collect::<Result<Vec<_>, _>>()
.map_err(sqlite_error)?
};
Ok(Some(results))
}
fn graph_edge_row(row: &rusqlite::Row<'_>) -> rusqlite::Result<EvidenceSearchResult> {
let edge_id: String = row.get(0)?;
let edge_type: String = row.get(1)?;
let from_id: String = row.get(2)?;
let to_id: String = row.get(3)?;
let source_block_id: String = row.get(4)?;
let text: String = row.get(5)?;
let locator_json: String = row.get(6)?;
let source = serde_json::from_str::<EvidenceLocator>(&locator_json).map_err(|error| {
rusqlite::Error::FromSqlConversionFailure(6, rusqlite::types::Type::Text, Box::new(error))
})?;
Ok(EvidenceSearchResult {
evidence_id: edge_id,
quote: format!("{edge_type}: {from_id} -> {to_id}\n{text}"),
score: 1.0,
source: EvidenceLocator {
block_id: source.block_id.or(Some(source_block_id)),
..source
},
})
}
fn evidence_locator_matches(
block_id: &str,
source: &EvidenceLocator,
locator: &EvidenceLocator,
) -> bool {
locator.block_id.as_deref().is_some_and(|locator_block_id| {
locator_block_id == block_id || source.block_id.as_deref() == Some(locator_block_id)
}) || source.owner_document_id == locator.owner_document_id
&& source.resource_path == locator.resource_path
&& source.source_map_path == locator.source_map_path
}
fn query_evidence_sqlite_fts(
connection: &Connection,
fts_query: &str,
display_query: &str,
owner_document_id: Option<&str>,
limit: u32,
) -> Result<Vec<EvidenceSearchResult>, WebError> {
let mut sql = String::from(
"SELECT b.block_id, b.text, b.locator_json, bm25(evidence_fts) AS rank \
FROM evidence_fts \
JOIN evidence_block b ON b.block_id = evidence_fts.block_id \
JOIN evidence_resource r ON r.resource_id = b.resource_id \
WHERE evidence_fts MATCH ?1",
);
if owner_document_id.is_some() {
sql.push_str(" AND r.owner_document_id = ?2");
}
sql.push_str(if owner_document_id.is_some() {
" ORDER BY rank LIMIT ?3"
} else {
" ORDER BY rank LIMIT ?2"
});
let limit = i64::from(limit.max(1));
let mut statement = connection.prepare(&sql).map_err(sqlite_error)?;
let rows = if let Some(owner_document_id) = owner_document_id {
statement
.query_map(params![fts_query, owner_document_id, limit], |row| {
evidence_result_from_sqlite_row(row, display_query)
})
.map_err(sqlite_error)?
.collect::<Result<Vec<_>, _>>()
} else {
statement
.query_map(params![fts_query, limit], |row| {
evidence_result_from_sqlite_row(row, display_query)
})
.map_err(sqlite_error)?
.collect::<Result<Vec<_>, _>>()
};
rows.map_err(sqlite_error)
}
fn query_evidence_sqlite_like(
connection: &Connection,
query: &str,
owner_document_id: Option<&str>,
limit: u32,
) -> Result<Vec<EvidenceSearchResult>, WebError> {
let mut sql = String::from(
"SELECT b.block_id, b.text, b.locator_json, 0.0 AS rank \
FROM evidence_block b \
JOIN evidence_resource r ON r.resource_id = b.resource_id \
WHERE b.text LIKE ?1 ESCAPE '\\'",
);
if owner_document_id.is_some() {
sql.push_str(" AND r.owner_document_id = ?2");
}
sql.push_str(if owner_document_id.is_some() {
" LIMIT ?3"
} else {
" LIMIT ?2"
});
let like_query = format!("%{}%", query.replace('%', "\\%").replace('_', "\\_"));
let limit = i64::from(limit.max(1));
let mut statement = connection.prepare(&sql).map_err(sqlite_error)?;
let rows = if let Some(owner_document_id) = owner_document_id {
statement
.query_map(params![like_query, owner_document_id, limit], |row| {
evidence_result_from_sqlite_row(row, query)
})
.map_err(sqlite_error)?
.collect::<Result<Vec<_>, _>>()
} else {
statement
.query_map(params![like_query, limit], |row| {
evidence_result_from_sqlite_row(row, query)
})
.map_err(sqlite_error)?
.collect::<Result<Vec<_>, _>>()
};
rows.map_err(sqlite_error)
}
fn evidence_result_from_sqlite_row(
row: &rusqlite::Row<'_>,
query: &str,
) -> rusqlite::Result<EvidenceSearchResult> {
let block_id: String = row.get(0)?;
let text: String = row.get(1)?;
let locator_json: String = row.get(2)?;
let rank: f64 = row.get(3)?;
let source = serde_json::from_str::<EvidenceLocator>(&locator_json).map_err(|error| {
rusqlite::Error::FromSqlConversionFailure(2, rusqlite::types::Type::Text, Box::new(error))
})?;
Ok(EvidenceSearchResult {
evidence_id: block_id,
quote: ocr_search_snippet(&text, query),
score: if rank == 0.0 {
1.0
} else {
1.0 / (1.0 + rank.abs())
},
source,
})
}
fn evidence_fts_phrase(query: &str) -> String {
format!("\"{}\"", query.replace('"', "\"\""))
}
#[cfg(test)]
pub(crate) fn refresh_local_search_index(
root_path: &Path,
root_uri: &str,
workspace_id: &str,
) -> Result<Value, WebError> {
let settings = read_local_index_settings_or_default(root_path)?;
refresh_local_search_index_with_settings(root_path, root_uri, workspace_id, &settings)
}
pub(crate) fn refresh_local_search_index_with_settings(
root_path: &Path,
root_uri: &str,
workspace_id: &str,
settings: &LocalIndexSettings,
) -> Result<Value, WebError> {
let index =
rebuild_local_search_index_with_settings(root_path, root_uri, workspace_id, settings)?;
Ok(json!({
"version": index.version,
"rootUri": index.root_uri,
"workspaceId": index.workspace_id,
"indexedPaths": index.indexed_paths,
"builtAt": index.built_at,
"documentCount": index.documents.len(),
"resourceCount": index.resources.len()
}))
}
#[cfg(test)]
pub(crate) fn refresh_local_search_index_for_path(
root_path: &Path,
root_uri: &str,
workspace_id: &str,
relative_path: &str,
) -> Result<Value, WebError> {
let settings = read_local_index_settings_or_default(root_path)?;
refresh_local_search_index_for_path_with_settings(
root_path,
root_uri,
workspace_id,
&settings,
relative_path,
)
}
pub(crate) fn refresh_local_search_index_for_path_with_settings(
root_path: &Path,
root_uri: &str,
workspace_id: &str,
settings: &LocalIndexSettings,
relative_path: &str,
) -> Result<Value, WebError> {
let relative_path = normalize_index_relative_path(relative_path)?;
let mut index = match read_local_search_index(root_path) {
Ok(Some(index))
if index.version == LOCAL_SEARCH_INDEX_VERSION
&& index.root_uri == root_uri
&& index.workspace_id == workspace_id
&& normalized_indexed_paths(&index.indexed_paths) == settings.include_paths =>
{
index
}
Ok(_) | Err(_) => {
return refresh_local_search_index_with_settings(
root_path,
root_uri,
workspace_id,
settings,
);
}
};
index
.documents
.retain(|document| document.path != relative_path);
index
.resources
.retain(|resource| resource.path != relative_path);
let included = index_relative_path_is_included(&relative_path, &settings.include_paths);
2026-06-01 10:07:42 +08:00
if local_ocr::is_local_ocr_sidecar_relative_path(&relative_path) {
index.built_at = now_ms();
write_local_search_index_json(root_path, &index)?;
if included {
refresh_evidence_sqlite_index_for_path(root_path, &index, &relative_path)?;
} else {
remove_evidence_sqlite_index_for_path(root_path, &index, &relative_path)?;
}
2026-06-01 10:07:42 +08:00
return Ok(json!({
"version": index.version,
"rootUri": index.root_uri,
"workspaceId": index.workspace_id,
"indexedPaths": index.indexed_paths,
"builtAt": index.built_at,
"documentCount": index.documents.len(),
"resourceCount": index.resources.len()
}));
}
if is_local_evidence_sidecar_relative_path(&relative_path) {
index.built_at = now_ms();
write_local_search_index_json(root_path, &index)?;
return Ok(json!({
"version": index.version,
"rootUri": index.root_uri,
"workspaceId": index.workspace_id,
"indexedPaths": index.indexed_paths,
2026-06-01 10:07:42 +08:00
"builtAt": index.built_at,
"documentCount": index.documents.len(),
"resourceCount": index.resources.len()
}));
}
let absolute_path = root_path.join(&relative_path);
if included
&& absolute_path.exists()
&& absolute_path.is_file()
&& is_markdown_path(&absolute_path)
{
index
.documents
.push(index_markdown_file(root_path, &absolute_path)?);
} else if included
&& absolute_path.exists()
&& absolute_path.is_file()
&& resource_type_from_path(&absolute_path).is_some()
{
index
.resources
.push(index_resource_file(root_path, &absolute_path)?);
}
index
.documents
.sort_by(|left, right| left.path.cmp(&right.path));
index
.resources
.sort_by(|left, right| left.path.cmp(&right.path));
index.built_at = now_ms();
write_local_search_index_json(root_path, &index)?;
if included {
refresh_evidence_sqlite_index_for_path(root_path, &index, &relative_path)?;
} else {
remove_evidence_sqlite_index_for_path(root_path, &index, &relative_path)?;
}
Ok(json!({
"version": index.version,
"rootUri": index.root_uri,
"workspaceId": index.workspace_id,
"indexedPaths": index.indexed_paths,
"builtAt": index.built_at,
"documentCount": index.documents.len(),
"resourceCount": index.resources.len()
}))
}
#[cfg(test)]
#[allow(dead_code)]
pub(crate) fn query_local_backlinks(
root_path: &Path,
root_uri: &str,
workspace_id: &str,
document_id: &str,
) -> Result<Value, WebError> {
let settings = read_local_index_settings_or_default(root_path)?;
query_local_backlinks_with_settings(
root_path,
root_uri,
workspace_id,
&settings,
&settings,
document_id,
)
}
pub(crate) fn query_local_backlinks_with_settings(
root_path: &Path,
root_uri: &str,
workspace_id: &str,
cache_settings: &LocalIndexSettings,
result_settings: &LocalIndexSettings,
document_id: &str,
) -> Result<Value, WebError> {
let index = load_or_rebuild_local_search_index_with_settings(
root_path,
root_uri,
workspace_id,
cache_settings,
)?;
let Some(target) = index.documents.iter().find(|document| {
document.document_id == document_id
&& index_relative_path_is_included(&document.path, &result_settings.include_paths)
}) else {
return Ok(json!({
"documentId": document_id,
"rootUri": root_uri,
"workspaceId": workspace_id,
"backlinks": []
}));
};
let target_keys = normalized_document_targets(target);
let backlinks = index
.documents
.iter()
.filter(|document| {
index_relative_path_is_included(&document.path, &result_settings.include_paths)
})
.filter(|document| document.document_id != target.document_id)
.filter_map(|document| {
let matched_targets = document
.backlinks
.iter()
.filter(|link| target_keys.contains(&normalize_search_text(link)))
.cloned()
.collect::<Vec<_>>();
if matched_targets.is_empty() {
return None;
}
Some(json!({
"documentId": document.document_id,
"title": document.title,
"path": document.path,
"resourceType": "markdown",
"matchedTargets": matched_targets,
"snippet": search_snippet(document, &normalize_search_text(&target.title))
}))
})
.collect::<Vec<_>>();
Ok(json!({
"documentId": target.document_id,
"title": target.title,
"path": target.path,
"rootUri": root_uri,
"workspaceId": workspace_id,
"backlinks": backlinks
}))
}
#[cfg(test)]
#[allow(dead_code)]
pub(crate) fn query_local_tags(
root_path: &Path,
root_uri: &str,
workspace_id: &str,
) -> Result<Value, WebError> {
let settings = read_local_index_settings_or_default(root_path)?;
query_local_tags_with_settings(root_path, root_uri, workspace_id, &settings, &settings)
}
pub(crate) fn query_local_tags_with_settings(
root_path: &Path,
root_uri: &str,
workspace_id: &str,
cache_settings: &LocalIndexSettings,
result_settings: &LocalIndexSettings,
) -> Result<Value, WebError> {
let index = load_or_rebuild_local_search_index_with_settings(
root_path,
root_uri,
workspace_id,
cache_settings,
)?;
let mut tags = std::collections::BTreeMap::<String, Vec<&LocalSearchDocument>>::new();
for document in &index.documents {
if !index_relative_path_is_included(&document.path, &result_settings.include_paths) {
continue;
}
for tag in &document.tags {
tags.entry(tag.clone()).or_default().push(document);
}
}
Ok(json!({
"rootUri": root_uri,
"workspaceId": workspace_id,
"tags": tags
.into_iter()
.map(|(tag, documents)| {
json!({
"tag": tag,
"count": documents.len(),
"documents": documents
.into_iter()
.map(|document| {
json!({
"documentId": document.document_id,
"title": document.title,
"path": document.path,
"resourceType": "markdown"
})
})
.collect::<Vec<_>>()
})
})
.collect::<Vec<_>>()
}))
}
#[cfg(test)]
#[allow(dead_code)]
fn rebuild_local_search_index(
root_path: &Path,
root_uri: &str,
workspace_id: &str,
) -> Result<LocalSearchIndex, WebError> {
let settings = read_local_index_settings_or_default(root_path)?;
rebuild_local_search_index_with_settings(root_path, root_uri, workspace_id, &settings)
}
fn rebuild_local_search_index_with_settings(
root_path: &Path,
root_uri: &str,
workspace_id: &str,
settings: &LocalIndexSettings,
) -> Result<LocalSearchIndex, WebError> {
let mut documents = Vec::new();
let mut resources = Vec::new();
for include_path in &settings.include_paths {
let base_path = if include_path == "." {
root_path.to_path_buf()
} else {
root_path.join(include_path)
};
if base_path.exists() {
collect_markdown_documents(root_path, &base_path, &mut documents, &mut resources)?;
}
}
documents.sort_by(|left, right| left.path.cmp(&right.path));
documents.dedup_by(|left, right| left.path == right.path);
resources.sort_by(|left, right| left.path.cmp(&right.path));
resources.dedup_by(|left, right| left.path == right.path);
let index = LocalSearchIndex {
version: LOCAL_SEARCH_INDEX_VERSION,
built_at: now_ms(),
root_uri: root_uri.to_string(),
workspace_id: workspace_id.to_string(),
indexed_paths: settings.include_paths.clone(),
documents,
resources,
};
write_local_search_index(root_path, &index)?;
Ok(index)
}
#[cfg(test)]
#[allow(dead_code)]
fn load_or_rebuild_local_search_index(
root_path: &Path,
root_uri: &str,
workspace_id: &str,
) -> Result<LocalSearchIndex, WebError> {
let settings = read_local_index_settings_or_default(root_path)?;
load_or_rebuild_local_search_index_with_settings(root_path, root_uri, workspace_id, &settings)
}
fn load_or_rebuild_local_search_index_with_settings(
root_path: &Path,
root_uri: &str,
workspace_id: &str,
settings: &LocalIndexSettings,
) -> Result<LocalSearchIndex, WebError> {
match read_local_search_index(root_path) {
Ok(Some(index))
if index.version == LOCAL_SEARCH_INDEX_VERSION
&& index.root_uri == root_uri
&& index.workspace_id == workspace_id
&& normalized_indexed_paths(&index.indexed_paths) == settings.include_paths =>
{
if local_index_schedule_is_due(&settings, index.built_at) {
return rebuild_local_search_index_with_settings(
root_path,
root_uri,
workspace_id,
settings,
);
}
ensure_evidence_sqlite_index(root_path, &index)?;
Ok(index)
}
Ok(_) | Err(_) => {
rebuild_local_search_index_with_settings(root_path, root_uri, workspace_id, settings)
}
}
}
fn read_local_search_index(root_path: &Path) -> Result<Option<LocalSearchIndex>, WebError> {
let index_path = root_path
.join(".mnote")
.join("index")
.join("search-index.json");
if !index_path.exists() {
return Ok(None);
}
let content = fs::read_to_string(&index_path).map_err(|error| {
WebError::bad_request_code(
"local_search_index_read_failed",
format!("无法读取本地搜索索引 {}: {error}", index_path.display()),
)
})?;
serde_json::from_str::<LocalSearchIndex>(&content)
.map(Some)
.map_err(|error| {
WebError::bad_request_code(
"local_search_index_invalid",
format!("本地搜索索引格式非法 {}: {error}", index_path.display()),
)
})
}
pub(crate) fn current_local_search_index_built_at(
root_path: &Path,
) -> Result<Option<u128>, WebError> {
Ok(read_local_search_index(root_path)?.map(|index| index.built_at))
}
fn default_indexed_paths() -> Vec<String> {
vec![".".to_string()]
}
fn local_index_settings_path(root_path: &Path) -> PathBuf {
root_path
.join(".mnote")
.join("index")
.join("local-index-settings.json")
}
pub(crate) fn read_local_index_settings_or_default(
root_path: &Path,
) -> Result<LocalIndexSettings, WebError> {
let path = local_index_settings_path(root_path);
if !path.exists() {
return Ok(default_local_index_settings());
}
let content = fs::read_to_string(&path).map_err(|error| {
WebError::bad_request_code(
"local_index_settings_read_failed",
format!("无法读取本地索引设置 {}: {error}", path.display()),
)
})?;
parse_local_index_settings_value(root_path, &content).map_err(|error| {
if error.code() == "local_index_settings_invalid" {
WebError::bad_request_code(
"local_index_settings_invalid",
format!(
"本地索引设置格式非法 {}: {}",
path.display(),
error.message()
),
)
} else {
error
}
})
}
fn write_local_index_settings_cache(
root_path: &Path,
settings: &LocalIndexSettings,
) -> Result<(), WebError> {
let path = local_index_settings_path(root_path);
if let Some(parent) = path.parent() {
fs::create_dir_all(parent).map_err(|error| {
WebError::bad_request_code(
"local_index_settings_create_failed",
format!("无法创建本地索引设置目录 {}: {error}", parent.display()),
)
})?;
}
let payload = serde_json::to_string_pretty(settings)
.map_err(|error| WebError::internal(format!("本地索引设置序列化失败: {error}")))?;
fs::write(&path, format!("{payload}\n")).map_err(|error| {
WebError::bad_request_code(
"local_index_settings_write_failed",
format!("无法写入本地索引设置 {}: {error}", path.display()),
)
})
}
fn parse_local_index_settings_value(
root_path: &Path,
value_json: &str,
) -> Result<LocalIndexSettings, WebError> {
let raw = serde_json::from_str::<LocalIndexSettings>(value_json).map_err(|error| {
WebError::bad_request_code(
"local_index_settings_invalid",
format!("本地索引设置 JSON 非法: {error}"),
)
})?;
Ok(LocalIndexSettings {
schema: LOCAL_INDEX_SETTINGS_SCHEMA.to_string(),
include_paths: normalize_index_include_paths(root_path, &raw.include_paths)?,
schedule_mode: normalize_index_schedule_mode(&raw.schedule_mode)?,
schedule_time: normalize_index_schedule_time(&raw.schedule_time)?,
schedule_date: normalize_index_schedule_date(raw.schedule_date.as_deref())?,
run_on_change: raw.run_on_change,
updated_at: raw.updated_at,
})
}
fn default_local_index_settings() -> LocalIndexSettings {
LocalIndexSettings {
schema: LOCAL_INDEX_SETTINGS_SCHEMA.to_string(),
include_paths: default_indexed_paths(),
schedule_mode: default_index_schedule_mode(),
schedule_time: default_index_schedule_time(),
schedule_date: None,
run_on_change: default_index_run_on_change(),
updated_at: 0,
}
}
fn default_index_schedule_mode() -> String {
DEFAULT_INDEX_SCHEDULE_MODE.to_string()
}
fn default_index_schedule_time() -> String {
DEFAULT_INDEX_SCHEDULE_TIME.to_string()
}
fn default_index_run_on_change() -> bool {
false
}
fn normalize_index_schedule_mode(value: &str) -> Result<String, WebError> {
let mode = value.trim();
if mode.is_empty() {
return Ok(default_index_schedule_mode());
}
match mode {
"daily" | "once" | "manual" => Ok(mode.to_string()),
_ => Err(WebError::bad_request_code(
"local_index_schedule_mode_invalid",
"本地索引调度模式只能是 daily、once 或 manual",
)),
}
}
fn normalize_index_schedule_time(value: &str) -> Result<String, WebError> {
let value = value.trim();
let (hour, minute) = parse_hh_mm(value)?;
Ok(format!("{hour:02}:{minute:02}"))
}
fn normalize_index_schedule_date(value: Option<&str>) -> Result<Option<String>, WebError> {
let Some(value) = value.map(str::trim).filter(|value| !value.is_empty()) else {
return Ok(None);
};
let _ = parse_yyyy_mm_dd(value)?;
Ok(Some(value.to_string()))
}
fn local_index_schedule_is_due(settings: &LocalIndexSettings, built_at_ms: u128) -> bool {
let now = OffsetDateTime::now_local().unwrap_or_else(|_| OffsetDateTime::now_utc());
let Ok((hour, minute)) = parse_hh_mm(&settings.schedule_time) else {
return false;
};
let Ok(schedule_time) = Time::from_hms(hour, minute, 0) else {
return false;
};
let target = match settings.schedule_mode.as_str() {
"manual" => return false,
"once" => {
let Some(date) = settings
.schedule_date
.as_deref()
.and_then(|value| parse_yyyy_mm_dd(value).ok())
else {
return false;
};
date.with_time(schedule_time).assume_offset(now.offset())
}
_ => now
.date()
.with_time(schedule_time)
.assume_offset(now.offset()),
};
if now < target {
return false;
}
let built_at = unix_ms_to_local_datetime(built_at_ms);
built_at.map(|value| value < target).unwrap_or(true)
}
fn unix_ms_to_local_datetime(value: u128) -> Option<OffsetDateTime> {
let seconds = i64::try_from(value / 1000).ok()?;
let nanos = u32::try_from((value % 1000) * 1_000_000).ok()?;
OffsetDateTime::from_unix_timestamp(seconds)
.ok()
.map(|datetime| datetime + Duration::nanoseconds(nanos as i64))
.map(|datetime| {
datetime.to_offset(
OffsetDateTime::now_local()
.unwrap_or_else(|_| OffsetDateTime::now_utc())
.offset(),
)
})
}
fn parse_hh_mm(value: &str) -> Result<(u8, u8), WebError> {
let parts = value.trim().split(':').collect::<Vec<_>>();
if parts.len() != 2 {
return Err(WebError::bad_request_code(
"local_index_schedule_time_invalid",
"本地索引时间必须是 HH:MM",
));
}
let hour = parts[0].parse::<u8>().map_err(|_| {
WebError::bad_request_code(
"local_index_schedule_time_invalid",
"本地索引时间必须是 HH:MM",
)
})?;
let minute = parts[1].parse::<u8>().map_err(|_| {
WebError::bad_request_code(
"local_index_schedule_time_invalid",
"本地索引时间必须是 HH:MM",
)
})?;
if hour > 23 || minute > 59 {
return Err(WebError::bad_request_code(
"local_index_schedule_time_invalid",
"本地索引时间必须是 00:00 到 23:59",
));
}
Ok((hour, minute))
}
fn parse_yyyy_mm_dd(value: &str) -> Result<Date, WebError> {
let parts = value.trim().split('-').collect::<Vec<_>>();
if parts.len() != 3 {
return Err(WebError::bad_request_code(
"local_index_schedule_date_invalid",
"本地索引日期必须是 YYYY-MM-DD",
));
}
let year = parts[0].parse::<i32>().map_err(|_| {
WebError::bad_request_code(
"local_index_schedule_date_invalid",
"本地索引日期必须是 YYYY-MM-DD",
)
})?;
let month = parts[1].parse::<u8>().map_err(|_| {
WebError::bad_request_code(
"local_index_schedule_date_invalid",
"本地索引日期必须是 YYYY-MM-DD",
)
})?;
let day = parts[2].parse::<u8>().map_err(|_| {
WebError::bad_request_code(
"local_index_schedule_date_invalid",
"本地索引日期必须是 YYYY-MM-DD",
)
})?;
let month = Month::try_from(month).map_err(|_| {
WebError::bad_request_code(
"local_index_schedule_date_invalid",
"本地索引日期必须是 YYYY-MM-DD",
)
})?;
Date::from_calendar_date(year, month, day).map_err(|_| {
WebError::bad_request_code(
"local_index_schedule_date_invalid",
"本地索引日期必须是 YYYY-MM-DD",
)
})
}
fn normalize_index_include_paths(
root_path: &Path,
include_paths: &[String],
) -> Result<Vec<String>, WebError> {
let mut output = Vec::new();
let root_canonical = root_path
.canonicalize()
.unwrap_or_else(|_| root_path.to_path_buf());
for value in include_paths {
let trimmed = value.trim();
if trimmed.is_empty() {
continue;
}
let normalized = if trimmed == "." || trimmed == "/" {
".".to_string()
} else {
normalize_index_relative_path(trimmed.trim_start_matches("./"))?
};
if normalized.split('/').any(|part| part == ".mnote") {
return Err(WebError::bad_request_code(
"local_index_scope_reserved",
"本地索引目录不能指向 .mnote 内部缓存",
));
}
let absolute = if normalized == "." {
root_path.to_path_buf()
} else {
root_path.join(&normalized)
};
if let Ok(canonical) = absolute.canonicalize() {
if !canonical.starts_with(&root_canonical) {
return Err(WebError::bad_request_code(
"local_index_scope_escape",
"本地索引目录必须位于当前授权 root 内",
));
}
if !canonical.is_dir() {
return Err(WebError::bad_request_code(
"local_index_scope_not_directory",
"本地索引范围必须是目录",
));
}
}
if output.iter().any(|existing| existing == &normalized) {
continue;
}
output.push(normalized);
}
if output.is_empty() {
output.push(".".to_string());
}
Ok(normalized_indexed_paths(&output))
}
fn normalized_indexed_paths(paths: &[String]) -> Vec<String> {
let mut values = paths
.iter()
.map(|value| value.trim().replace('\\', "/"))
.filter(|value| !value.is_empty())
.map(|value| {
if value == "/" || value == "./" {
".".to_string()
} else {
value
.trim_start_matches("./")
.trim_end_matches('/')
.to_string()
}
})
.map(|value| {
if value.is_empty() {
".".to_string()
} else {
value
}
})
.collect::<Vec<_>>();
values.sort();
values.dedup();
values
}
fn normalize_index_relative_path(relative_path: &str) -> Result<String, WebError> {
let normalized = relative_path.trim().replace('\\', "/");
if normalized.is_empty() {
return Err(WebError::bad_request_code(
"local_search_index_path_required",
"本地搜索索引更新缺少相对路径",
));
}
let path = PathBuf::from(&normalized);
if path.is_absolute()
|| path.components().any(|component| {
matches!(
component,
Component::ParentDir | Component::RootDir | Component::Prefix(_)
)
})
{
return Err(WebError::bad_request_code(
"local_search_index_path_escape",
"本地搜索索引相对路径不能越过 root",
));
}
Ok(normalized)
}
fn index_relative_path_is_included(relative_path: &str, include_paths: &[String]) -> bool {
let relative_path = relative_path.trim().replace('\\', "/");
include_paths.iter().any(|include| {
include == "."
|| relative_path == *include
|| relative_path
.strip_prefix(include)
.map(|rest| rest.starts_with('/'))
.unwrap_or(false)
})
}
fn collect_markdown_documents(
root_path: &Path,
current: &Path,
documents: &mut Vec<LocalSearchDocument>,
resources: &mut Vec<LocalSearchResource>,
) -> Result<(), WebError> {
let entries = match fs::read_dir(current) {
Ok(entries) => entries,
Err(error) => {
return Err(WebError::bad_request_code(
"local_search_index_read_dir_failed",
format!("无法读取本地搜索索引目录 {}: {error}", current.display()),
));
}
};
for entry in entries {
let entry = entry.map_err(|error| {
WebError::bad_request_code(
"local_search_index_read_entry_failed",
format!("无法读取本地搜索索引条目: {error}"),
)
})?;
let path = entry.path();
let file_name = path
.file_name()
.and_then(|value| value.to_str())
.unwrap_or_default();
if file_name == ".mnote" {
continue;
}
let file_type = entry.file_type().map_err(|error| {
WebError::bad_request_code(
"local_search_index_stat_failed",
format!("无法读取本地搜索索引文件状态 {}: {error}", path.display()),
)
})?;
if file_type.is_dir() {
collect_markdown_documents(root_path, &path, documents, resources)?;
continue;
}
if !file_type.is_file() {
continue;
}
let relative_path = path
.strip_prefix(root_path)
.unwrap_or(&path)
.to_string_lossy()
.replace('\\', "/");
if is_local_evidence_sidecar_relative_path(&relative_path) {
continue;
}
if is_markdown_path(&path) {
2026-06-01 09:29:12 +08:00
if local_ocr::is_local_ocr_sidecar_relative_path(&relative_path) {
continue;
}
documents.push(index_markdown_file(root_path, &path)?);
} else if resource_type_from_path(&path).is_some() {
resources.push(index_resource_file(root_path, &path)?);
}
}
Ok(())
}
fn index_markdown_file(root_path: &Path, path: &Path) -> Result<LocalSearchDocument, WebError> {
let markdown = fs::read_to_string(path).map_err(|error| {
WebError::bad_request_code(
"local_search_index_read_markdown_failed",
format!("无法读取本地 Markdown 索引文件 {}: {error}", path.display()),
)
})?;
let file_name = path
.file_name()
.and_then(|value| value.to_str())
.unwrap_or("");
let parsed = parse_markdown_page(&markdown, file_name);
let relative_path = path
.strip_prefix(root_path)
.unwrap_or(path)
.to_string_lossy()
.replace('\\', "/");
2026-05-20 10:43:38 +08:00
let document_id = format!("local-md:{}", encode_local_id_segment(&relative_path));
let metadata = fs::metadata(path).map_err(|error| {
WebError::bad_request_code(
"local_search_index_stat_failed",
format!(
"无法读取本地 Markdown 索引文件状态 {}: {error}",
path.display()
),
)
})?;
Ok(LocalSearchDocument {
document_id,
title: parsed.title,
path: relative_path,
raw_text: parsed.body.clone(),
tags: extract_tags(&markdown),
backlinks: extract_backlinks(&parsed.body),
resource_refs: extract_resource_refs(&parsed.body),
updated_at: metadata
.modified()
.ok()
.and_then(|value| value.duration_since(UNIX_EPOCH).ok())
.map(|value| value.as_millis())
.unwrap_or_default(),
})
}
fn index_resource_file(root_path: &Path, path: &Path) -> Result<LocalSearchResource, WebError> {
let resource_type = resource_type_from_path(path).ok_or_else(|| {
WebError::bad_request_code(
"local_search_index_resource_type_unsupported",
format!("不支持索引该资源文件 {}", path.display()),
)
})?;
let relative_path = path
.strip_prefix(root_path)
.unwrap_or(path)
.to_string_lossy()
.replace('\\', "/");
let title = path
.file_stem()
.and_then(|value| value.to_str())
.unwrap_or("resource")
.trim()
.trim_end_matches(".mindmap")
.to_string();
let metadata = fs::metadata(path).map_err(|error| {
WebError::bad_request_code(
"local_search_index_stat_failed",
format!("无法读取本地资源索引文件状态 {}: {error}", path.display()),
)
})?;
Ok(LocalSearchResource {
resource_id: format!("local-resource:{}", relative_path.replace('/', "~2F")),
resource_type: resource_type.to_string(),
title,
path: relative_path,
updated_at: metadata
.modified()
.ok()
.and_then(|value| value.duration_since(UNIX_EPOCH).ok())
.map(|value| value.as_millis())
.unwrap_or_default(),
})
}
fn write_local_search_index(root_path: &Path, index: &LocalSearchIndex) -> Result<(), WebError> {
write_local_search_index_json(root_path, index)?;
write_evidence_sqlite_index(root_path, index)
}
fn write_local_search_index_json(
root_path: &Path,
index: &LocalSearchIndex,
) -> Result<(), WebError> {
let index_dir = root_path.join(".mnote").join("index");
fs::create_dir_all(&index_dir).map_err(|error| {
WebError::bad_request_code(
"local_search_index_create_failed",
format!("无法创建本地搜索索引目录 {}: {error}", index_dir.display()),
)
})?;
let index_path = index_dir.join("search-index.json");
let payload = serde_json::to_string_pretty(index)
.map_err(|error| WebError::internal(format!("本地搜索索引序列化失败: {error}")))?;
fs::write(&index_path, format!("{payload}\n")).map_err(|error| {
WebError::bad_request_code(
"local_search_index_write_failed",
format!("无法写入本地搜索索引 {}: {error}", index_path.display()),
)
})
}
fn ensure_evidence_sqlite_index(
root_path: &Path,
index: &LocalSearchIndex,
) -> Result<(), WebError> {
let path = evidence_sqlite_path(root_path);
if path.exists() {
return Ok(());
}
write_evidence_sqlite_index(root_path, index)
}
fn write_evidence_sqlite_index(root_path: &Path, index: &LocalSearchIndex) -> Result<(), WebError> {
let Some(parent) = evidence_sqlite_path(root_path)
.parent()
.map(Path::to_path_buf)
else {
return Ok(());
};
fs::create_dir_all(&parent).map_err(|error| {
WebError::bad_request_code(
"evidence_index_create_failed",
format!("无法创建 evidence 索引目录 {}: {error}", parent.display()),
)
})?;
let path = evidence_sqlite_path(root_path);
let mut connection = Connection::open(&path).map_err(|error| {
WebError::bad_request_code(
"evidence_index_open_failed",
format!("无法打开 evidence 索引 {}: {error}", path.display()),
)
})?;
let tx = connection.transaction().map_err(sqlite_error)?;
create_evidence_schema(&tx)?;
tx.execute("DELETE FROM evidence_fts", [])
.map_err(sqlite_error)?;
for table in [
"evidence_edge",
"evidence_section",
"evidence_block",
"evidence_resource",
] {
tx.execute(&format!("DELETE FROM {table}"), [])
.map_err(sqlite_error)?;
}
for document in &index.documents {
insert_document_evidence(&tx, index, document)?;
}
for resource in &index.resources {
insert_resource_evidence(&tx, root_path, index, resource)?;
}
for entry in local_ocr::ocr_index_entries(root_path)?
.into_iter()
.filter(|entry| {
index_relative_path_is_included(&entry.source_root_relative_path, &index.indexed_paths)
})
{
insert_ocr_evidence(&tx, root_path, index, &entry)?;
}
for document in &index.documents {
insert_document_graph_edges(&tx, document)?;
}
tx.execute(
"INSERT OR REPLACE INTO evidence_meta(key, value) VALUES('schema_version', ?1)",
params![EVIDENCE_SQLITE_SCHEMA_VERSION.to_string()],
)
.map_err(sqlite_error)?;
tx.execute(
"INSERT OR REPLACE INTO evidence_meta(key, value) VALUES('workspace_id', ?1)",
params![index.workspace_id],
)
.map_err(sqlite_error)?;
tx.commit().map_err(sqlite_error)
}
fn create_evidence_schema(connection: &Connection) -> Result<(), WebError> {
connection
.execute_batch(
r#"
CREATE TABLE IF NOT EXISTS evidence_meta(
key TEXT PRIMARY KEY,
value TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS evidence_resource(
resource_id TEXT PRIMARY KEY,
owner_document_id TEXT NOT NULL,
owner_document_path TEXT NOT NULL,
source_root_relative_path TEXT NOT NULL,
provider TEXT NOT NULL,
source_hash TEXT NOT NULL,
artifact_root_relative_path TEXT NOT NULL,
source_map_root_relative_path TEXT NOT NULL,
updated_at_ms INTEGER NOT NULL
);
CREATE TABLE IF NOT EXISTS evidence_block(
block_id TEXT PRIMARY KEY,
resource_id TEXT NOT NULL,
text TEXT NOT NULL,
section_path_json TEXT NOT NULL,
page_start INTEGER,
page_end INTEGER,
bbox_json TEXT,
char_range_json TEXT,
line_range_json TEXT,
locator_json TEXT NOT NULL
);
CREATE VIRTUAL TABLE IF NOT EXISTS evidence_fts USING fts5(
block_id UNINDEXED,
text
);
CREATE TABLE IF NOT EXISTS evidence_section(
section_id TEXT PRIMARY KEY,
resource_id TEXT NOT NULL,
title TEXT NOT NULL,
path_json TEXT NOT NULL,
summary TEXT,
page_start INTEGER,
page_end INTEGER,
parent_section_id TEXT
);
CREATE TABLE IF NOT EXISTS evidence_edge(
edge_id TEXT PRIMARY KEY,
from_id TEXT NOT NULL,
to_id TEXT NOT NULL,
edge_type TEXT NOT NULL,
source_block_id TEXT,
confidence REAL NOT NULL,
created_by TEXT NOT NULL
);
"#,
)
.map_err(sqlite_error)
}
fn refresh_evidence_sqlite_index_for_path(
root_path: &Path,
index: &LocalSearchIndex,
relative_path: &str,
) -> Result<(), WebError> {
if let Some(parent) = evidence_sqlite_path(root_path)
.parent()
.map(Path::to_path_buf)
{
fs::create_dir_all(&parent).map_err(|error| {
WebError::bad_request_code(
"evidence_index_create_failed",
format!("无法创建 evidence 索引目录 {}: {error}", parent.display()),
)
})?;
}
let mut connection = Connection::open(evidence_sqlite_path(root_path)).map_err(|error| {
WebError::bad_request_code(
"evidence_index_open_failed",
format!("无法打开 evidence 索引: {error}"),
)
})?;
let tx = connection.transaction().map_err(sqlite_error)?;
create_evidence_schema(&tx)?;
let mut affected_resource_ids = affected_evidence_resource_ids(index, relative_path);
affected_resource_ids.extend(existing_evidence_resource_ids_for_path(&tx, relative_path)?);
affected_resource_ids.sort();
affected_resource_ids.dedup();
for resource_id in &affected_resource_ids {
delete_evidence_resource_projection(&tx, resource_id)?;
}
if let Some(document) = index
.documents
.iter()
.find(|document| document.path == relative_path)
{
insert_document_evidence(&tx, index, document)?;
insert_document_graph_edges(&tx, document)?;
}
if let Some(resource) = index
.resources
.iter()
.find(|resource| resource.path == relative_path)
{
insert_resource_evidence(&tx, root_path, index, resource)?;
}
for entry in local_ocr::ocr_index_entries(root_path)?
.into_iter()
.filter(|entry| {
entry.ocr_root_relative_path == relative_path
|| entry.source_root_relative_path == relative_path
})
{
insert_ocr_evidence(&tx, root_path, index, &entry)?;
}
tx.execute(
"INSERT OR REPLACE INTO evidence_meta(key, value) VALUES('schema_version', ?1)",
params![EVIDENCE_SQLITE_SCHEMA_VERSION.to_string()],
)
.map_err(sqlite_error)?;
tx.execute(
"INSERT OR REPLACE INTO evidence_meta(key, value) VALUES('workspace_id', ?1)",
params![index.workspace_id],
)
.map_err(sqlite_error)?;
tx.commit().map_err(sqlite_error)
}
fn remove_evidence_sqlite_index_for_path(
root_path: &Path,
index: &LocalSearchIndex,
relative_path: &str,
) -> Result<(), WebError> {
let path = evidence_sqlite_path(root_path);
if !path.exists() {
return Ok(());
}
let mut connection = Connection::open(&path).map_err(|error| {
WebError::bad_request_code(
"evidence_index_open_failed",
format!("无法打开 evidence 索引 {}: {error}", path.display()),
)
})?;
let tx = connection.transaction().map_err(sqlite_error)?;
create_evidence_schema(&tx)?;
let mut affected_resource_ids = affected_evidence_resource_ids(index, relative_path);
affected_resource_ids.extend(existing_evidence_resource_ids_for_path(&tx, relative_path)?);
affected_resource_ids.sort();
affected_resource_ids.dedup();
for resource_id in &affected_resource_ids {
delete_evidence_resource_projection(&tx, resource_id)?;
}
tx.commit().map_err(sqlite_error)
}
fn affected_evidence_resource_ids(index: &LocalSearchIndex, relative_path: &str) -> Vec<String> {
let mut ids = Vec::new();
let document_id = format!("local-md:{}", encode_local_id_segment(relative_path));
ids.push(document_id.clone());
ids.push(format!(
"local-resource:{}",
relative_path.replace('/', "~2F")
));
ids.extend(
index
.documents
.iter()
.filter(|document| document.path == relative_path)
.map(|document| document.document_id.clone()),
);
ids.extend(
index
.resources
.iter()
.filter(|resource| resource.path == relative_path)
.map(|resource| resource.resource_id.clone()),
);
if let Some(stripped) = relative_path.strip_suffix(".ocr.md") {
ids.push(format!("{document_id}#ocr:{stripped}"));
}
ids
}
fn existing_evidence_resource_ids_for_path(
connection: &Connection,
relative_path: &str,
) -> Result<Vec<String>, WebError> {
let mut statement = connection
.prepare(
"SELECT resource_id FROM evidence_resource \
WHERE source_root_relative_path = ?1 \
OR artifact_root_relative_path = ?1 \
OR source_map_root_relative_path = ?1",
)
.map_err(sqlite_error)?;
let resource_ids = statement
.query_map(params![relative_path], |row| row.get::<_, String>(0))
.map_err(sqlite_error)?
.collect::<Result<Vec<_>, _>>()
.map_err(sqlite_error)?;
Ok(resource_ids)
}
fn delete_evidence_resource_projection(
connection: &Connection,
resource_id: &str,
) -> Result<(), WebError> {
let mut statement = connection
.prepare("SELECT block_id FROM evidence_block WHERE resource_id = ?1")
.map_err(sqlite_error)?;
let block_ids = statement
.query_map(params![resource_id], |row| row.get::<_, String>(0))
.map_err(sqlite_error)?
.collect::<Result<Vec<_>, _>>()
.map_err(sqlite_error)?;
for block_id in &block_ids {
connection
.execute(
"DELETE FROM evidence_fts WHERE block_id = ?1",
params![block_id],
)
.map_err(sqlite_error)?;
connection
.execute(
"DELETE FROM evidence_edge WHERE source_block_id = ?1",
params![block_id],
)
.map_err(sqlite_error)?;
}
connection
.execute(
"DELETE FROM evidence_edge WHERE from_id = ?1",
params![resource_id],
)
.map_err(sqlite_error)?;
connection
.execute(
"DELETE FROM evidence_section WHERE resource_id = ?1",
params![resource_id],
)
.map_err(sqlite_error)?;
connection
.execute(
"DELETE FROM evidence_block WHERE resource_id = ?1",
params![resource_id],
)
.map_err(sqlite_error)?;
connection
.execute(
"DELETE FROM evidence_resource WHERE resource_id = ?1",
params![resource_id],
)
.map(|_| ())
.map_err(sqlite_error)
}
fn insert_document_evidence(
connection: &Connection,
index: &LocalSearchIndex,
document: &LocalSearchDocument,
) -> Result<(), WebError> {
let resource_id = document.document_id.as_str();
let artifact = markdown_parsed_artifact(document);
insert_evidence_resource_from_artifact(connection, resource_id, &artifact)?;
for section in markdown_evidence_sections(document) {
insert_markdown_evidence_section(connection, resource_id, &section)?;
}
let blocks = markdown_evidence_blocks(document);
if blocks.is_empty() {
let block_id = format!("{resource_id}#line1");
let line_range = json!({"start": 1, "end": 1});
let locator = markdown_locator(
index,
&document.document_id,
&document.path,
&block_id,
Vec::new(),
Some(line_range.clone()),
);
return insert_evidence_block_with_metadata(
connection,
&block_id,
resource_id,
&document.raw_text,
Vec::new(),
Some(line_range),
locator,
);
}
for block in blocks {
let locator = markdown_locator(
index,
&document.document_id,
&document.path,
&block.block_id,
block.section_path.clone(),
Some(block.line_range.clone()),
);
insert_evidence_block_with_metadata(
connection,
&block.block_id,
resource_id,
&block.text,
block.section_path,
Some(block.line_range),
locator,
)?;
}
Ok(())
}
fn markdown_parsed_artifact(document: &LocalSearchDocument) -> ParsedResourceArtifact {
ParsedResourceArtifact {
schema: PARSED_RESOURCE_ARTIFACT_SCHEMA.into(),
provider: "markdown".into(),
model_version: None,
owner_document_id: document.document_id.clone(),
owner_document_path: document.path.clone(),
source_root_relative_path: document.path.clone(),
source_hash: format!("updated:{}", document.updated_at),
artifact_root_relative_path: document.path.clone(),
source_map_root_relative_path: format!("{}.source-map.json", document.path),
updated_at_ms: document.updated_at as u64,
}
}
fn insert_evidence_resource_from_artifact(
connection: &Connection,
resource_id: &str,
artifact: &ParsedResourceArtifact,
) -> Result<(), WebError> {
insert_evidence_resource(
connection,
resource_id,
&artifact.owner_document_id,
&artifact.owner_document_path,
&artifact.source_root_relative_path,
&artifact.provider,
&artifact.source_hash,
&artifact.artifact_root_relative_path,
&artifact.source_map_root_relative_path,
artifact.updated_at_ms as u128,
)
}
fn insert_resource_evidence(
connection: &Connection,
root_path: &Path,
index: &LocalSearchIndex,
resource: &LocalSearchResource,
) -> Result<(), WebError> {
insert_evidence_resource(
connection,
&resource.resource_id,
&resource.resource_id,
&resource.path,
&resource.path,
"resource",
&format!("updated:{}", resource.updated_at),
&resource.path,
"",
resource.updated_at,
)?;
let locator = json!({
"schema": "mnote.evidence_locator.v1",
"rootUri": index.root_uri,
"ownerDocumentId": resource.resource_id,
"ownerDocumentPath": resource.path,
"resourcePath": resource.path,
"resourceKind": evidence_resource_kind_for_type(&resource.resource_type),
"openAction": {
"actionType": "mnote.open_resource_locator",
"url": format!("/?treeView=filetree&sourceKind=local_folder&rootUri={}", encode_query_component(&index.root_uri)),
"params": {"resourcePath": resource.path}
}
});
insert_evidence_block(
connection,
&format!("{}#resource", resource.resource_id),
&resource.resource_id,
&format!("{}\n{}", resource.title, resource.path),
locator,
)?;
if let Some(output) = parse_resource_evidence_artifact(root_path, index, resource)? {
insert_source_map_artifact_evidence(
connection,
index,
&parsed_resource_id(resource),
&output.artifact,
&output.source_map,
&output.markdown,
)?;
}
Ok(())
}
fn parse_resource_evidence_artifact(
root_path: &Path,
index: &LocalSearchIndex,
resource: &LocalSearchResource,
) -> Result<Option<crate::evidence_parse::ParseProviderOutput>, WebError> {
if !resource_parse_supported(resource) || !liteparse_runtime_available() {
return Ok(None);
}
let owner = evidence_owner_for_resource(index, resource);
let mut output = match parse_liteparse_input_blocking(ParseInput {
root_path: root_path.to_path_buf(),
root_uri: index.root_uri.clone(),
owner_document_id: owner.document_id.clone(),
owner_document_path: owner.path.clone(),
source_root_relative_path: resource.path.clone(),
mode: ParseProviderMode::NoOcr,
}) {
Ok(output) => output,
Err(_) => return Ok(None),
};
let sidecar = resource_parse_sidecar_paths(&owner.path, &resource.path);
write_parsed_resource_sidecars(root_path, &sidecar, &output.markdown, &output.source_map)?;
output.artifact.artifact_root_relative_path = sidecar.parse_root_relative_path;
output.artifact.source_map_root_relative_path = sidecar.source_map_root_relative_path.clone();
output.source_map.owner_document_path = owner.path;
Ok(Some(output))
}
fn resource_parse_supported(resource: &LocalSearchResource) -> bool {
matches!(resource.resource_type.as_str(), "pdf" | "office")
}
fn parsed_resource_id(resource: &LocalSearchResource) -> String {
format!("{}#parse", resource.resource_id)
}
#[derive(Debug, Clone)]
struct EvidenceResourceOwner {
document_id: String,
path: String,
}
fn evidence_owner_for_resource(
index: &LocalSearchIndex,
resource: &LocalSearchResource,
) -> EvidenceResourceOwner {
if let Some(document) = index.documents.iter().find(|document| {
document.resource_refs.iter().any(|reference| {
normalize_local_reference_path(&document.path, reference) == resource.path
})
}) {
return EvidenceResourceOwner {
document_id: document.document_id.clone(),
path: document.path.clone(),
};
}
EvidenceResourceOwner {
document_id: resource.resource_id.clone(),
path: resource.path.clone(),
}
}
#[derive(Debug, Clone)]
struct ResourceParseSidecarPaths {
parse_root_relative_path: String,
source_map_root_relative_path: String,
}
fn resource_parse_sidecar_paths(
owner_document_path: &str,
source_root_relative_path: &str,
) -> ResourceParseSidecarPaths {
let owner_parent = Path::new(owner_document_path)
.parent()
.map(Path::to_path_buf)
.unwrap_or_default();
let owner_stem = Path::new(owner_document_path)
.file_stem()
.and_then(|value| value.to_str())
.unwrap_or("Page");
let source_leaf = Path::new(source_root_relative_path)
.file_name()
.and_then(|value| value.to_str())
.unwrap_or("source");
let sidecar_dir = owner_parent.join(format!("{owner_stem}.ocr"));
let parse_root_relative_path = sidecar_dir
.join(format!("{source_leaf}.parse.md"))
.to_string_lossy()
.replace('\\', "/");
let source_map_root_relative_path = sidecar_dir
.join(format!("{source_leaf}.source-map.json"))
.to_string_lossy()
.replace('\\', "/");
ResourceParseSidecarPaths {
parse_root_relative_path,
source_map_root_relative_path,
}
}
fn write_parsed_resource_sidecars(
root_path: &Path,
sidecar: &ResourceParseSidecarPaths,
markdown: &str,
source_map: &ResourceSourceMap,
) -> Result<(), WebError> {
let parse_path = root_path.join(&sidecar.parse_root_relative_path);
let source_map_path = root_path.join(&sidecar.source_map_root_relative_path);
for path in [&parse_path, &source_map_path] {
if let Some(parent) = path.parent() {
fs::create_dir_all(parent).map_err(|error| {
WebError::bad_request_code(
"evidence_parse_sidecar_create_failed",
format!(
"无法创建证据解析 sidecar 目录 {}: {error}",
parent.display()
),
)
})?;
}
}
fs::write(&parse_path, markdown.trim_end()).map_err(|error| {
WebError::bad_request_code(
"evidence_parse_sidecar_write_failed",
format!(
"无法写入证据解析 Markdown {}: {error}",
parse_path.display()
),
)
})?;
let source_map_json = serde_json::to_string_pretty(source_map)
.map_err(|error| WebError::internal(format!("证据 source-map 序列化失败: {error}")))?;
fs::write(&source_map_path, format!("{source_map_json}\n")).map_err(|error| {
WebError::bad_request_code(
"evidence_parse_sidecar_write_failed",
format!(
"无法写入证据 source-map {}: {error}",
source_map_path.display()
),
)
})
}
fn insert_ocr_evidence(
connection: &Connection,
root_path: &Path,
index: &LocalSearchIndex,
entry: &local_ocr::OcrIndexEntry,
) -> Result<(), WebError> {
if entry.status != "done" {
return Ok(());
}
let ocr_path = root_path.join(&entry.ocr_root_relative_path);
let Ok(markdown) = fs::read_to_string(&ocr_path) else {
return Ok(());
};
let body = local_ocr::strip_ocr_frontmatter(&markdown);
let source_map = path_source_map_path(&entry.ocr_root_relative_path).unwrap_or_default();
let resource_id = format!(
"{}#ocr:{}",
entry.owner_document_id, entry.source_root_relative_path
);
let artifact = ocr_parsed_artifact(entry, &source_map);
if !source_map.is_empty() {
let source_map_path = root_path.join(&source_map);
if let Ok(source_map_content) = fs::read_to_string(&source_map_path) {
if let Ok(resource_source_map) =
serde_json::from_str::<ResourceSourceMap>(&source_map_content)
{
return insert_source_map_artifact_evidence(
connection,
index,
&resource_id,
&artifact,
&resource_source_map,
body,
);
}
}
}
insert_evidence_resource_from_artifact(connection, &resource_id, &artifact)?;
let locator = json!({
"schema": "mnote.evidence_locator.v1",
"rootUri": index.root_uri,
"ownerDocumentId": entry.owner_document_id,
"ownerDocumentPath": entry.owner_document_path,
"resourcePath": entry.source_root_relative_path,
"resourceKind": evidence_resource_kind_for_path(&entry.source_root_relative_path),
"sourceMapPath": source_map,
"openAction": {
"actionType": "mnote.open_resource_locator",
"url": format!("/documents/{}?sourceKind=local_folder&rootUri={}", entry.owner_document_id, encode_query_component(&index.root_uri)),
"params": {
"resourcePath": entry.source_root_relative_path,
"sourceMapPath": source_map
}
}
});
insert_evidence_block(connection, &resource_id, &resource_id, body, locator)
}
fn ocr_parsed_artifact(
entry: &local_ocr::OcrIndexEntry,
source_map_root_relative_path: &str,
) -> ParsedResourceArtifact {
ParsedResourceArtifact {
schema: PARSED_RESOURCE_ARTIFACT_SCHEMA.into(),
provider: entry.provider.clone(),
model_version: Some(entry.model_version.clone()),
owner_document_id: entry.owner_document_id.clone(),
owner_document_path: entry.owner_document_path.clone(),
source_root_relative_path: entry.source_root_relative_path.clone(),
source_hash: format!("size:{}:mtime:{}", entry.source_size, entry.source_mtime_ms),
artifact_root_relative_path: entry.ocr_root_relative_path.clone(),
source_map_root_relative_path: source_map_root_relative_path.to_string(),
updated_at_ms: entry.updated_at_ms as u64,
}
}
fn insert_source_map_artifact_evidence(
connection: &Connection,
index: &LocalSearchIndex,
resource_id: &str,
artifact: &ParsedResourceArtifact,
source_map: &ResourceSourceMap,
fallback_text: &str,
) -> Result<(), WebError> {
insert_evidence_resource_from_artifact(connection, resource_id, artifact)?;
for section in &source_map.sections {
insert_source_map_evidence_section(connection, resource_id, section)?;
}
let mut inserted = 0_usize;
for page in &source_map.pages {
for block in &page.blocks {
if block.text.trim().is_empty() {
continue;
}
let block_id = format!("{resource_id}#{}", block.id);
let section_path = source_map_section_path_for_block(source_map, &block.id);
let locator = source_map_block_locator(
index,
artifact,
&block.id,
Some(page.page),
block,
section_path.clone(),
);
insert_evidence_block_with_metadata(
connection,
&block_id,
resource_id,
&block.text,
section_path,
None,
locator,
)?;
inserted += 1;
}
}
if inserted == 0 {
let block_id = format!("{resource_id}#artifact");
let fallback_block = empty_source_map_block(fallback_text);
let locator = source_map_block_locator(
index,
artifact,
&fallback_block.id,
None,
&fallback_block,
Vec::new(),
);
insert_evidence_block_with_metadata(
connection,
&block_id,
resource_id,
fallback_text,
Vec::new(),
None,
locator,
)?;
}
Ok(())
}
fn insert_source_map_evidence_section(
connection: &Connection,
resource_id: &str,
section: &core_protocol::SourceMapSection,
) -> Result<(), WebError> {
let path_json = serde_json::to_string(&section.path).map_err(|error| {
WebError::bad_request_code(
"evidence_index_section_serialize_failed",
format!("source-map section path 序列化失败: {error}"),
)
})?;
connection
.execute(
"INSERT OR REPLACE INTO evidence_section(section_id, resource_id, title, path_json, summary, page_start, page_end, parent_section_id) VALUES(?1, ?2, ?3, ?4, NULL, ?5, ?6, NULL)",
params![
format!("{resource_id}#{}", section.id),
resource_id,
section.title,
path_json,
section.page_start,
section.page_end,
],
)
.map(|_| ())
.map_err(sqlite_error)
}
fn source_map_section_path_for_block(
source_map: &ResourceSourceMap,
block_id: &str,
) -> Vec<String> {
source_map
.sections
.iter()
.find(|section| section.block_ids.iter().any(|id| id == block_id))
.map(|section| section.path.clone())
.unwrap_or_default()
}
fn source_map_block_locator(
index: &LocalSearchIndex,
artifact: &ParsedResourceArtifact,
source_map_block_id: &str,
page: Option<u32>,
block: &SourceMapBlock,
section_path: Vec<String>,
) -> Value {
let mut params = json!({
"resourcePath": artifact.source_root_relative_path,
"sourceMapPath": artifact.source_map_root_relative_path,
"blockId": source_map_block_id,
});
if let Some(params_object) = params.as_object_mut() {
if let Some(page) = page {
params_object.insert("page".into(), json!(page));
}
if let Some(bbox) = &block.bbox {
params_object.insert("bbox".into(), json!(bbox));
}
if let Some(char_range) = &block.char_range {
params_object.insert("charRange".into(), json!(char_range));
}
}
json!({
"schema": "mnote.evidence_locator.v1",
"rootUri": index.root_uri,
"ownerDocumentId": artifact.owner_document_id,
"ownerDocumentPath": artifact.owner_document_path,
"resourcePath": artifact.source_root_relative_path,
"resourceKind": evidence_resource_kind_for_path(&artifact.source_root_relative_path),
"page": page,
"bbox": block.bbox,
"sectionPath": section_path,
"charRange": block.char_range,
"blockId": source_map_block_id,
"sourceMapPath": artifact.source_map_root_relative_path,
"openAction": {
"actionType": "mnote.open_resource_locator",
"url": format!("/documents/{}?sourceKind=local_folder&rootUri={}", artifact.owner_document_id, encode_query_component(&index.root_uri)),
"params": params
}
})
}
fn empty_source_map_block(text: &str) -> SourceMapBlock {
SourceMapBlock {
id: "artifact".into(),
block_type: core_protocol::SourceMapBlockKind::Text,
text: text.to_string(),
bbox: None,
char_range: None,
}
}
fn insert_evidence_resource(
connection: &Connection,
resource_id: &str,
owner_document_id: &str,
owner_document_path: &str,
source_root_relative_path: &str,
provider: &str,
source_hash: &str,
artifact_root_relative_path: &str,
source_map_root_relative_path: &str,
updated_at_ms: u128,
) -> Result<(), WebError> {
connection
.execute(
"INSERT OR REPLACE INTO evidence_resource(resource_id, owner_document_id, owner_document_path, source_root_relative_path, provider, source_hash, artifact_root_relative_path, source_map_root_relative_path, updated_at_ms) VALUES(?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)",
params![
resource_id,
owner_document_id,
owner_document_path,
source_root_relative_path,
provider,
source_hash,
artifact_root_relative_path,
source_map_root_relative_path,
updated_at_ms.to_string(),
],
)
.map(|_| ())
.map_err(sqlite_error)
}
fn insert_evidence_block(
connection: &Connection,
block_id: &str,
resource_id: &str,
text: &str,
locator: Value,
) -> Result<(), WebError> {
insert_evidence_block_with_metadata(
connection,
block_id,
resource_id,
text,
Vec::new(),
None,
locator,
)
}
fn insert_evidence_block_with_metadata(
connection: &Connection,
block_id: &str,
resource_id: &str,
text: &str,
section_path: Vec<String>,
line_range: Option<Value>,
locator: Value,
) -> Result<(), WebError> {
let locator_json = serde_json::to_string(&locator).map_err(|error| {
WebError::bad_request_code(
"evidence_index_locator_serialize_failed",
format!("locator 序列化失败: {error}"),
)
})?;
let section_path_json = serde_json::to_string(&section_path).map_err(|error| {
WebError::bad_request_code(
"evidence_index_section_serialize_failed",
format!("sectionPath 序列化失败: {error}"),
)
})?;
let line_range_json = line_range
.map(|value| serde_json::to_string(&value))
.transpose()
.map_err(|error| {
WebError::bad_request_code(
"evidence_index_line_range_serialize_failed",
format!("lineRange 序列化失败: {error}"),
)
})?;
connection
.execute(
"INSERT OR REPLACE INTO evidence_block(block_id, resource_id, text, section_path_json, page_start, page_end, bbox_json, char_range_json, line_range_json, locator_json) VALUES(?1, ?2, ?3, ?4, NULL, NULL, NULL, NULL, ?5, ?6)",
params![block_id, resource_id, text, section_path_json, line_range_json, locator_json],
)
.map_err(sqlite_error)?;
connection
.execute(
"INSERT INTO evidence_fts(block_id, text) VALUES(?1, ?2)",
params![block_id, text],
)
.map(|_| ())
.map_err(sqlite_error)
}
#[derive(Debug, Clone)]
struct MarkdownEvidenceBlock {
block_id: String,
text: String,
section_path: Vec<String>,
line_range: Value,
}
#[derive(Debug, Clone)]
struct MarkdownEvidenceSection {
section_id: String,
title: String,
path: Vec<String>,
line_start: u64,
parent_section_id: Option<String>,
}
fn markdown_evidence_blocks(document: &LocalSearchDocument) -> Vec<MarkdownEvidenceBlock> {
let mut headings: Vec<String> = Vec::new();
let mut blocks = Vec::new();
for (index, line) in document.raw_text.lines().enumerate() {
let line_number = (index + 1) as u64;
let trimmed = line.trim();
if trimmed.is_empty() {
continue;
}
if let Some((level, title)) = markdown_heading(trimmed) {
headings.truncate(level.saturating_sub(1));
headings.push(title);
}
blocks.push(MarkdownEvidenceBlock {
block_id: format!("{}#line{}", document.document_id, line_number),
text: trimmed.to_string(),
section_path: headings.clone(),
line_range: json!({"start": line_number, "end": line_number}),
});
}
blocks
}
fn markdown_evidence_sections(document: &LocalSearchDocument) -> Vec<MarkdownEvidenceSection> {
let mut headings: Vec<String> = Vec::new();
let mut sections = Vec::new();
for (index, line) in document.raw_text.lines().enumerate() {
let line_number = (index + 1) as u64;
let Some((level, title)) = markdown_heading(line.trim()) else {
continue;
};
headings.truncate(level.saturating_sub(1));
headings.push(title.clone());
let path = headings.clone();
let parent_section_id = if path.len() > 1 {
Some(format!(
"{}#section:{}",
document.document_id,
encode_local_id_segment(&path[..path.len() - 1].join("/"))
))
} else {
None
};
sections.push(MarkdownEvidenceSection {
section_id: format!(
"{}#section:{}",
document.document_id,
encode_local_id_segment(&path.join("/"))
),
title,
path,
line_start: line_number,
parent_section_id,
});
}
sections
}
fn markdown_heading(line: &str) -> Option<(usize, String)> {
let trimmed = line.trim_start();
let marker_count = trimmed.chars().take_while(|value| *value == '#').count();
if marker_count == 0 || marker_count > 6 {
return None;
}
let rest = trimmed.get(marker_count..)?.trim();
if rest.is_empty() {
return None;
}
Some((marker_count, rest.trim_matches('#').trim().to_string()))
}
fn insert_markdown_evidence_section(
connection: &Connection,
resource_id: &str,
section: &MarkdownEvidenceSection,
) -> Result<(), WebError> {
let path_json = serde_json::to_string(&section.path).map_err(|error| {
WebError::bad_request_code(
"evidence_index_section_serialize_failed",
format!("section path 序列化失败: {error}"),
)
})?;
connection
.execute(
"INSERT OR REPLACE INTO evidence_section(section_id, resource_id, title, path_json, summary, page_start, page_end, parent_section_id) VALUES(?1, ?2, ?3, ?4, NULL, ?5, ?5, ?6)",
params![
section.section_id,
resource_id,
section.title,
path_json,
section.line_start,
section.parent_section_id,
],
)
.map(|_| ())
.map_err(sqlite_error)
}
fn insert_document_graph_edges(
connection: &Connection,
document: &LocalSearchDocument,
) -> Result<(), WebError> {
let source_block_id = first_markdown_evidence_block_id(document);
for target in &document.backlinks {
let to_id = markdown_reference_target_id(&document.path, target);
insert_evidence_edge(
connection,
&document.document_id,
&to_id,
"markdown_links_to",
&source_block_id,
)?;
}
for target in &document.resource_refs {
let to_id = resource_reference_target_id(&document.path, target);
insert_evidence_edge(
connection,
&document.document_id,
&to_id,
"resource_refers_to",
&source_block_id,
)?;
insert_evidence_edge(
connection,
&document.document_id,
&to_id,
"document_contains_resource",
&source_block_id,
)?;
}
for mention in markdown_mention_edges(document) {
insert_evidence_edge(
connection,
&document.document_id,
&mention.target_id,
"mentions",
&mention.source_block_id,
)?;
}
Ok(())
}
fn first_markdown_evidence_block_id(document: &LocalSearchDocument) -> String {
document
.raw_text
.lines()
.enumerate()
.find(|(_, line)| !line.trim().is_empty())
.map(|(index, _)| format!("{}#line{}", document.document_id, index + 1))
.unwrap_or_else(|| format!("{}#line1", document.document_id))
}
fn insert_evidence_edge(
connection: &Connection,
from_id: &str,
to_id: &str,
edge_type: &str,
source_block_id: &str,
) -> Result<(), WebError> {
let has_source_locator = connection
.query_row(
"SELECT 1 FROM evidence_block WHERE block_id = ?1 AND locator_json <> '' LIMIT 1",
params![source_block_id],
|_| Ok(true),
)
.optional()
.map_err(sqlite_error)?
.unwrap_or(false);
if !has_source_locator {
return Ok(());
}
let edge_id = format!(
"{}:{}:{}",
edge_type,
encode_local_id_segment(from_id),
encode_local_id_segment(to_id)
);
connection
.execute(
"INSERT OR REPLACE INTO evidence_edge(edge_id, from_id, to_id, edge_type, source_block_id, confidence, created_by) VALUES(?1, ?2, ?3, ?4, ?5, 1.0, 'deterministic_markdown')",
params![edge_id, from_id, to_id, edge_type, source_block_id],
)
.map(|_| ())
.map_err(sqlite_error)
}
fn markdown_reference_target_id(owner_path: &str, target: &str) -> String {
let target = target.trim();
if target.starts_with("local-md:") {
return target.to_string();
}
if is_markdown_reference(target) {
return format!(
"local-md:{}",
encode_local_id_segment(&normalize_local_reference_path(owner_path, target))
);
}
format!("markdown-title:{}", encode_local_id_segment(target))
}
fn resource_reference_target_id(owner_path: &str, target: &str) -> String {
format!(
"local-resource:{}",
encode_local_id_segment(&normalize_local_reference_path(owner_path, target))
)
}
#[derive(Debug, Clone)]
struct MarkdownMentionEdge {
target_id: String,
source_block_id: String,
}
fn markdown_mention_edges(document: &LocalSearchDocument) -> Vec<MarkdownMentionEdge> {
let mut seen = std::collections::BTreeSet::new();
let mut mentions = Vec::new();
for (line_index, line) in document.raw_text.lines().enumerate() {
let source_block_id = format!("{}#line{}", document.document_id, line_index + 1);
for mention in extract_line_mentions(line) {
let target_id = format!("entity:{}", encode_local_id_segment(&mention));
if seen.insert((target_id.clone(), source_block_id.clone())) {
mentions.push(MarkdownMentionEdge {
target_id,
source_block_id: source_block_id.clone(),
});
}
}
}
mentions
}
fn extract_line_mentions(line: &str) -> Vec<String> {
let chars = line.chars().collect::<Vec<_>>();
let mut mentions = Vec::new();
let mut index = 0;
while index < chars.len() {
if chars[index] != '@' {
index += 1;
continue;
}
if index > 0 && chars[index - 1].is_alphanumeric() {
index += 1;
continue;
}
let mut end = index + 1;
while end < chars.len()
&& (chars[end].is_alphanumeric()
|| chars[end] == '_'
|| chars[end] == '-'
|| chars[end] == '.')
{
end += 1;
}
if end > index + 1 {
let value = chars[index + 1..end].iter().collect::<String>();
mentions.push(value);
}
index = end.max(index + 1);
}
mentions.sort();
mentions.dedup();
mentions
}
fn markdown_locator(
index: &LocalSearchIndex,
document_id: &str,
path: &str,
block_id: &str,
section_path: Vec<String>,
line_range: Option<Value>,
) -> Value {
json!({
"schema": "mnote.evidence_locator.v1",
"rootUri": index.root_uri,
"ownerDocumentId": document_id,
"ownerDocumentPath": path,
"resourcePath": path,
"resourceKind": "markdown",
"sectionPath": section_path,
"lineRange": line_range,
"blockId": block_id,
"openAction": {
"actionType": "mnote.open_resource_locator",
"url": format!("/documents/{document_id}?sourceKind=local_folder&rootUri={}", encode_query_component(&index.root_uri)),
"params": {"resourcePath": path, "blockId": block_id, "lineRange": line_range}
}
})
}
fn evidence_sqlite_path(root_path: &Path) -> PathBuf {
root_path
.join(".mnote")
.join("index")
.join("evidence.sqlite")
}
fn count_evidence_blocks(path: &Path) -> Result<u64, WebError> {
let connection = Connection::open(path).map_err(|error| {
WebError::bad_request_code(
"evidence_index_open_failed",
format!("无法打开 evidence 索引 {}: {error}", path.display()),
)
})?;
connection
.query_row("SELECT COUNT(*) FROM evidence_block", [], |row| {
row.get::<_, u64>(0)
})
.map_err(sqlite_error)
}
fn evidence_resource_kind_for_type(resource_type: &str) -> &'static str {
match resource_type {
"mindmap" => "mindmap",
"office" => "office",
"pdf" => "pdf",
"image" => "image",
"markdown" => "markdown",
_ => "raw_file",
}
}
fn evidence_resource_kind_for_path(path: &str) -> &'static str {
match Path::new(path)
.extension()
.and_then(|value| value.to_str())
.unwrap_or_default()
.to_ascii_lowercase()
.as_str()
{
"pdf" => "pdf",
"png" | "jpg" | "jpeg" | "webp" => "image",
"doc" | "docx" | "xls" | "xlsx" | "ppt" | "pptx" => "office",
"md" => "markdown",
"mm" | "mindmap" => "mindmap",
_ => "raw_file",
}
}
fn path_source_map_path(path: &str) -> Option<String> {
if let Some(stripped) = path.strip_suffix(".ocr.md") {
return Some(format!("{stripped}.source-map.json"));
}
if let Some(stripped) = path.strip_suffix(".parse.md") {
return Some(format!("{stripped}.source-map.json"));
}
None
}
fn is_local_evidence_sidecar_relative_path(relative_path: &str) -> bool {
let normalized = relative_path.trim().replace('\\', "/");
let path = Path::new(&normalized);
let Some(file_name) = path.file_name().and_then(|value| value.to_str()) else {
return false;
};
let Some(parent_name) = path
.parent()
.and_then(|value| value.file_name())
.and_then(|value| value.to_str())
else {
return false;
};
parent_name.ends_with(".ocr")
&& (file_name.ends_with(".parse.md") || file_name.ends_with(".source-map.json"))
}
fn normalize_local_reference_path(owner_path: &str, target: &str) -> String {
let target = target
.split('#')
.next()
.unwrap_or(target)
.split('?')
.next()
.unwrap_or(target)
.trim();
if target.starts_with('/') {
return target.trim_start_matches('/').replace('\\', "/");
}
let owner_parent = Path::new(owner_path)
.parent()
.unwrap_or_else(|| Path::new(""));
normalize_path_components(owner_parent.join(target))
}
fn normalize_path_components(path: PathBuf) -> String {
let mut parts = Vec::new();
for component in path.components() {
match component {
Component::Normal(value) => parts.push(value.to_string_lossy().to_string()),
Component::ParentDir => {
parts.pop();
}
Component::CurDir => {}
_ => {}
}
}
parts.join("/")
}
fn sqlite_error(error: rusqlite::Error) -> WebError {
WebError::bad_request_code(
"evidence_index_sqlite_error",
format!("evidence SQLite 错误: {error}"),
)
}
fn local_search_document_matches(
document: &LocalSearchDocument,
query: &str,
title_only: bool,
exact: bool,
) -> bool {
if query.is_empty() {
return false;
}
let haystack = if title_only {
normalize_search_text(&document.title)
} else {
normalize_search_text(&format!(
"{}\n{}\n{}\n{}",
document.title,
document.raw_text,
document.tags.join(" "),
document.resource_refs.join(" ")
))
};
if exact {
haystack == query
} else {
haystack.contains(query)
}
}
fn local_search_resource_matches(
resource: &LocalSearchResource,
query: &str,
title_only: bool,
exact: bool,
) -> bool {
if query.is_empty() {
return false;
}
let haystack = if title_only {
normalize_search_text(&resource.title)
} else {
normalize_search_text(&format!(
"{}\n{}\n{}",
resource.title, resource.path, resource.resource_type
))
};
if exact {
haystack == query
} else {
haystack.contains(query)
}
}
2026-06-01 09:29:12 +08:00
fn local_search_ocr_matches(
entry: &local_ocr::OcrIndexEntry,
body: &str,
query: &str,
title_only: bool,
exact: bool,
) -> bool {
if query.is_empty() {
return false;
}
let haystack = if title_only {
normalize_search_text(&format!(
"{}\n{}",
entry.source_root_relative_path, entry.ocr_root_relative_path
))
} else {
normalize_search_text(&format!(
"{}\n{}\n{}",
entry.source_root_relative_path, entry.ocr_root_relative_path, body
))
};
if exact {
haystack == query
} else {
haystack.contains(query)
}
}
fn local_search_document_projection(
document: &LocalSearchDocument,
root_uri: &str,
query: &str,
) -> Value {
json!({
"id": document.document_id,
"documentId": document.document_id,
"title": document.title,
"path": document.path,
"resourceType": "markdown",
"sourceKind": "local_folder",
"rootUri": root_uri,
"snippet": search_snippet(document, query),
"tags": document.tags,
"backlinks": document.backlinks,
"resourceRefs": document.resource_refs,
"updatedAt": document.updated_at,
2026-05-20 11:57:23 +08:00
"publicPath": format!(
"/documents/{}?sourceKind=local_folder&rootUri={}",
document.document_id,
encode_query_component(root_uri),
)
})
}
fn local_search_resource_projection(resource: &LocalSearchResource, root_uri: &str) -> Value {
json!({
"id": resource.resource_id,
"documentId": resource.resource_id,
"title": resource.title,
"path": resource.path,
"resourceType": resource.resource_type,
"sourceKind": "local_folder",
"rootUri": root_uri,
"updatedAt": resource.updated_at,
2026-05-20 11:57:23 +08:00
"publicPath": format!(
"/?treeView=filetree&sourceKind=local_folder&rootUri={}",
encode_query_component(root_uri),
)
})
}
2026-06-01 09:29:12 +08:00
fn local_search_ocr_projection(
entry: &local_ocr::OcrIndexEntry,
body: &str,
root_uri: &str,
query: &str,
) -> Value {
let title = Path::new(&entry.owner_document_path)
.file_stem()
.and_then(|value| value.to_str())
.unwrap_or("OCR")
.to_string();
json!({
"id": format!("{}#ocr:{}", entry.owner_document_id, entry.source_root_relative_path),
"documentId": entry.owner_document_id,
"title": title,
"path": entry.owner_document_path,
"resourceType": "markdown",
"sourceKind": "local_folder",
"rootUri": root_uri,
"hasOcr": true,
"snippet": ocr_search_snippet(body, query),
"ocrEvidence": {
"sourceRootRelativePath": entry.source_root_relative_path,
"ocrRootRelativePath": entry.ocr_root_relative_path,
"provider": entry.provider,
"status": entry.status,
},
"updatedAt": entry.updated_at_ms,
"publicPath": format!(
"/documents/{}?sourceKind=local_folder&rootUri={}",
entry.owner_document_id,
encode_query_component(root_uri),
)
})
}
2026-05-20 11:57:23 +08:00
fn encode_query_component(value: &str) -> String {
let mut encoded = String::with_capacity(value.len());
for byte in value.as_bytes() {
if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~') {
encoded.push(*byte as char);
} else {
encoded.push_str(&format!("%{byte:02X}"));
}
}
encoded
}
fn local_recent_changes_projection(documents: &[LocalSearchDocument], root_uri: &str) -> Value {
let mut sorted = documents.iter().collect::<Vec<_>>();
sorted.sort_by(|left, right| right.updated_at.cmp(&left.updated_at));
Value::Array(
sorted
.into_iter()
.take(20)
.map(|document| {
json!({
"id": document.document_id,
"documentId": document.document_id,
"title": document.title,
"path": document.path,
"resourceType": "markdown",
"sourceKind": "local_folder",
"rootUri": root_uri,
"updatedAt": document.updated_at
})
})
.collect(),
)
}
fn normalized_document_targets(
document: &LocalSearchDocument,
) -> std::collections::BTreeSet<String> {
let mut values = std::collections::BTreeSet::new();
values.insert(normalize_search_text(&document.document_id));
values.insert(normalize_search_text(&document.path));
values.insert(normalize_search_text(&document.title));
if let Some(encoded) = document.document_id.strip_prefix("local-md:") {
values.insert(normalize_search_text(&encoded.replace("~2F", "/")));
}
values
}
fn extract_tags(markdown: &str) -> Vec<String> {
let Some(frontmatter) = split_frontmatter(markdown).0 else {
return Vec::new();
};
for line in frontmatter.lines() {
let trimmed = line.trim();
let Some(value) = trimmed
.strip_prefix("tags:")
.or_else(|| trimmed.strip_prefix("tag:"))
else {
continue;
};
return value
.trim()
.trim_matches(['[', ']'])
.split([',', ' '])
.map(|item| item.trim().trim_matches(['"', '\'']))
.filter(|item| !item.is_empty())
.map(ToOwned::to_owned)
.collect();
}
Vec::new()
}
fn extract_backlinks(markdown: &str) -> Vec<String> {
let mut values = extract_markdown_links(markdown)
.into_iter()
.filter(|target| is_markdown_reference(target))
.collect::<Vec<_>>();
let mut rest = markdown;
while let Some(start) = rest.find("[[") {
let after_start = &rest[start + 2..];
let Some(end) = after_start.find("]]") else {
break;
};
let link = after_start[..end].trim();
if !link.is_empty() {
values.push(link.to_string());
}
rest = &after_start[end + 2..];
}
values.sort();
values.dedup();
values
}
fn extract_resource_refs(markdown: &str) -> Vec<String> {
let mut values = markdown
.lines()
.filter_map(|line| parse_markdown_attachment_link(line.trim()))
.map(|(_, target)| target)
.chain(
extract_markdown_links(markdown)
.into_iter()
.filter(|target| {
!is_markdown_reference(target) && is_local_resource_reference(target)
}),
)
.collect::<Vec<_>>();
values.sort();
values.dedup();
values
}
fn extract_markdown_links(markdown: &str) -> Vec<String> {
let mut links = Vec::new();
let mut rest = markdown;
while let Some(label_end) = rest.find("](") {
let after = &rest[label_end + 2..];
let Some(target_end) = after.find(')') else {
break;
};
let target = after[..target_end].trim();
if !target.is_empty() {
links.push(target.to_string());
}
rest = &after[target_end + 1..];
}
links
}
fn is_markdown_reference(target: &str) -> bool {
let lower = target.to_lowercase();
lower.ends_with(".md") || lower.ends_with(".markdown") || lower.starts_with("local-md:")
}
fn is_local_resource_reference(target: &str) -> bool {
let lower = target.to_lowercase();
!lower.starts_with("http://")
&& !lower.starts_with("https://")
&& !lower.starts_with("mailto:")
&& !lower.starts_with('#')
&& Path::new(target).extension().is_some()
}
fn search_snippet(document: &LocalSearchDocument, query: &str) -> String {
for line in document.raw_text.lines() {
let trimmed = line.trim();
if trimmed.is_empty() {
continue;
}
if normalize_search_text(trimmed).contains(query) {
return trimmed.chars().take(180).collect();
}
}
document.raw_text.chars().take(180).collect()
}
2026-06-01 09:29:12 +08:00
fn ocr_search_snippet(body: &str, query: &str) -> String {
let normalized_body = body.replace('\n', " ");
let normalized_query = query.trim();
if normalized_query.is_empty() {
return normalized_body.chars().take(160).collect();
}
let lower = normalized_body.to_ascii_lowercase();
let lower_query = normalized_query.to_ascii_lowercase();
if let Some(byte_index) = lower.find(&lower_query) {
let start = normalized_body[..byte_index]
.char_indices()
.rev()
.nth(40)
.map(|(idx, _)| idx)
.unwrap_or(0);
normalized_body[start..].chars().take(160).collect()
} else {
normalized_body.chars().take(160).collect()
}
}
fn is_markdown_path(path: &Path) -> bool {
path.extension()
.and_then(|value| value.to_str())
.map(|value| value.eq_ignore_ascii_case("md") || value.eq_ignore_ascii_case("markdown"))
.unwrap_or(false)
}
fn resource_type_from_path(path: &Path) -> Option<&'static str> {
let file_name = path
.file_name()
.and_then(|value| value.to_str())
.unwrap_or_default()
.to_lowercase();
if file_name.ends_with(".mindmap.json") {
return Some("mindmap");
}
let extension = path
.extension()
.and_then(|value| value.to_str())
.unwrap_or_default()
.to_lowercase();
match extension.as_str() {
"pdf" => Some("pdf"),
"doc" | "docx" | "odt" | "ppt" | "pptx" | "odp" | "xls" | "xlsx" | "ods" => Some("office"),
_ => None,
}
}
fn normalize_search_text(value: &str) -> String {
value.trim().to_lowercase()
}
fn now_ms() -> u128 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|value| value.as_millis())
.unwrap_or_default()
}
#[cfg(test)]
mod tests {
use super::*;
use control_plane::SqliteControlPlaneStore;
use std::path::PathBuf;
use std::sync::{Mutex, OnceLock};
fn env_lock() -> &'static Mutex<()> {
static LOCK: OnceLock<Mutex<()>> = OnceLock::new();
LOCK.get_or_init(|| Mutex::new(()))
}
fn temp_root(name: &str) -> PathBuf {
let root = std::env::temp_dir().join(format!("{name}-{}", std::process::id()));
let _ = fs::remove_dir_all(&root);
fs::create_dir_all(root.join("docs")).expect("create root");
root
}
#[test]
fn local_search_index_extracts_backlinks_resource_refs_and_tags() {
let root = temp_root("mnote-local-search-index");
fs::write(
root.join("README.md"),
"---\ntitle: Home\ntags: [alpha, beta]\n---\n# Home\nSee [[Daily]] and [Child](docs/child.md).\n[Spec](assets/spec.pdf)\n[Map](maps/idea.mindmap.json)\n[Sheet](office/report.xlsx)\n![Diagram](assets/diagram.png)\n",
)
.expect("write readme");
fs::write(
root.join("docs").join("child.md"),
"---\nmnote_id: child-page\ntitle: Child\n---\nChild body with alpha and office.xlsx\n",
)
.expect("write child");
fs::create_dir_all(root.join("maps")).expect("create maps");
fs::create_dir_all(root.join("office")).expect("create office");
fs::write(
root.join("maps").join("idea.mindmap.json"),
r#"{"title":"Idea Map","nodes":[]}"#,
)
.expect("write mindmap");
fs::create_dir_all(root.join("assets")).expect("create assets");
fs::write(root.join("assets").join("spec.pdf"), b"%PDF-1.4\n").expect("write pdf");
fs::write(root.join("office").join("report.xlsx"), b"office bytes").expect("write office");
let projection = query_local_search_index(
&root,
&format!("file://{}", root.display()),
"local-ws-test",
"alpha",
None,
10,
false,
false,
2026-06-01 09:29:12 +08:00
false,
)
.expect("projection");
let results = projection["results"].as_array().expect("results");
let home = results
.iter()
.find(|item| item["documentId"].as_str() == Some("local-md:README.md"))
.expect("home result");
2026-05-29 11:13:05 +08:00
assert!(home["tags"]
.as_array()
.unwrap()
.iter()
.any(|tag| tag.as_str() == Some("alpha")));
assert!(home["backlinks"]
.as_array()
.unwrap()
.iter()
.any(|link| link.as_str() == Some("Daily")));
assert!(home["backlinks"]
.as_array()
.unwrap()
.iter()
.any(|link| link.as_str() == Some("docs/child.md")));
assert!(home["resourceRefs"]
.as_array()
.unwrap()
.iter()
.any(|reference| reference.as_str() == Some("assets/spec.pdf")));
assert!(home["resourceRefs"]
.as_array()
.unwrap()
.iter()
.any(|reference| reference.as_str() == Some("maps/idea.mindmap.json")));
assert!(home["resourceRefs"]
.as_array()
.unwrap()
.iter()
.any(|reference| reference.as_str() == Some("office/report.xlsx")));
assert!(home["publicPath"]
.as_str()
.is_some_and(|path| path.starts_with(
"/documents/local-md:README.md?sourceKind=local_folder&rootUri=file%3A%2F%2F"
)));
let evidence_db = root.join(".mnote").join("index").join("evidence.sqlite");
let connection = Connection::open(&evidence_db).expect("open evidence sqlite");
let markdown_edge_count: i64 = connection
.query_row(
"SELECT COUNT(*) FROM evidence_edge WHERE edge_type = 'markdown_links_to'",
[],
|row| row.get(0),
)
.expect("markdown edge count");
let resource_edge_count: i64 = connection
.query_row(
"SELECT COUNT(*) FROM evidence_edge WHERE edge_type = 'resource_refers_to'",
[],
|row| row.get(0),
)
.expect("resource edge count");
let contains_edge_count: i64 = connection
.query_row(
"SELECT COUNT(*) FROM evidence_edge WHERE edge_type = 'document_contains_resource'",
[],
|row| row.get(0),
)
.expect("contains edge count");
assert!(markdown_edge_count >= 2);
assert!(resource_edge_count >= 4);
assert!(contains_edge_count >= 4);
2026-05-20 10:43:38 +08:00
// 含 mnote_id 的 Markdown 仍返回路径型 documentId,不应出现 local-mdid:
let child_search = query_local_search_index(
&root,
&format!("file://{}", root.display()),
"local-ws-test",
"office.xlsx",
None,
10,
false,
false,
2026-06-01 09:29:12 +08:00
false,
2026-05-20 10:43:38 +08:00
)
.expect("child search");
let child_result = child_search["results"]
.as_array()
.unwrap()
.iter()
.find(|item| item["path"].as_str() == Some("docs/child.md"))
.expect("child in search results");
assert_eq!(
child_result["documentId"].as_str(),
Some("local-md:docs~2Fchild.md")
);
assert_ne!(
child_result["documentId"].as_str(),
Some("local-mdid:child-page")
);
2026-05-29 11:13:05 +08:00
assert!(root
.join(".mnote")
.join("index")
.join("search-index.json")
.exists());
let mindmap_projection = query_local_search_index(
&root,
&format!("file://{}", root.display()),
"local-ws-test",
"idea.mindmap",
None,
10,
false,
false,
2026-06-01 09:29:12 +08:00
false,
)
.expect("mindmap projection");
2026-05-29 11:13:05 +08:00
assert!(mindmap_projection["results"]
.as_array()
.unwrap()
.iter()
.any(|item| item["resourceType"].as_str() == Some("mindmap")
&& item["path"].as_str() == Some("maps/idea.mindmap.json")
&& item["publicPath"]
.as_str()
.is_some_and(|path| path
2026-05-20 11:57:23 +08:00
.starts_with("/?treeView=filetree&sourceKind=local_folder&rootUri="))
2026-05-29 11:13:05 +08:00
&& item["publicPath"]
.as_str()
.is_some_and(|path| !path.starts_with("/tree?"))));
let office_projection = query_local_search_index(
&root,
&format!("file://{}", root.display()),
"local-ws-test",
"report.xlsx",
None,
10,
false,
false,
2026-06-01 09:29:12 +08:00
false,
)
.expect("office projection");
2026-05-29 11:13:05 +08:00
assert!(office_projection["results"]
.as_array()
.unwrap()
.iter()
.any(|item| item["resourceType"].as_str() == Some("office")
&& item["path"].as_str() == Some("office/report.xlsx")));
let pdf_projection = query_local_search_index(
&root,
&format!("file://{}", root.display()),
"local-ws-test",
"spec.pdf",
None,
10,
false,
false,
false,
)
.expect("pdf projection");
assert!(pdf_projection["results"]
.as_array()
.unwrap()
.iter()
.any(|item| item["resourceType"].as_str() == Some("pdf")
&& item["path"].as_str() == Some("assets/spec.pdf")));
let _ = fs::remove_dir_all(&root);
}
#[test]
fn local_index_settings_restricts_search_and_evidence_scope() {
let root = temp_root("mnote-local-index-settings-scope");
fs::create_dir_all(root.join("docs").join("included")).expect("create included");
fs::create_dir_all(root.join("docs").join("excluded")).expect("create excluded");
fs::write(
root.join("docs").join("included").join("keep.md"),
"# Keep\nScopedToken\n",
)
.expect("write included");
fs::write(
root.join("docs").join("excluded").join("skip.md"),
"# Skip\nExcludedToken\n",
)
.expect("write excluded");
let root_uri = format!("file://{}", root.display());
write_local_index_settings(
&root,
&[String::from("docs/included")],
None,
None,
None,
None,
)
.expect("settings");
refresh_local_search_index(&root, &root_uri, "local-ws-scope").expect("refresh");
let included = query_local_search_index(
&root,
&root_uri,
"local-ws-scope",
"ScopedToken",
None,
10,
false,
false,
false,
)
.expect("included query");
assert_eq!(included["results"].as_array().unwrap().len(), 1);
let excluded = query_local_search_index(
&root,
&root_uri,
"local-ws-scope",
"ExcludedToken",
None,
10,
false,
false,
false,
)
.expect("excluded query");
assert_eq!(excluded["results"].as_array().unwrap().len(), 0);
let evidence = query_evidence_sqlite_results(&root, "ScopedToken", None, 10)
.expect("evidence query")
.expect("evidence index");
assert_eq!(evidence.len(), 1);
let status = local_index_status(&root, &root_uri, "local-ws-scope").expect("status");
assert_eq!(
status["settings"]["includePaths"].as_array().unwrap()[0].as_str(),
Some("docs/included")
);
assert_eq!(status["documentCount"].as_u64(), Some(1));
let _ = fs::remove_dir_all(&root);
}
#[test]
fn local_index_settings_rejects_escape_and_internal_cache_scope() {
let root = temp_root("mnote-local-index-settings-escape");
let escape = write_local_index_settings(
&root,
&[String::from("../outside")],
None,
None,
None,
None,
)
.expect_err("escape should fail");
assert_eq!(escape.code(), "local_search_index_path_escape");
let internal = write_local_index_settings(
&root,
&[String::from(".mnote/index")],
None,
None,
None,
None,
)
.expect_err("internal cache should fail");
assert_eq!(internal.code(), "local_index_scope_reserved");
let _ = fs::remove_dir_all(&root);
}
#[test]
fn local_index_settings_defaults_and_schedule_are_persisted() {
let root = temp_root("mnote-local-index-settings-schedule");
let root_uri = format!("file://{}", root.display());
let initial_status =
local_index_status(&root, &root_uri, "local-ws-schedule").expect("initial status");
assert_eq!(
initial_status["settings"]["scheduleMode"].as_str(),
Some("daily")
);
assert_eq!(
initial_status["settings"]["scheduleTime"].as_str(),
Some("02:00")
);
assert_eq!(
initial_status["settings"]["runOnChange"].as_bool(),
Some(false)
);
let settings = write_local_index_settings(
&root,
&[String::from("docs")],
Some("once"),
Some("08:30"),
Some("2026-06-04"),
Some(true),
)
.expect("write schedule settings");
assert_eq!(settings.include_paths, vec![String::from("docs")]);
assert_eq!(settings.schedule_mode, "once");
assert_eq!(settings.schedule_time, "08:30");
assert_eq!(settings.schedule_date.as_deref(), Some("2026-06-04"));
assert!(settings.run_on_change);
let status = local_index_status(&root, &root_uri, "local-ws-schedule").expect("status");
assert_eq!(status["settings"]["scheduleMode"].as_str(), Some("once"));
assert_eq!(status["settings"]["scheduleTime"].as_str(), Some("08:30"));
assert_eq!(
status["settings"]["scheduleDate"].as_str(),
Some("2026-06-04")
);
assert_eq!(status["settings"]["runOnChange"].as_bool(), Some(true));
let _ = fs::remove_dir_all(&root);
}
#[test]
fn local_index_settings_rejects_invalid_schedule() {
let root = temp_root("mnote-local-index-settings-invalid-schedule");
let invalid_mode = write_local_index_settings(
&root,
&[String::from(".")],
Some("weekly"),
None,
None,
None,
)
.expect_err("invalid mode should fail");
assert_eq!(invalid_mode.code(), "local_index_schedule_mode_invalid");
let invalid_time = write_local_index_settings(
&root,
&[String::from(".")],
Some("daily"),
Some("24:00"),
None,
None,
)
.expect_err("invalid time should fail");
assert_eq!(invalid_time.code(), "local_index_schedule_time_invalid");
let invalid_date = write_local_index_settings(
&root,
&[String::from(".")],
Some("once"),
Some("02:00"),
Some("2026-13-01"),
None,
)
.expect_err("invalid date should fail");
assert_eq!(invalid_date.code(), "local_index_schedule_date_invalid");
let _ = fs::remove_dir_all(&root);
}
#[test]
fn local_index_schedule_due_respects_modes() {
let today = OffsetDateTime::now_local()
.unwrap_or_else(|_| OffsetDateTime::now_utc())
.date();
let yesterday = today - Duration::days(1);
let tomorrow = today + Duration::days(1);
let yesterday = format!(
"{:04}-{:02}-{:02}",
yesterday.year(),
u8::from(yesterday.month()),
yesterday.day()
);
let tomorrow = format!(
"{:04}-{:02}-{:02}",
tomorrow.year(),
u8::from(tomorrow.month()),
tomorrow.day()
);
let mut settings = default_local_index_settings();
settings.schedule_mode = "manual".to_string();
assert!(!local_index_schedule_is_due(&settings, 0));
settings.schedule_mode = "once".to_string();
settings.schedule_time = "00:00".to_string();
settings.schedule_date = Some(yesterday);
assert!(local_index_schedule_is_due(&settings, 0));
settings.schedule_date = Some(tomorrow);
assert!(!local_index_schedule_is_due(&settings, 0));
settings.schedule_mode = "daily".to_string();
settings.schedule_date = None;
settings.schedule_time = "00:00".to_string();
assert!(local_index_schedule_is_due(&settings, 0));
assert!(!local_index_schedule_is_due(&settings, now_ms()));
}
#[test]
fn multi_user_local_index_settings_share_one_cache_and_filter_results() {
let root = temp_root("mnote-local-index-multi-user");
fs::create_dir_all(root.join("docs").join("alice")).expect("create alice dir");
fs::create_dir_all(root.join("docs").join("bob")).expect("create bob dir");
fs::write(
root.join("docs").join("alice").join("keep.md"),
"# Alice\nAliceScopedToken\n",
)
.expect("write alice doc");
fs::write(
root.join("docs").join("bob").join("keep.md"),
"# Bob\nBobScopedToken\n",
)
.expect("write bob doc");
let root_uri = format!("file://{}", root.display());
let workspace_id = "local-ws-multi-user";
let store = SqliteControlPlaneStore::in_memory().expect("init control plane");
write_user_local_index_settings(
&store,
"alice",
workspace_id,
&root,
&[String::from("docs/alice")],
Some("manual"),
Some("02:00"),
None,
Some(false),
)
.expect("write alice settings");
write_user_local_index_settings(
&store,
"bob",
workspace_id,
&root,
&[String::from("docs/bob")],
Some("manual"),
Some("02:00"),
None,
Some(true),
)
.expect("write bob settings");
let sqlite_records = store
.list_user_ui_preferences_for_scope(
Some(workspace_id),
Some(LOCAL_INDEX_SOURCE_KIND),
LOCAL_INDEX_SETTINGS_SCOPE_KIND,
&local_index_scope_id(&root),
LOCAL_INDEX_SETTINGS_KEY,
)
.expect("read raw sqlite preference rows");
assert_eq!(sqlite_records.len(), 2);
let alice_sqlite_record = sqlite_records
.iter()
.find(|record| record.user_id == "alice")
.expect("alice sqlite preference row");
let alice_sqlite_value: serde_json::Value =
serde_json::from_str(&alice_sqlite_record.value_json).expect("alice sqlite json");
assert_eq!(
alice_sqlite_value["includePaths"],
serde_json::json!(["docs/alice"]),
"索引范围必须写入 SQLite user_ui_preferences.value_json"
);
let alice_settings =
read_user_local_index_settings(&store, "alice", workspace_id, &root).expect("alice");
let bob_settings =
read_user_local_index_settings(&store, "bob", workspace_id, &root).expect("bob");
let effective = effective_local_index_settings_for_root(&store, workspace_id, &root)
.expect("effective");
assert_eq!(
effective.include_paths,
vec![String::from("docs/alice"), String::from("docs/bob")]
);
assert!(effective.run_on_change);
refresh_local_search_index_with_settings(&root, &root_uri, workspace_id, &effective)
.expect("refresh shared index");
let alice_results = query_local_search_index_with_settings(
&root,
&root_uri,
workspace_id,
&effective,
&alice_settings,
"AliceScopedToken",
None,
10,
false,
false,
false,
)
.expect("alice query");
assert_eq!(alice_results["results"].as_array().map(Vec::len), Some(1));
let alice_hidden = query_local_search_index_with_settings(
&root,
&root_uri,
workspace_id,
&effective,
&alice_settings,
"BobScopedToken",
None,
10,
false,
false,
false,
)
.expect("alice hidden query");
assert_eq!(alice_hidden["results"].as_array().map(Vec::len), Some(0));
let bob_results = query_local_search_index_with_settings(
&root,
&root_uri,
workspace_id,
&effective,
&bob_settings,
"BobScopedToken",
None,
10,
false,
false,
false,
)
.expect("bob query");
assert_eq!(bob_results["results"].as_array().map(Vec::len), Some(1));
let status = local_index_status_with_settings(
&root,
&root_uri,
workspace_id,
&alice_settings,
&effective,
)
.expect("status");
assert_eq!(
status["settings"]["includePaths"].as_array().map(Vec::len),
Some(1)
);
assert_eq!(
status["effectiveSettings"]["includePaths"]
.as_array()
.map(Vec::len),
Some(2)
);
let _ = fs::remove_dir_all(&root);
}
#[test]
fn local_index_schedule_due_uses_any_user_schedule() {
let root = temp_root("mnote-local-index-schedule-any-user");
let workspace_id = "local-ws-schedule-any-user";
let store = SqliteControlPlaneStore::in_memory().expect("init control plane");
let today = OffsetDateTime::now_local()
.unwrap_or_else(|_| OffsetDateTime::now_utc())
.date();
let yesterday = today - Duration::days(1);
let yesterday = format!(
"{:04}-{:02}-{:02}",
yesterday.year(),
u8::from(yesterday.month()),
yesterday.day()
);
write_user_local_index_settings(
&store,
"alice",
workspace_id,
&root,
&[String::from(".")],
Some("manual"),
Some("02:00"),
None,
Some(false),
)
.expect("write alice settings");
write_user_local_index_settings(
&store,
"bob",
workspace_id,
&root,
&[String::from(".")],
Some("once"),
Some("00:00"),
Some(&yesterday),
Some(false),
)
.expect("write bob settings");
assert!(
local_index_any_schedule_due_for_root(&store, workspace_id, &root, 0)
.expect("schedule due"),
"任一用户到期就应该触发共享索引重建"
);
let _ = fs::remove_dir_all(&root);
}
#[test]
fn refresh_local_search_index_for_change_path_with_store_uses_sqlite_effective_settings() {
let root = temp_root("mnote-local-index-change-with-store");
fs::create_dir_all(root.join("docs").join("included")).expect("create included");
fs::create_dir_all(root.join("docs").join("excluded")).expect("create excluded");
fs::write(
root.join("docs").join("included").join("keep.md"),
"# Keep\nKeepScopedToken\n",
)
.expect("write included");
fs::write(
root.join("docs").join("excluded").join("skip.md"),
"# Skip\nSkipScopedToken\n",
)
.expect("write excluded");
let root_uri = format!("file://{}", root.display());
let workspace_id = "local-ws-change-with-store";
let store = SqliteControlPlaneStore::in_memory().expect("init control plane");
write_local_index_settings(
&root,
&[String::from("docs/excluded")],
Some("manual"),
Some("02:00"),
None,
Some(false),
)
.expect("write stale cache settings");
write_user_local_index_settings(
&store,
"alice",
workspace_id,
&root,
&[String::from("docs/included")],
Some("manual"),
Some("02:00"),
None,
Some(true),
)
.expect("write sqlite settings");
let refreshed = refresh_local_search_index_for_change_path_with_store(
&store,
&root,
&root_uri,
workspace_id,
"docs/included/keep.md",
)
.expect("refresh")
.expect("refresh result");
assert_eq!(refreshed["documentCount"].as_u64(), Some(1));
let keep_results = query_local_search_index_with_settings(
&root,
&root_uri,
workspace_id,
&effective_local_index_settings_for_root(&store, workspace_id, &root)
.expect("effective"),
&read_user_local_index_settings(&store, "alice", workspace_id, &root).expect("alice"),
"KeepScopedToken",
None,
10,
false,
false,
false,
)
.expect("keep query");
assert_eq!(keep_results["results"].as_array().map(Vec::len), Some(1));
let skip_results = query_local_search_index_with_settings(
&root,
&root_uri,
workspace_id,
&effective_local_index_settings_for_root(&store, workspace_id, &root)
.expect("effective"),
&read_user_local_index_settings(&store, "alice", workspace_id, &root).expect("alice"),
"SkipScopedToken",
None,
10,
false,
false,
false,
)
.expect("skip query");
assert_eq!(skip_results["results"].as_array().map(Vec::len), Some(0));
let _ = fs::remove_dir_all(&root);
}
#[test]
fn local_search_index_incrementally_updates_single_markdown_path() {
let root = temp_root("mnote-local-search-index-incremental");
let root_uri = format!("file://{}", root.display());
let workspace_id = "local-ws-incremental";
fs::write(root.join("README.md"), "# Home\nRoot body.\n").expect("write home");
fs::write(
root.join("docs").join("child.md"),
"---\ntitle: Child\n---\n# Child\nOriginal body.\n",
)
.expect("write child");
refresh_local_search_index(&root, &root_uri, workspace_id).expect("initial refresh");
fs::write(
root.join("docs").join("child.md"),
"---\ntitle: Child Updated\n---\n# Child Updated\nChangedToken body.\n",
)
.expect("update child");
refresh_local_search_index_for_path(&root, &root_uri, workspace_id, "docs/child.md")
.expect("incremental update");
let index = read_local_search_index(&root)
.expect("read index")
.expect("index exists");
2026-05-29 11:13:05 +08:00
assert!(index
.documents
.iter()
.any(|document| document.path == "README.md"));
let child = index
.documents
.iter()
.find(|document| document.path == "docs/child.md")
.expect("child document");
2026-05-25 23:34:03 +08:00
// 本地 Markdown 的树标题统一来自文件名;frontmatter/H1 只进入正文和索引文本。
assert_eq!(child.title, "child");
assert!(child.raw_text.contains("ChangedToken"));
let changed_evidence = query_evidence_sqlite_results(&root, "ChangedToken", None, 10)
.expect("changed evidence")
.expect("sqlite exists");
assert_eq!(changed_evidence.len(), 1);
assert_eq!(
changed_evidence[0].source.owner_document_path,
"docs/child.md"
);
let original_evidence = query_evidence_sqlite_results(&root, "Original body", None, 10)
.expect("original evidence")
.expect("sqlite exists");
assert!(original_evidence.is_empty());
fs::remove_file(root.join("docs").join("child.md")).expect("remove child");
refresh_local_search_index_for_path(&root, &root_uri, workspace_id, "docs/child.md")
.expect("incremental remove");
let index = read_local_search_index(&root)
.expect("read index")
.expect("index exists");
2026-05-29 11:13:05 +08:00
assert!(!index
.documents
.iter()
.any(|document| document.path == "docs/child.md"));
assert!(index
.documents
.iter()
.any(|document| document.path == "README.md"));
let removed_evidence = query_evidence_sqlite_results(&root, "ChangedToken", None, 10)
.expect("removed evidence")
.expect("sqlite exists");
assert!(removed_evidence.is_empty());
let root_evidence = query_evidence_sqlite_results(&root, "Root body", None, 10)
.expect("root evidence")
.expect("sqlite exists");
assert_eq!(root_evidence.len(), 1);
let _ = fs::remove_dir_all(&root);
}
2026-06-01 09:29:12 +08:00
#[test]
fn local_search_ocr_sidecar_requires_include_ocr_and_returns_owner_page() {
let root = temp_root("mnote-local-search-ocr");
let root_uri = format!("file://{}", root.display());
let workspace_id = "local-ws-ocr";
fs::write(root.join("docs").join("Page.md"), "# Page\n正文\n").expect("page");
fs::create_dir_all(root.join("docs").join("Page.assets")).expect("assets");
fs::write(
root.join("docs").join("Page.assets").join("photo.png"),
b"png",
)
.expect("photo");
fs::create_dir_all(root.join("docs").join("Page.ocr")).expect("ocr dir");
fs::write(
root.join("docs").join("Page.ocr").join("photo.png.ocr.md"),
"---\nmnote_ocr_version: 1\nprovider: mock\nmodel_version: vlm\nowner_document: ../Page.md\nsource_path: ./Page.assets/photo.png\nsource_root_relative_path: docs/Page.assets/photo.png\nsource_size: 3\nsource_mtime_ms: 1\nstatus: done\ncreated_at: 1\nupdated_at: 1\n---\n\nOCR-only-token 识别正文\n",
)
.expect("ocr markdown");
2026-06-01 10:07:42 +08:00
fs::write(
root.join("docs").join("Page.ocr").join("photo.png-704905.ocr.md"),
"---\nmnote_ocr_version: 1\nprovider: mock\nmodel_version: vlm\nowner_document: ../Page.md\nsource_path: ./Page.assets/photo.png\nsource_root_relative_path: docs/Page.assets/photo.png\nsource_size: 3\nsource_mtime_ms: 2\nstatus: done\ncreated_at: 2\nupdated_at: 2\n---\n\nOCR-hash-token 识别正文\n",
)
.expect("hashed ocr markdown");
2026-06-01 09:29:12 +08:00
fs::create_dir_all(root.join(".mnote")).expect("mnote dir");
fs::write(
root.join(".mnote").join("ocr-index.json"),
serde_json::to_string_pretty(&json!({
"version": 1,
"entries": {
"docs/Page.assets/photo.png": {
"jobId": "ocr_test",
"ownerDocumentId": "local-md:docs~2FPage.md",
"ownerDocumentPath": "docs/Page.md",
"sourceRootRelativePath": "docs/Page.assets/photo.png",
"ocrRootRelativePath": "docs/Page.ocr/photo.png.ocr.md",
"provider": "mock",
"modelVersion": "vlm",
"status": "done",
"sourceSize": 3,
"sourceMtimeMs": 1,
"createdAtMs": 1,
"updatedAtMs": 1,
"plainTextPreview": "OCR-only-token 识别正文"
}
}
}))
.expect("serialize index"),
)
.expect("ocr index");
let without_ocr = query_local_search_index(
&root,
&root_uri,
workspace_id,
"OCR-only-token",
None,
10,
false,
false,
false,
)
.expect("without ocr");
assert_eq!(without_ocr["results"].as_array().map(Vec::len), Some(0));
2026-06-01 10:07:42 +08:00
let without_hashed_ocr = query_local_search_index(
&root,
&root_uri,
workspace_id,
"OCR-hash-token",
None,
10,
false,
false,
false,
)
.expect("without hashed ocr");
assert_eq!(
without_hashed_ocr["results"].as_array().map(Vec::len),
Some(0)
);
2026-06-01 09:29:12 +08:00
let with_ocr = query_local_search_index(
&root,
&root_uri,
workspace_id,
"OCR-only-token",
None,
10,
false,
false,
true,
)
.expect("with ocr");
let result = with_ocr["results"]
.as_array()
.and_then(|items| items.first())
.expect("ocr result");
assert_eq!(
result["documentId"].as_str(),
Some("local-md:docs~2FPage.md")
);
assert_eq!(result["hasOcr"].as_bool(), Some(true));
assert_eq!(
result["ocrEvidence"]["sourceRootRelativePath"].as_str(),
Some("docs/Page.assets/photo.png")
);
assert_eq!(
result["ocrEvidence"]["ocrRootRelativePath"].as_str(),
Some("docs/Page.ocr/photo.png.ocr.md")
);
let index = read_local_search_index(&root)
.expect("read search index")
.expect("search index");
assert!(!index
.documents
.iter()
.any(|document| document.path == "docs/Page.ocr/photo.png.ocr.md"));
2026-06-01 10:07:42 +08:00
assert!(!index
.documents
.iter()
.any(|document| document.path == "docs/Page.ocr/photo.png-704905.ocr.md"));
refresh_local_search_index_for_path(
&root,
&root_uri,
workspace_id,
"docs/Page.ocr/photo.png-704905.ocr.md",
)
.expect("refresh hashed ocr sidecar");
let refreshed_index = read_local_search_index(&root)
.expect("read refreshed search index")
.expect("refreshed search index");
assert!(!refreshed_index
.documents
.iter()
.any(|document| document.path == "docs/Page.ocr/photo.png-704905.ocr.md"));
2026-06-01 09:29:12 +08:00
let _ = fs::remove_dir_all(&root);
}
#[test]
fn local_search_query_reads_existing_index_without_rebuilding() {
let root = temp_root("mnote-local-search-query-cache");
let root_uri = format!("file://{}", root.display());
let workspace_id = "local-ws-query-cache";
let child_path = root.join("docs").join("child.md");
fs::write(
&child_path,
"---\ntitle: Child\n---\n# Child\nOriginalToken body.\n",
)
.expect("write child");
let first_projection = query_local_search_index(
&root,
&root_uri,
workspace_id,
"OriginalToken",
None,
10,
false,
false,
2026-06-01 09:29:12 +08:00
false,
)
.expect("first query");
assert_eq!(
first_projection["results"].as_array().map(Vec::len),
Some(1)
);
fs::write(
&child_path,
"---\ntitle: Child\n---\n# Child\nUnindexedToken body.\n",
)
.expect("update child without refresh");
let stale_projection = query_local_search_index(
&root,
&root_uri,
workspace_id,
"UnindexedToken",
None,
10,
false,
false,
2026-06-01 09:29:12 +08:00
false,
)
.expect("query existing index");
assert_eq!(
stale_projection["results"].as_array().map(Vec::len),
Some(0)
);
refresh_local_search_index_for_path(&root, &root_uri, workspace_id, "docs/child.md")
.expect("incremental refresh");
let refreshed_projection = query_local_search_index(
&root,
&root_uri,
workspace_id,
"UnindexedToken",
None,
10,
false,
false,
2026-06-01 09:29:12 +08:00
false,
)
.expect("query refreshed index");
assert_eq!(
refreshed_projection["results"].as_array().map(Vec::len),
Some(1)
);
let _ = fs::remove_dir_all(&root);
}
#[test]
fn evidence_sqlite_query_returns_locator_results() {
let root = temp_root("mnote-evidence-sqlite-query");
let root_uri = format!("file://{}", root.display());
let workspace_id = "local-ws-evidence-sqlite";
fs::write(
root.join("README.md"),
"# Home\nIntro body.\n## Evidence Section\nEvidenceToken root body.\nAsk @Reasonix for citation.\n",
)
.expect("write home");
fs::write(
root.join("docs").join("child.md"),
"# Child\nEvidenceToken child body.\n",
)
.expect("write child");
refresh_local_search_index(&root, &root_uri, workspace_id).expect("refresh");
let results = query_evidence_sqlite_results(&root, "EvidenceToken", None, 10)
.expect("sqlite query")
.expect("sqlite exists");
assert!(results.len() >= 2);
assert!(results
.iter()
.any(|result| result.source.owner_document_path == "README.md"));
assert!(results
.iter()
.any(|result| result.source.owner_document_path == "docs/child.md"));
assert!(results
.iter()
.all(|result| result.source.schema == "mnote.evidence_locator.v1"));
let home_result = results
.iter()
.find(|result| result.source.owner_document_path == "README.md")
.expect("home evidence result");
assert_eq!(
home_result.source.section_path,
vec!["Home".to_string(), "Evidence Section".to_string()]
);
assert_eq!(
home_result
.source
.line_range
.as_ref()
.map(|range| range.start),
Some(4)
);
assert_eq!(
home_result.source.block_id.as_deref(),
Some("local-md:README.md#line4")
);
let scoped = query_evidence_sqlite_results(
&root,
"EvidenceToken",
Some("local-md:docs~2Fchild.md"),
10,
)
.expect("scoped sqlite query")
.expect("sqlite exists");
assert_eq!(scoped.len(), 1);
assert_eq!(scoped[0].source.owner_document_path, "docs/child.md");
let connection =
Connection::open(root.join(".mnote").join("index").join("evidence.sqlite"))
.expect("open evidence sqlite");
let section_count: i64 = connection
.query_row("SELECT COUNT(*) FROM evidence_section", [], |row| {
row.get(0)
})
.expect("section count");
assert!(section_count >= 3);
let mention_source_block: String = connection
.query_row(
"SELECT source_block_id FROM evidence_edge WHERE edge_type = 'mentions' AND to_id = 'entity:Reasonix'",
[],
|row| row.get(0),
)
.expect("mention edge");
assert_eq!(mention_source_block, "local-md:README.md#line5");
let mention_locator: String = connection
.query_row(
"SELECT locator_json FROM evidence_block WHERE block_id = ?1",
rusqlite::params![mention_source_block],
|row| row.get(0),
)
.expect("mention source locator");
assert!(mention_locator.contains("mnote.evidence_locator.v1"));
let graph_results = query_evidence_graph_results(&root, "Reasonix", None, 10)
.expect("graph query")
.expect("sqlite exists");
let mention_result = graph_results
.iter()
.find(|result| result.evidence_id.contains("mentions"))
.expect("mention graph result");
assert!(mention_result.quote.contains("mentions:"));
assert_eq!(
mention_result.source.block_id.as_deref(),
Some("local-md:README.md#line5")
);
connection
.execute("DELETE FROM evidence_edge", [])
.expect("delete graph projection");
let search_after_graph_delete =
query_evidence_sqlite_results(&root, "EvidenceToken", None, 10)
.expect("sqlite query after graph delete")
.expect("sqlite exists");
assert!(!search_after_graph_delete.is_empty());
let _ = fs::remove_dir_all(&root);
}
#[test]
fn evidence_sqlite_read_context_returns_anchor_block() {
let root = temp_root("mnote-evidence-sqlite-read-context");
let root_uri = format!("file://{}", root.display());
let workspace_id = "local-ws-evidence-read";
fs::write(
root.join("README.md"),
"# Home\nReadContextToken root body.\n",
)
.expect("write home");
refresh_local_search_index(&root, &root_uri, workspace_id).expect("refresh");
let results = query_evidence_sqlite_results(&root, "ReadContextToken", None, 10)
.expect("sqlite query")
.expect("sqlite exists");
let locator = results
.first()
.map(|item| item.source.clone())
.expect("locator");
let context = read_evidence_sqlite_context(&root, &locator, 1, 1)
.expect("read context")
.expect("sqlite exists");
assert_eq!(context.len(), 2);
assert!(context
.iter()
.all(|item| item.source.owner_document_path == "README.md"));
assert!(context
.iter()
.any(|item| item.quote.contains("ReadContextToken")));
let _ = fs::remove_dir_all(&root);
}
#[test]
#[cfg(unix)]
fn evidence_index_parses_resource_body_with_liteparse_sidecar() {
use std::os::unix::fs::PermissionsExt;
let _guard = env_lock().lock().expect("env lock");
let root = temp_root("mnote-evidence-resource-liteparse");
let root_uri = format!("file://{}", root.display());
let workspace_id = "local-ws-evidence-resource";
fs::create_dir_all(root.join("docs").join("Page.assets")).expect("assets");
fs::write(
root.join("docs").join("Page.md"),
"# Page\n[Spec](./Page.assets/spec.pdf)\n",
)
.expect("write owner");
fs::write(
root.join("docs").join("Page.assets").join("spec.pdf"),
b"%PDF-1.4",
)
.expect("write pdf");
let lit = root.join("fake-lit");
fs::write(
&lit,
r#"#!/usr/bin/env bash
cat <<'JSON'
{
"version": "liteparse-index-test",
"pages": [{
"page": 2,
"width": 612,
"height": 792,
"blocks": [
{"id": "b1", "type": "heading", "level": 1, "text": "配方要求", "bbox": [72, 90, 180, 120]},
{"id": "b2", "type": "text", "text": "ResourceBodyToken 来自 PDF 正文", "bbox": [72, 130, 330, 160]}
]
}]
}
JSON
"#,
)
.expect("fake lit");
let mut permissions = fs::metadata(&lit).expect("fake lit metadata").permissions();
permissions.set_mode(0o755);
fs::set_permissions(&lit, permissions).expect("chmod fake lit");
let old_bin = std::env::var("MNOTE_LITEPARSE_BIN").ok();
std::env::set_var("MNOTE_LITEPARSE_BIN", &lit);
refresh_local_search_index(&root, &root_uri, workspace_id).expect("refresh");
if let Some(value) = old_bin {
std::env::set_var("MNOTE_LITEPARSE_BIN", value);
} else {
std::env::remove_var("MNOTE_LITEPARSE_BIN");
}
let results = query_evidence_sqlite_results(&root, "ResourceBodyToken", None, 10)
.expect("sqlite query")
.expect("sqlite exists");
let hit = results
.iter()
.find(|result| result.quote.contains("ResourceBodyToken"))
.expect("parsed resource body hit");
assert_eq!(
hit.source.owner_document_id, "local-md:docs~2FPage.md",
"资源正文证据应归属引用它的 owner Markdown"
);
assert_eq!(
hit.source.resource_path.as_deref(),
Some("docs/Page.assets/spec.pdf")
);
assert_eq!(hit.source.page, Some(2));
assert!(hit.source.bbox.is_some());
assert_eq!(
hit.source.source_map_path.as_deref(),
Some("docs/Page.ocr/spec.pdf.source-map.json")
);
assert!(root
.join("docs")
.join("Page.ocr")
.join("spec.pdf.parse.md")
.exists());
assert!(root
.join("docs")
.join("Page.ocr")
.join("spec.pdf.source-map.json")
.exists());
let _ = fs::remove_dir_all(&root);
}
}