feat: advance local-first conflict and search indexing
- 补齐本地 markdown 冲突处理与合并写回路径\n- 增加本地搜索索引路由、刷新与 browser smoke\n- 同步更新 current-priority checklist 的阶段进度
This commit is contained in:
lix-2026
@@ -0,0 +1,615 @@
|
||||
use crate::error::WebError;
|
||||
use crate::routes::local_markdown_parser::{
|
||||
parse_markdown_attachment_link, parse_markdown_page, split_frontmatter,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{json, Value};
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
const LOCAL_SEARCH_INDEX_VERSION: u32 = 1;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct LocalSearchIndex {
|
||||
version: u32,
|
||||
built_at: u128,
|
||||
root_uri: String,
|
||||
workspace_id: String,
|
||||
documents: Vec<LocalSearchDocument>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct LocalSearchDocument {
|
||||
document_id: String,
|
||||
title: String,
|
||||
path: String,
|
||||
raw_text: String,
|
||||
tags: Vec<String>,
|
||||
backlinks: Vec<String>,
|
||||
resource_refs: Vec<String>,
|
||||
updated_at: u128,
|
||||
}
|
||||
|
||||
pub(crate) fn query_local_search_index(
|
||||
root_path: &Path,
|
||||
root_uri: &str,
|
||||
workspace_id: &str,
|
||||
query: &str,
|
||||
page_id: Option<&str>,
|
||||
limit: u32,
|
||||
title_only: bool,
|
||||
exact: bool,
|
||||
) -> Result<Value, WebError> {
|
||||
let index = rebuild_local_search_index(root_path, root_uri, workspace_id)?;
|
||||
let normalized_query = normalize_search_text(query);
|
||||
let page_id = page_id.map(str::trim).filter(|value| !value.is_empty());
|
||||
let recent_changes = local_recent_changes_projection(&index.documents, root_uri);
|
||||
let mut results = Vec::new();
|
||||
for document in index.documents.iter() {
|
||||
if let Some(page_id) = page_id {
|
||||
if document.document_id != page_id {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if !local_search_document_matches(document, &normalized_query, title_only, exact) {
|
||||
continue;
|
||||
}
|
||||
results.push(local_search_document_projection(
|
||||
document,
|
||||
root_uri,
|
||||
&normalized_query,
|
||||
));
|
||||
if results.len() >= limit.max(1) as usize {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Ok(json!({
|
||||
"enqueueAssetIds": [],
|
||||
"projectionOwner": "rust-kernel",
|
||||
"sourceKind": "local_folder",
|
||||
"index": {
|
||||
"version": index.version,
|
||||
"rootUri": index.root_uri,
|
||||
"workspaceId": index.workspace_id,
|
||||
"builtAt": index.built_at,
|
||||
"documentCount": index.documents.len()
|
||||
},
|
||||
"recentChanges": recent_changes,
|
||||
"results": results
|
||||
}))
|
||||
}
|
||||
|
||||
pub(crate) fn refresh_local_search_index(
|
||||
root_path: &Path,
|
||||
root_uri: &str,
|
||||
workspace_id: &str,
|
||||
) -> Result<Value, WebError> {
|
||||
let index = rebuild_local_search_index(root_path, root_uri, workspace_id)?;
|
||||
Ok(json!({
|
||||
"version": index.version,
|
||||
"rootUri": index.root_uri,
|
||||
"workspaceId": index.workspace_id,
|
||||
"builtAt": index.built_at,
|
||||
"documentCount": index.documents.len()
|
||||
}))
|
||||
}
|
||||
|
||||
pub(crate) fn query_local_backlinks(
|
||||
root_path: &Path,
|
||||
root_uri: &str,
|
||||
workspace_id: &str,
|
||||
document_id: &str,
|
||||
) -> Result<Value, WebError> {
|
||||
let index = rebuild_local_search_index(root_path, root_uri, workspace_id)?;
|
||||
let Some(target) = index
|
||||
.documents
|
||||
.iter()
|
||||
.find(|document| document.document_id == document_id)
|
||||
else {
|
||||
return Ok(json!({
|
||||
"documentId": document_id,
|
||||
"rootUri": root_uri,
|
||||
"workspaceId": workspace_id,
|
||||
"backlinks": []
|
||||
}));
|
||||
};
|
||||
let target_keys = normalized_document_targets(target);
|
||||
let backlinks = index
|
||||
.documents
|
||||
.iter()
|
||||
.filter(|document| document.document_id != target.document_id)
|
||||
.filter_map(|document| {
|
||||
let matched_targets = document
|
||||
.backlinks
|
||||
.iter()
|
||||
.filter(|link| target_keys.contains(&normalize_search_text(link)))
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
if matched_targets.is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some(json!({
|
||||
"documentId": document.document_id,
|
||||
"title": document.title,
|
||||
"path": document.path,
|
||||
"resourceType": "markdown",
|
||||
"matchedTargets": matched_targets,
|
||||
"snippet": search_snippet(document, &normalize_search_text(&target.title))
|
||||
}))
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
Ok(json!({
|
||||
"documentId": target.document_id,
|
||||
"title": target.title,
|
||||
"path": target.path,
|
||||
"rootUri": root_uri,
|
||||
"workspaceId": workspace_id,
|
||||
"backlinks": backlinks
|
||||
}))
|
||||
}
|
||||
|
||||
pub(crate) fn query_local_tags(
|
||||
root_path: &Path,
|
||||
root_uri: &str,
|
||||
workspace_id: &str,
|
||||
) -> Result<Value, WebError> {
|
||||
let index = rebuild_local_search_index(root_path, root_uri, workspace_id)?;
|
||||
let mut tags = std::collections::BTreeMap::<String, Vec<&LocalSearchDocument>>::new();
|
||||
for document in &index.documents {
|
||||
for tag in &document.tags {
|
||||
tags.entry(tag.clone()).or_default().push(document);
|
||||
}
|
||||
}
|
||||
Ok(json!({
|
||||
"rootUri": root_uri,
|
||||
"workspaceId": workspace_id,
|
||||
"tags": tags
|
||||
.into_iter()
|
||||
.map(|(tag, documents)| {
|
||||
json!({
|
||||
"tag": tag,
|
||||
"count": documents.len(),
|
||||
"documents": documents
|
||||
.into_iter()
|
||||
.map(|document| {
|
||||
json!({
|
||||
"documentId": document.document_id,
|
||||
"title": document.title,
|
||||
"path": document.path,
|
||||
"resourceType": "markdown"
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
}))
|
||||
}
|
||||
|
||||
fn rebuild_local_search_index(
|
||||
root_path: &Path,
|
||||
root_uri: &str,
|
||||
workspace_id: &str,
|
||||
) -> Result<LocalSearchIndex, WebError> {
|
||||
let mut documents = Vec::new();
|
||||
collect_markdown_documents(root_path, root_path, &mut documents)?;
|
||||
documents.sort_by(|left, right| left.path.cmp(&right.path));
|
||||
let index = LocalSearchIndex {
|
||||
version: LOCAL_SEARCH_INDEX_VERSION,
|
||||
built_at: now_ms(),
|
||||
root_uri: root_uri.to_string(),
|
||||
workspace_id: workspace_id.to_string(),
|
||||
documents,
|
||||
};
|
||||
write_local_search_index(root_path, &index)?;
|
||||
Ok(index)
|
||||
}
|
||||
|
||||
fn collect_markdown_documents(
|
||||
root_path: &Path,
|
||||
current: &Path,
|
||||
documents: &mut Vec<LocalSearchDocument>,
|
||||
) -> Result<(), WebError> {
|
||||
let entries = match fs::read_dir(current) {
|
||||
Ok(entries) => entries,
|
||||
Err(error) => {
|
||||
return Err(WebError::bad_request_code(
|
||||
"local_search_index_read_dir_failed",
|
||||
format!("无法读取本地搜索索引目录 {}: {error}", current.display()),
|
||||
));
|
||||
}
|
||||
};
|
||||
for entry in entries {
|
||||
let entry = entry.map_err(|error| {
|
||||
WebError::bad_request_code(
|
||||
"local_search_index_read_entry_failed",
|
||||
format!("无法读取本地搜索索引条目: {error}"),
|
||||
)
|
||||
})?;
|
||||
let path = entry.path();
|
||||
let file_name = path
|
||||
.file_name()
|
||||
.and_then(|value| value.to_str())
|
||||
.unwrap_or_default();
|
||||
if file_name == ".mnote" {
|
||||
continue;
|
||||
}
|
||||
let file_type = entry.file_type().map_err(|error| {
|
||||
WebError::bad_request_code(
|
||||
"local_search_index_stat_failed",
|
||||
format!("无法读取本地搜索索引文件状态 {}: {error}", path.display()),
|
||||
)
|
||||
})?;
|
||||
if file_type.is_dir() {
|
||||
collect_markdown_documents(root_path, &path, documents)?;
|
||||
continue;
|
||||
}
|
||||
if !file_type.is_file() || !is_markdown_path(&path) {
|
||||
continue;
|
||||
}
|
||||
documents.push(index_markdown_file(root_path, &path)?);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn index_markdown_file(root_path: &Path, path: &Path) -> Result<LocalSearchDocument, WebError> {
|
||||
let markdown = fs::read_to_string(path).map_err(|error| {
|
||||
WebError::bad_request_code(
|
||||
"local_search_index_read_markdown_failed",
|
||||
format!("无法读取本地 Markdown 索引文件 {}: {error}", path.display()),
|
||||
)
|
||||
})?;
|
||||
let file_name = path
|
||||
.file_name()
|
||||
.and_then(|value| value.to_str())
|
||||
.unwrap_or("");
|
||||
let parsed = parse_markdown_page(&markdown, file_name);
|
||||
let relative_path = path
|
||||
.strip_prefix(root_path)
|
||||
.unwrap_or(path)
|
||||
.to_string_lossy()
|
||||
.replace('\\', "/");
|
||||
let document_id = parsed
|
||||
.mnote_id
|
||||
.as_deref()
|
||||
.map(|id| format!("local-mdid:{id}"))
|
||||
.unwrap_or_else(|| format!("local-md:{}", relative_path.replace('/', "~2F")));
|
||||
let metadata = fs::metadata(path).map_err(|error| {
|
||||
WebError::bad_request_code(
|
||||
"local_search_index_stat_failed",
|
||||
format!(
|
||||
"无法读取本地 Markdown 索引文件状态 {}: {error}",
|
||||
path.display()
|
||||
),
|
||||
)
|
||||
})?;
|
||||
Ok(LocalSearchDocument {
|
||||
document_id,
|
||||
title: parsed.title,
|
||||
path: relative_path,
|
||||
raw_text: parsed.body.clone(),
|
||||
tags: extract_tags(&markdown),
|
||||
backlinks: extract_backlinks(&parsed.body),
|
||||
resource_refs: extract_resource_refs(&parsed.body),
|
||||
updated_at: metadata
|
||||
.modified()
|
||||
.ok()
|
||||
.and_then(|value| value.duration_since(UNIX_EPOCH).ok())
|
||||
.map(|value| value.as_millis())
|
||||
.unwrap_or_default(),
|
||||
})
|
||||
}
|
||||
|
||||
fn write_local_search_index(root_path: &Path, index: &LocalSearchIndex) -> Result<(), WebError> {
|
||||
let index_dir = root_path.join(".mnote").join("index");
|
||||
fs::create_dir_all(&index_dir).map_err(|error| {
|
||||
WebError::bad_request_code(
|
||||
"local_search_index_create_failed",
|
||||
format!("无法创建本地搜索索引目录 {}: {error}", index_dir.display()),
|
||||
)
|
||||
})?;
|
||||
let index_path = index_dir.join("search-index.json");
|
||||
let payload = serde_json::to_string_pretty(index)
|
||||
.map_err(|error| WebError::internal(format!("本地搜索索引序列化失败: {error}")))?;
|
||||
fs::write(&index_path, format!("{payload}\n")).map_err(|error| {
|
||||
WebError::bad_request_code(
|
||||
"local_search_index_write_failed",
|
||||
format!("无法写入本地搜索索引 {}: {error}", index_path.display()),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn local_search_document_matches(
|
||||
document: &LocalSearchDocument,
|
||||
query: &str,
|
||||
title_only: bool,
|
||||
exact: bool,
|
||||
) -> bool {
|
||||
if query.is_empty() {
|
||||
return false;
|
||||
}
|
||||
let haystack = if title_only {
|
||||
normalize_search_text(&document.title)
|
||||
} else {
|
||||
normalize_search_text(&format!(
|
||||
"{}\n{}\n{}\n{}",
|
||||
document.title,
|
||||
document.raw_text,
|
||||
document.tags.join(" "),
|
||||
document.resource_refs.join(" ")
|
||||
))
|
||||
};
|
||||
if exact {
|
||||
haystack == query
|
||||
} else {
|
||||
haystack.contains(query)
|
||||
}
|
||||
}
|
||||
|
||||
fn local_search_document_projection(
|
||||
document: &LocalSearchDocument,
|
||||
root_uri: &str,
|
||||
query: &str,
|
||||
) -> Value {
|
||||
json!({
|
||||
"id": document.document_id,
|
||||
"documentId": document.document_id,
|
||||
"title": document.title,
|
||||
"path": document.path,
|
||||
"resourceType": "markdown",
|
||||
"sourceKind": "local_folder",
|
||||
"rootUri": root_uri,
|
||||
"snippet": search_snippet(document, query),
|
||||
"tags": document.tags,
|
||||
"backlinks": document.backlinks,
|
||||
"resourceRefs": document.resource_refs,
|
||||
"updatedAt": document.updated_at,
|
||||
"publicPath": format!("/documents/{}?sourceKind=local_folder&rootUri={}", document.document_id, root_uri)
|
||||
})
|
||||
}
|
||||
|
||||
fn local_recent_changes_projection(documents: &[LocalSearchDocument], root_uri: &str) -> Value {
|
||||
let mut sorted = documents.iter().collect::<Vec<_>>();
|
||||
sorted.sort_by(|left, right| right.updated_at.cmp(&left.updated_at));
|
||||
Value::Array(
|
||||
sorted
|
||||
.into_iter()
|
||||
.take(20)
|
||||
.map(|document| {
|
||||
json!({
|
||||
"id": document.document_id,
|
||||
"documentId": document.document_id,
|
||||
"title": document.title,
|
||||
"path": document.path,
|
||||
"resourceType": "markdown",
|
||||
"sourceKind": "local_folder",
|
||||
"rootUri": root_uri,
|
||||
"updatedAt": document.updated_at
|
||||
})
|
||||
})
|
||||
.collect(),
|
||||
)
|
||||
}
|
||||
|
||||
fn normalized_document_targets(
|
||||
document: &LocalSearchDocument,
|
||||
) -> std::collections::BTreeSet<String> {
|
||||
let mut values = std::collections::BTreeSet::new();
|
||||
values.insert(normalize_search_text(&document.document_id));
|
||||
values.insert(normalize_search_text(&document.path));
|
||||
values.insert(normalize_search_text(&document.title));
|
||||
if let Some(encoded) = document.document_id.strip_prefix("local-md:") {
|
||||
values.insert(normalize_search_text(&encoded.replace("~2F", "/")));
|
||||
}
|
||||
values
|
||||
}
|
||||
|
||||
fn extract_tags(markdown: &str) -> Vec<String> {
|
||||
let Some(frontmatter) = split_frontmatter(markdown).0 else {
|
||||
return Vec::new();
|
||||
};
|
||||
for line in frontmatter.lines() {
|
||||
let trimmed = line.trim();
|
||||
let Some(value) = trimmed
|
||||
.strip_prefix("tags:")
|
||||
.or_else(|| trimmed.strip_prefix("tag:"))
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
return value
|
||||
.trim()
|
||||
.trim_matches(['[', ']'])
|
||||
.split([',', ' '])
|
||||
.map(|item| item.trim().trim_matches(['"', '\'']))
|
||||
.filter(|item| !item.is_empty())
|
||||
.map(ToOwned::to_owned)
|
||||
.collect();
|
||||
}
|
||||
Vec::new()
|
||||
}
|
||||
|
||||
fn extract_backlinks(markdown: &str) -> Vec<String> {
|
||||
let mut values = extract_markdown_links(markdown)
|
||||
.into_iter()
|
||||
.filter(|target| is_markdown_reference(target))
|
||||
.collect::<Vec<_>>();
|
||||
let mut rest = markdown;
|
||||
while let Some(start) = rest.find("[[") {
|
||||
let after_start = &rest[start + 2..];
|
||||
let Some(end) = after_start.find("]]") else {
|
||||
break;
|
||||
};
|
||||
let link = after_start[..end].trim();
|
||||
if !link.is_empty() {
|
||||
values.push(link.to_string());
|
||||
}
|
||||
rest = &after_start[end + 2..];
|
||||
}
|
||||
values.sort();
|
||||
values.dedup();
|
||||
values
|
||||
}
|
||||
|
||||
fn extract_resource_refs(markdown: &str) -> Vec<String> {
|
||||
let mut values = markdown
|
||||
.lines()
|
||||
.filter_map(|line| parse_markdown_attachment_link(line.trim()))
|
||||
.map(|(_, target)| target)
|
||||
.chain(
|
||||
extract_markdown_links(markdown)
|
||||
.into_iter()
|
||||
.filter(|target| {
|
||||
!is_markdown_reference(target) && is_local_resource_reference(target)
|
||||
}),
|
||||
)
|
||||
.collect::<Vec<_>>();
|
||||
values.sort();
|
||||
values.dedup();
|
||||
values
|
||||
}
|
||||
|
||||
fn extract_markdown_links(markdown: &str) -> Vec<String> {
|
||||
let mut links = Vec::new();
|
||||
let mut rest = markdown;
|
||||
while let Some(label_end) = rest.find("](") {
|
||||
let after = &rest[label_end + 2..];
|
||||
let Some(target_end) = after.find(')') else {
|
||||
break;
|
||||
};
|
||||
let target = after[..target_end].trim();
|
||||
if !target.is_empty() {
|
||||
links.push(target.to_string());
|
||||
}
|
||||
rest = &after[target_end + 1..];
|
||||
}
|
||||
links
|
||||
}
|
||||
|
||||
fn is_markdown_reference(target: &str) -> bool {
|
||||
let lower = target.to_lowercase();
|
||||
lower.ends_with(".md") || lower.ends_with(".markdown") || lower.starts_with("local-md:")
|
||||
}
|
||||
|
||||
fn is_local_resource_reference(target: &str) -> bool {
|
||||
let lower = target.to_lowercase();
|
||||
!lower.starts_with("http://")
|
||||
&& !lower.starts_with("https://")
|
||||
&& !lower.starts_with("mailto:")
|
||||
&& !lower.starts_with('#')
|
||||
&& Path::new(target).extension().is_some()
|
||||
}
|
||||
|
||||
fn search_snippet(document: &LocalSearchDocument, query: &str) -> String {
|
||||
for line in document.raw_text.lines() {
|
||||
let trimmed = line.trim();
|
||||
if trimmed.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if normalize_search_text(trimmed).contains(query) {
|
||||
return trimmed.chars().take(180).collect();
|
||||
}
|
||||
}
|
||||
document.raw_text.chars().take(180).collect()
|
||||
}
|
||||
|
||||
fn is_markdown_path(path: &Path) -> bool {
|
||||
path.extension()
|
||||
.and_then(|value| value.to_str())
|
||||
.map(|value| value.eq_ignore_ascii_case("md") || value.eq_ignore_ascii_case("markdown"))
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn normalize_search_text(value: &str) -> String {
|
||||
value.trim().to_lowercase()
|
||||
}
|
||||
|
||||
fn now_ms() -> u128 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map(|value| value.as_millis())
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::path::PathBuf;
|
||||
|
||||
fn temp_root(name: &str) -> PathBuf {
|
||||
let root = std::env::temp_dir().join(format!("{name}-{}", std::process::id()));
|
||||
let _ = fs::remove_dir_all(&root);
|
||||
fs::create_dir_all(root.join("docs")).expect("create root");
|
||||
root
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn local_search_index_extracts_backlinks_resource_refs_and_tags() {
|
||||
let root = temp_root("mnote-local-search-index");
|
||||
fs::write(
|
||||
root.join("README.md"),
|
||||
"---\ntitle: Home\ntags: [alpha, beta]\n---\n# Home\nSee [[Daily]] and [Child](docs/child.md).\n[Spec](assets/spec.pdf)\n[Map](maps/idea.mindmap.json)\n[Sheet](office/report.xlsx)\n\n",
|
||||
)
|
||||
.expect("write readme");
|
||||
fs::write(
|
||||
root.join("docs").join("child.md"),
|
||||
"---\nmnote_id: child-page\ntitle: Child\n---\nChild body with alpha and office.xlsx\n",
|
||||
)
|
||||
.expect("write child");
|
||||
|
||||
let projection = query_local_search_index(
|
||||
&root,
|
||||
&format!("file://{}", root.display()),
|
||||
"local-ws-test",
|
||||
"alpha",
|
||||
None,
|
||||
10,
|
||||
false,
|
||||
false,
|
||||
)
|
||||
.expect("projection");
|
||||
let results = projection["results"].as_array().expect("results");
|
||||
let home = results
|
||||
.iter()
|
||||
.find(|item| item["documentId"].as_str() == Some("local-md:README.md"))
|
||||
.expect("home result");
|
||||
assert!(home["tags"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.any(|tag| tag.as_str() == Some("alpha")));
|
||||
assert!(home["backlinks"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.any(|link| link.as_str() == Some("Daily")));
|
||||
assert!(home["backlinks"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.any(|link| link.as_str() == Some("docs/child.md")));
|
||||
assert!(home["resourceRefs"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.any(|reference| reference.as_str() == Some("assets/spec.pdf")));
|
||||
assert!(home["resourceRefs"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.any(|reference| reference.as_str() == Some("maps/idea.mindmap.json")));
|
||||
assert!(home["resourceRefs"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.any(|reference| reference.as_str() == Some("office/report.xlsx")));
|
||||
assert!(root
|
||||
.join(".mnote")
|
||||
.join("index")
|
||||
.join("search-index.json")
|
||||
.exists());
|
||||
let _ = fs::remove_dir_all(&root);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user