lix/mnote
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve local evidence search and AI capabilities

Browse Source
This commit is contained in:
lix-2026
2026-06-05 23:00:53 +08:00
parent a1dcfc9f76
commit 4dbd9a978b
52 changed files with 6415 additions and 527 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rust/crates/mnote-web/src/routes/local_folder_source.rs
+198 -6
View File
@@ -295,6 +295,7 @@ struct LocalFolderRow {
capabilities: Vec<String>,
workspace_id: String,
root_source_uri: String,
index_status: Option<String>,
}
#[derive(Debug, Clone)]
@@ -303,6 +304,40 @@ struct LocalFolderScanResult {
watch_revision: LocalFolderWatchRevision,
}
#[derive(Debug, Clone, Default)]
struct LocalFileTreeIndexState {
indexed_paths: BTreeSet<String>,
failed_paths: BTreeSet<String>,
}
impl LocalFileTreeIndexState {
fn load(root: &Path) -> Self {
local_search_index::local_evidence_source_statuses(root)
.map(|statuses| Self {
indexed_paths: statuses.indexed_paths,
failed_paths: statuses.failed_paths,
})
.unwrap_or_default()
}
fn status_for_entry(&self, entry: &LocalFolderEntry) -> Option<String> {
if entry.is_dir || entry.is_symlink {
return None;
}
let path = entry.relative_path.trim();
if path.is_empty() {
return None;
}
if self.failed_paths.contains(path) {
return Some("failed".to_string());
}
if self.indexed_paths.contains(path) {
return Some("indexed".to_string());
}
None
}
}
#[derive(Debug, Clone)]
pub(crate) struct LocalUploadFile {
name: String,
@@ -2717,6 +2752,7 @@ fn load_local_folder_file_tree_scope_snapshot(
let workspace_id = local_workspace_id(&canonical_root);
let root_source_uri = file_uri_for_path(&canonical_root);
let metadata = load_local_folder_metadata(&canonical_root)?;
let index_state = LocalFileTreeIndexState::load(&canonical_root);
let parent_relative_path = parent_relative_path
.map(str::trim)
.filter(|value| !value.is_empty() && *value != ".")
@@ -2746,6 +2782,7 @@ fn load_local_folder_file_tree_scope_snapshot(
&root_source_uri,
&workspace_id,
&metadata,
&index_state,
)?;
append_file_tree_reveal_rows(
&canonical_root,
@@ -2754,6 +2791,7 @@ fn load_local_folder_file_tree_scope_snapshot(
&root_source_uri,
&workspace_id,
&metadata,
&index_state,
&mut scan_result.rows,
)?;
@@ -6648,6 +6686,7 @@ fn scan_directory(
root_source_uri: &str,
workspace_id: &str,
metadata: &LocalFolderMetadata,
index_state: &LocalFileTreeIndexState,
rows: &mut Vec<LocalFolderRow>,
) -> Result<(), WebError> {
let mut entries = read_sorted_entries(directory, root)?;
@@ -6698,6 +6737,7 @@ fn scan_directory(
capabilities: local_entry_capabilities(&entry),
workspace_id: workspace_id.to_string(),
root_source_uri: root_source_uri.to_string(),
index_status: index_state.status_for_entry(&entry),
});
if entry.is_dir && !entry.is_symlink && max_depth.map_or(true, |limit| depth < limit) {
scan_directory(
@@ -6709,6 +6749,7 @@ fn scan_directory(
root_source_uri,
workspace_id,
metadata,
index_state,
rows,
)?;
}
@@ -6725,6 +6766,7 @@ fn scan_directory_shallow_with_revision(
root_source_uri: &str,
workspace_id: &str,
metadata: &LocalFolderMetadata,
index_state: &LocalFileTreeIndexState,
) -> Result<LocalFolderScanResult, WebError> {
let mut entries = read_sorted_entries(directory, root)?;
let parent_key = file_order_parent_key_for_directory(root, directory)?;
@@ -6797,6 +6839,7 @@ fn scan_directory_shallow_with_revision(
capabilities: local_entry_capabilities(entry),
workspace_id: workspace_id.to_string(),
root_source_uri: root_source_uri.to_string(),
index_status: index_state.status_for_entry(entry),
});
}
@@ -6867,6 +6910,7 @@ fn append_file_tree_reveal_rows(
root_source_uri: &str,
workspace_id: &str,
metadata: &LocalFolderMetadata,
index_state: &LocalFileTreeIndexState,
rows: &mut Vec<LocalFolderRow>,
) -> Result<(), WebError> {
let Some(reveal_relative_path) = reveal_relative_path
@@ -6924,6 +6968,7 @@ fn append_file_tree_reveal_rows(
root_source_uri,
workspace_id,
metadata,
index_state,
&mut scoped_rows,
)?;
for row in scoped_rows {
@@ -7124,9 +7169,25 @@ fn should_ignore_entry(relative_path: &str, file_name: &str) -> bool {
}
relative_path == ".mnote/trash"
|| relative_path.starts_with(".mnote/trash/")
|| is_local_index_artifact_entry(relative_path)
|| is_local_ocr_intermediate_entry(relative_path, file_name)
}
fn is_local_index_artifact_entry(relative_path: &str) -> bool {
let normalized = relative_path.trim().trim_matches('/').replace('\\', "/");
if normalized.is_empty() {
return false;
}
let segments = normalized
.split('/')
.filter(|segment| !segment.trim().is_empty())
.collect::<Vec<_>>();
segments
.windows(2)
.any(|window| window[0] == ".mnote" && window[1] == "index")
|| segments.iter().any(|segment| segment.ends_with(".ocr"))
}
fn is_local_ocr_intermediate_entry(relative_path: &str, file_name: &str) -> bool {
let normalized = relative_path.trim().trim_matches('/').replace('\\', "/");
if normalized.is_empty() {
@@ -7435,6 +7496,7 @@ fn scan_markdown_page_tree(
capabilities: local_entry_capabilities(&entry),
workspace_id: workspace_id.to_string(),
root_source_uri: root_source_uri.to_string(),
index_status: None,
});
directory_rows.extend(child_rows);
contains_markdown = true;
@@ -7483,6 +7545,7 @@ fn scan_markdown_page_tree(
capabilities: local_entry_capabilities(&entry),
workspace_id: workspace_id.to_string(),
root_source_uri: root_source_uri.to_string(),
index_status: None,
});
}
directory_rows.extend(child_rows);
@@ -7510,6 +7573,7 @@ fn scan_markdown_page_tree(
capabilities: local_entry_capabilities(&entry),
workspace_id: workspace_id.to_string(),
root_source_uri: root_source_uri.to_string(),
index_status: None,
});
continue;
}
@@ -7546,6 +7610,7 @@ fn scan_markdown_page_tree(
capabilities: local_entry_capabilities(&entry),
workspace_id: workspace_id.to_string(),
root_source_uri: root_source_uri.to_string(),
index_status: None,
});
contains_markdown = true;
}
@@ -7691,6 +7756,10 @@ fn local_folder_row_to_projection_item(row: &LocalFolderRow) -> Value {
item["assetId"] = Value::String(asset_id.clone());
item["resourceMeta"]["assetId"] = Value::String(asset_id);
}
if let Some(index_status) = row.index_status.as_deref() {
item["indexStatus"] = Value::String(index_status.to_string());
item["resourceMeta"]["indexStatus"] = Value::String(index_status.to_string());
}
// Phase A1: 为 File/Page/Resource tree 输出统一 workspacePath
let object_kind = match row.row_kind.as_str() {
@@ -11648,6 +11717,15 @@ fn main() {}
.expect("write md");
let root_uri = format!("file://{}", root.display());
crate::routes::local_search_index::write_local_index_settings(
&root,
&[String::from(".")],
None,
None,
None,
Some(true),
)
.expect("settings");
crate::routes::local_search_index::refresh_local_search_index(
&root,
&root_uri,
@@ -12369,6 +12447,15 @@ fn main() {}
std::fs::write(root.join("README.md"), "# Before\nold-token\n").expect("write md");
let root_uri = format!("file://{}", root.display());
crate::routes::local_search_index::write_local_index_settings(
&root,
&[String::from(".")],
None,
None,
None,
Some(true),
)
.expect("settings");
crate::routes::local_search_index::refresh_local_search_index(
&root,
&root_uri,
@@ -13474,7 +13561,7 @@ fn main() {}
}
#[test]
fn local_ocr_sidecar_is_filetree_resource_not_page_tree_document() {
fn local_ocr_sidecar_artifacts_are_hidden_from_filetree_and_page_tree() {
let root = temp_root("mnote-local-ocr-sidecar-page-tree");
init_workspace(&root);
let root_uri = format!("file://{}", root.display());
@@ -13515,15 +13602,22 @@ fn main() {}
)
.expect("ocr image asset");
let docs_file_tree =
load_local_folder_file_tree_children_snapshot(&root_uri, "docs").expect("file tree");
let docs_file_items = docs_file_tree.projection["items"]
.as_array()
.expect("file items");
assert!(!docs_file_items
.iter()
.any(|item| item["title"].as_str() == Some("Page.ocr")));
let file_tree = load_local_folder_file_tree_children_snapshot(&root_uri, "docs/Page.ocr")
.expect("file tree");
.expect("direct hidden sidecar file tree");
let file_items = file_tree.projection["items"]
.as_array()
.expect("file items");
assert!(file_items
.iter()
.any(|item| item["title"].as_str() == Some("photo.png.ocr.md")));
for hidden_title in [
"photo.png.ocr.md",
"layout.json",
"abc_content_list.json",
"abc_model.json",
@@ -13534,7 +13628,7 @@ fn main() {}
!file_items
.iter()
.any(|item| item["title"].as_str() == Some(hidden_title)),
"OCR 中间产物不应出现在 FileTree: {hidden_title}"
"OCR / 索引产物不应出现在 FileTree: {hidden_title}"
);
}
@@ -13551,6 +13645,104 @@ fn main() {}
let _ = std::fs::remove_dir_all(&root);
}
#[test]
fn local_file_tree_marks_indexed_and_failed_source_files() {
let root = temp_root("mnote-local-filetree-index-status");
init_workspace(&root);
std::fs::write(root.join("ok.pdf"), b"%PDF-1.4\nok").expect("ok pdf");
std::fs::write(root.join("failed.pdf"), b"%PDF-1.4\nfailed").expect("failed pdf");
std::fs::write(root.join("draft.md"), "# Draft\n").expect("draft");
let index_dir = root.join(".mnote").join("index");
std::fs::create_dir_all(&index_dir).expect("index dir");
let evidence_path = index_dir.join("evidence.sqlite");
let connection = rusqlite::Connection::open(&evidence_path).expect("evidence sqlite");
connection
.execute_batch(
r#"
CREATE TABLE evidence_resource(
resource_id TEXT PRIMARY KEY,
owner_document_id TEXT NOT NULL,
owner_document_path TEXT NOT NULL,
source_root_relative_path TEXT NOT NULL,
provider TEXT NOT NULL,
source_hash TEXT NOT NULL,
artifact_root_relative_path TEXT NOT NULL,
source_map_root_relative_path TEXT NOT NULL,
updated_at_ms INTEGER NOT NULL
);
"#,
)
.expect("evidence schema");
connection
.execute(
"INSERT INTO evidence_resource(resource_id, owner_document_id, owner_document_path, source_root_relative_path, provider, source_hash, artifact_root_relative_path, source_map_root_relative_path, updated_at_ms) VALUES(?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)",
rusqlite::params![
"local-resource:ok.pdf#parse",
"local-resource:ok.pdf",
"ok.pdf",
"ok.pdf",
"liteparse",
"hash",
"ok.ocr/ok.pdf.parse.md",
"ok.ocr/ok.pdf.source-map.json",
1_i64,
],
)
.expect("insert indexed evidence");
connection
.execute(
"INSERT INTO evidence_resource(resource_id, owner_document_id, owner_document_path, source_root_relative_path, provider, source_hash, artifact_root_relative_path, source_map_root_relative_path, updated_at_ms) VALUES(?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)",
rusqlite::params![
"local-md:draft.md",
"local-md:draft.md",
"draft.md",
"draft.md",
"markdown",
"hash",
"draft.md",
"draft.md.source-map.json",
1_i64,
],
)
.expect("insert markdown evidence");
let root_uri = format!("file://{}", root.display());
let search_index = json!({
"version": 1,
"builtAt": 1,
"rootUri": root_uri,
"workspaceId": "local-filetree-index-status",
"indexedPaths": ["."],
"documents": [],
"resources": [
{"resourceId": "local-resource:ok.pdf", "resourceType": "pdf", "title": "ok", "path": "ok.pdf", "updatedAt": 1},
{"resourceId": "local-resource:failed.pdf", "resourceType": "pdf", "title": "failed", "path": "failed.pdf", "updatedAt": 1}
]
});
std::fs::write(
index_dir.join("search-index.json"),
format!("{}\n", serde_json::to_string_pretty(&search_index).unwrap()),
)
.expect("search index");
let file_tree = load_local_folder_file_tree_snapshot(&format!("file://{}", root.display()))
.expect("file tree");
let items = file_tree.projection["items"].as_array().expect("items");
let status_for = |path: &str| {
items
.iter()
.find(|item| {
item["resourceMeta"]["extra"]["source"]["relativePath"].as_str() == Some(path)
})
.and_then(|item| item["indexStatus"].as_str())
};
assert_eq!(status_for("ok.pdf"), Some("indexed"));
assert_eq!(status_for("failed.pdf"), Some("failed"));
assert_eq!(status_for("draft.md"), None);
let _ = std::fs::remove_dir_all(&root);
}
#[test]
fn local_rename_markdown_page_renames_nested_bundle() {
let root = temp_root("mnote-local-rename-nested-bundle");