lix/mnote
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Optimize local folder file tree lazy loading

Browse Source
This commit is contained in:
lix-2026
2026-05-26 12:30:01 +08:00
parent 8cd8b44db3
commit 4744757883
10 changed files with 728 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rust/crates/mnote-web/src/routes/gateway.rs
+23 -5
View File
@@ -67,6 +67,7 @@ pub(crate) struct RootEntryQuery {
workspace_id: Option<String>,
source_kind: Option<String>,
root_uri: Option<String>,
tree_view: Option<String>,
restore_focus_row_id: Option<String>,
}
@@ -294,6 +295,11 @@ pub async fn root_entry(
.filter(|value| !value.is_empty())
.is_none()
&& requested_page_id.is_none();
let requests_filetree_first = query
.tree_view
.as_deref()
.map(str::trim)
.is_some_and(|value| value == "filetree");
let (
workspace_id,
workspace_projection,
@@ -312,7 +318,14 @@ pub async fn root_entry(
WebError::bad_request_code("local_folder_root_required", "缺少本地文件夹 rootUri")
})?;
ensure_local_workspace_read_access_with_state(&state, &context, root_uri)?;
let snapshot = load_local_folder_page_tree_snapshot(root_uri)?;
let snapshot = if requests_filetree_first {
crate::routes::local_folder_source::load_local_folder_file_tree_snapshot_with_reveal(
root_uri,
requested_page_id.as_deref(),
)?
} else {
load_local_folder_page_tree_snapshot(root_uri)?
};
let workspace_id = snapshot
.dataset
.get("workspace")
@@ -339,8 +352,11 @@ pub async fn root_entry(
.first()
.map(|item| item.id.as_str()),
);
let sidebar_tree_html =
render_local_sidebar_tree_html(root_uri, selected_active_page_id.as_deref())?;
let sidebar_tree_html = if requests_filetree_first {
String::new()
} else {
render_local_sidebar_tree_html(root_uri, selected_active_page_id.as_deref())?
};
let restore_focus_row_id = query
.restore_focus_row_id
.as_deref()
@@ -2755,8 +2771,9 @@ mod tests {
let html = String::from_utf8(body.to_vec()).expect("utf8");
assert!(html.contains(r#"data-mnote-shell="workspace""#));
assert!(html.contains("local_folder"));
assert!(html.contains("Local Root"));
assert!(html.contains("README.md"));
assert!(html.contains(r#"data-row-id="local:folder:docs""#));
assert!(!html.contains(r#"data-row-id="local:markdown:docs/child.md""#));
assert!(html.contains(r#"data-row-id="local:asset:plain.txt""#));
assert!(!html.contains("legacy_next_compat_disabled"));
assert!(!html.contains(r#"href="/tree"#));
@@ -2798,7 +2815,8 @@ mod tests {
.expect("body");
let html = String::from_utf8(body.to_vec()).expect("utf8");
assert!(html.contains(r#"data-mnote-shell="workspace""#));
assert!(html.contains(r#"data-row-id="local:asset:attachments/report-a.pdf""#));
assert!(html.contains(r#"data-row-id="local:folder:attachments""#));
assert!(!html.contains(r#"data-row-id="local:asset:attachments/report-a.pdf""#));
assert!(html.contains(r#"data-testid="mnote-document-workspace""#));
assert!(html.contains(r#"data-mnote-resource-tab-host="true""#));
assert!(html.contains("__MNOTE_DOCUMENT_PANES_BOOTSTRAP__"));
rust/crates/mnote-web/src/routes/kernel.rs
+74 -2
View File
@@ -2,7 +2,8 @@ use crate::app::AppState;
use crate::context::RequestContext;
use crate::error::WebError;
use crate::routes::local_folder_source::{
ensure_local_workspace_read_access_with_state, load_local_folder_file_tree_snapshot,
ensure_local_workspace_read_access_with_state, load_local_folder_file_tree_children_snapshot,
load_local_folder_file_tree_snapshot, load_local_folder_file_tree_snapshot_with_reveal,
load_local_folder_page_tree_snapshot,
};
use crate::routes::query_support::resolve_effective_workspace_id;
@@ -28,6 +29,7 @@ pub struct KernelProjectionQuery {
pub max_results: Option<usize>,
pub source_kind: Option<String>,
pub root_uri: Option<String>,
pub parent_relative_path: Option<String>,
}
#[derive(Debug, Deserialize)]
@@ -88,7 +90,27 @@ async fn project_projection(
ensure_local_workspace_read_access_with_state(&state, &context, root_uri)
.map_err(|error| error.with_context(&context))?;
let snapshot = if projection == KernelProjectionKind::FileTree {
load_local_folder_file_tree_snapshot(root_uri)?
if let Some(parent_relative_path) = query
.parent_relative_path
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty())
{
load_local_folder_file_tree_children_snapshot(root_uri, parent_relative_path)?
} else if query
.root_node_id
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty())
.is_some()
{
load_local_folder_file_tree_snapshot_with_reveal(
root_uri,
query.root_node_id.as_deref(),
)?
} else {
load_local_folder_file_tree_snapshot(root_uri)?
}
} else {
load_local_folder_page_tree_snapshot(root_uri)?
};
@@ -371,6 +393,56 @@ mod tests {
let _ = std::fs::remove_dir_all(&root);
}
#[tokio::test]
async fn local_folder_file_projection_can_load_direct_children_by_parent_path() {
let root = std::env::temp_dir().join(format!(
"mnote-local-kernel-children-projection-{}",
std::process::id()
));
let _ = std::fs::remove_dir_all(&root);
std::fs::create_dir_all(root.join("docs").join("nested")).expect("create nested dirs");
std::fs::write(root.join("docs").join("README.md"), "# README\n").expect("write readme");
std::fs::write(root.join("docs").join("nested").join("deep.md"), "# Deep\n")
.expect("write deep");
let root_uri = format!("file://{}", root.display());
initialize_local_workspace_for_actor("dev-user", &root_uri).expect("init local workspace");
let response = app()
.oneshot(
Request::builder()
.uri(format!(
"/api/tree/projections/file?workspaceId=local-ws:dev-user:my-space&sourceKind=local_folder&rootUri={root_uri}&parentRelativePath=docs"
))
.header("x-mnote-actor-id", "dev-user")
.header("x-mnote-actor-type", "user")
.body(Body::empty())
.expect("request"),
)
.await
.expect("response");
assert_eq!(response.status(), StatusCode::OK);
let body = to_bytes(response.into_body(), usize::MAX)
.await
.expect("body");
let payload: Value = serde_json::from_slice(&body).expect("json");
let items = payload["result"]["items"].as_array().expect("items");
let titles = items
.iter()
.filter_map(|item| item["title"].as_str())
.collect::<Vec<_>>();
assert_eq!(payload["result"]["parentRelativePath"], "docs");
assert!(titles.contains(&"README.md"));
assert!(titles.contains(&"nested"));
assert!(!titles.contains(&"deep.md"));
assert!(items
.iter()
.all(|item| item["parentNodeId"].as_str() == Some("local:node:docs")));
let _ = std::fs::remove_dir_all(&root);
}
#[tokio::test]
async fn tree_projection_routes_keep_ok_response_shape() {
let response = app()
rust/crates/mnote-web/src/routes/local_folder_source.rs
+272 -14
View File
@@ -2554,6 +2554,31 @@ fn local_workspace_manifest_json(manifest: &LocalWorkspaceManifest) -> Value {
pub fn load_local_folder_file_tree_snapshot(
root_uri: &str,
) -> Result<ProjectionSnapshot, WebError> {
load_local_folder_file_tree_scope_snapshot(root_uri, None, None)
}
pub fn load_local_folder_file_tree_snapshot_with_reveal(
root_uri: &str,
reveal_document_id: Option<&str>,
) -> Result<ProjectionSnapshot, WebError> {
let reveal_relative_path = reveal_document_id
.and_then(local_markdown_relative_path_from_document_id)
.map(|path| path.replace('\\', "/"));
load_local_folder_file_tree_scope_snapshot(root_uri, None, reveal_relative_path.as_deref())
}
pub fn load_local_folder_file_tree_children_snapshot(
root_uri: &str,
parent_relative_path: &str,
) -> Result<ProjectionSnapshot, WebError> {
load_local_folder_file_tree_scope_snapshot(root_uri, Some(parent_relative_path), None)
}
fn load_local_folder_file_tree_scope_snapshot(
root_uri: &str,
parent_relative_path: Option<&str>,
reveal_relative_path: Option<&str>,
) -> Result<ProjectionSnapshot, WebError> {
let root_path = parse_file_root_uri(root_uri)?;
let canonical_root = root_path.canonicalize().map_err(|error| {
@@ -2572,23 +2597,56 @@ pub fn load_local_folder_file_tree_snapshot(
let workspace_id = local_workspace_id(&canonical_root);
let root_source_uri = file_uri_for_path(&canonical_root);
let metadata = load_local_folder_metadata(&canonical_root)?;
let parent_relative_path = parent_relative_path
.map(str::trim)
.filter(|value| !value.is_empty() && *value != ".")
.unwrap_or("");
let scan_root = if parent_relative_path.is_empty() {
canonical_root.clone()
} else {
resolve_metadata_relative_path(&canonical_root, parent_relative_path)?
};
if !scan_root.is_dir() {
return Err(WebError::bad_request_code(
"local_folder_parent_not_directory",
"FileTree 懒加载 parentRelativePath 必须指向目录",
));
}
let parent_node_id = if parent_relative_path.is_empty() {
None
} else {
Some(local_node_id(parent_relative_path))
};
let depth = local_folder_relative_depth(parent_relative_path);
let mut rows = Vec::new();
scan_directory(
&canonical_root,
&canonical_root,
None,
0,
&scan_root,
parent_node_id,
depth,
Some(depth),
&root_source_uri,
&workspace_id,
&metadata,
&mut rows,
)?;
if parent_relative_path.is_empty() {
append_file_tree_reveal_rows(
&canonical_root,
reveal_relative_path,
&root_source_uri,
&workspace_id,
&metadata,
&mut rows,
)?;
}
let items = rows
.iter()
.map(local_folder_row_to_projection_item)
.collect::<Vec<_>>();
let watch_revision = local_folder_watch_revision_for_root(&canonical_root, &root_source_uri)?;
let watch_revision =
local_folder_watch_revision_for_directory(&canonical_root, &scan_root, &root_source_uri)?;
let dataset = json!({
"workspace": {
"id": &workspace_id,
@@ -2607,6 +2665,8 @@ pub fn load_local_folder_file_tree_snapshot(
"projection": "file_tree",
"sourceKind": "local_folder",
"rootUri": root_source_uri,
"parentRelativePath": parent_relative_path,
"lazy": true,
"watchRevision": watch_revision,
"items": items,
});
@@ -6044,6 +6104,7 @@ fn scan_directory(
directory: &Path,
parent_node_id: Option<String>,
depth: u32,
max_depth: Option<u32>,
root_source_uri: &str,
workspace_id: &str,
metadata: &LocalFolderMetadata,
@@ -6052,7 +6113,6 @@ fn scan_directory(
let mut entries = read_sorted_entries(directory, root)?;
let parent_key = file_order_parent_key_for_directory(root, directory)?;
sort_entries_with_file_order(&mut entries, metadata, &parent_key);
let entry_count = entries.len();
for (position, entry) in entries.into_iter().enumerate() {
let node_id = local_node_id(if entry.relative_path.is_empty() {
"."
@@ -6060,7 +6120,11 @@ fn scan_directory(
&entry.relative_path
});
let child_count = if entry.is_dir && !entry.is_symlink {
count_visible_children(&entry.path, root)?
if has_visible_child(&entry.path, root)? {
1
} else {
0
}
} else {
0
};
@@ -6088,7 +6152,7 @@ fn scan_directory(
source_uri: file_uri_for_path(&entry.path),
child_count,
expandable: entry.is_dir && child_count > 0,
expanded_by_default: depth < 1 && entry.is_dir && entry_count <= 80,
expanded_by_default: false,
document_id: if let Some(asset) = uploaded_asset {
Some(asset.document_id.clone())
} else if is_markdown_file(&entry.file_name) {
@@ -6100,12 +6164,13 @@ fn scan_directory(
workspace_id: workspace_id.to_string(),
root_source_uri: root_source_uri.to_string(),
});
if entry.is_dir && !entry.is_symlink {
if entry.is_dir && !entry.is_symlink && max_depth.map_or(true, |limit| depth < limit) {
scan_directory(
root,
&entry.path,
Some(node_id),
depth + 1,
max_depth,
root_source_uri,
workspace_id,
metadata,
@@ -6117,6 +6182,91 @@ fn scan_directory(
Ok(())
}
fn local_markdown_relative_path_from_document_id(document_id: &str) -> Option<String> {
let encoded = document_id.trim().strip_prefix("local-md:")?;
decode_local_id_segment(encoded).ok()
}
fn ancestor_directories_for_relative_path(relative_path: &str) -> Vec<String> {
let normalized = relative_path.trim().trim_matches('/');
if normalized.is_empty() || normalized == "." {
return Vec::new();
}
let mut ancestors = Vec::new();
let mut current = Path::new(normalized).parent();
while let Some(parent) = current {
let value = parent
.components()
.map(|component| component.as_os_str().to_string_lossy().to_string())
.collect::<Vec<_>>()
.join("/");
if value.is_empty() || value == "." {
break;
}
ancestors.push(value);
current = parent.parent();
}
ancestors.reverse();
ancestors
}
fn append_file_tree_reveal_rows(
root: &Path,
reveal_relative_path: Option<&str>,
root_source_uri: &str,
workspace_id: &str,
metadata: &LocalFolderMetadata,
rows: &mut Vec<LocalFolderRow>,
) -> Result<(), WebError> {
let Some(reveal_relative_path) = reveal_relative_path
.map(str::trim)
.filter(|value| !value.is_empty())
else {
return Ok(());
};
let ancestors = ancestor_directories_for_relative_path(reveal_relative_path);
if ancestors.is_empty() {
return Ok(());
}
let ancestor_set = ancestors.iter().cloned().collect::<BTreeSet<_>>();
let mut seen = rows
.iter()
.map(|row| row.row_id.clone())
.collect::<BTreeSet<_>>();
for ancestor in &ancestors {
let directory = resolve_metadata_relative_path(root, ancestor)?;
if !directory.is_dir() {
continue;
}
let depth = local_folder_relative_depth(ancestor);
let mut scoped_rows = Vec::new();
scan_directory(
root,
&directory,
Some(local_node_id(ancestor)),
depth,
Some(depth),
root_source_uri,
workspace_id,
metadata,
&mut scoped_rows,
)?;
for row in scoped_rows {
if seen.insert(row.row_id.clone()) {
rows.push(row);
}
}
}
for row in rows.iter_mut() {
if ancestor_set.contains(&row.relative_path) {
row.expanded_by_default = true;
}
}
Ok(())
}
fn read_sorted_entries(directory: &Path, root: &Path) -> Result<Vec<LocalFolderEntry>, WebError> {
let read_dir = fs::read_dir(directory).map_err(|error| {
WebError::bad_request_code(
@@ -6160,8 +6310,28 @@ fn read_sorted_entries(directory: &Path, root: &Path) -> Result<Vec<LocalFolderE
Ok(entries)
}
fn count_visible_children(directory: &Path, root: &Path) -> Result<u32, WebError> {
Ok(read_sorted_entries(directory, root)?.len() as u32)
fn has_visible_child(directory: &Path, root: &Path) -> Result<bool, WebError> {
let read_dir = fs::read_dir(directory).map_err(|error| {
WebError::bad_request_code(
"local_folder_scan_failed",
format!("无法读取本地目录 {}: {error}", directory.display()),
)
})?;
for entry in read_dir {
let entry = entry.map_err(|error| {
WebError::bad_request_code(
"local_folder_scan_failed",
format!("读取目录项失败: {error}"),
)
})?;
let path = entry.path();
let file_name = entry.file_name().to_string_lossy().to_string();
let relative_path = normalize_relative_path(root, &path)?;
if !should_ignore_entry(&relative_path, &file_name) {
return Ok(true);
}
}
Ok(false)
}
fn local_entry_capabilities(entry: &LocalFolderEntry) -> Vec<String> {
@@ -6192,6 +6362,17 @@ fn normalize_relative_path(root: &Path, path: &Path) -> Result<String, WebError>
.join("/"))
}
fn local_folder_relative_depth(relative_path: &str) -> u32 {
let trimmed = relative_path.trim().trim_matches('/');
if trimmed.is_empty() || trimmed == "." {
return 0;
}
trimmed
.split('/')
.filter(|segment| !segment.trim().is_empty())
.count() as u32
}
fn resolve_metadata_relative_path(root: &Path, relative_path: &str) -> Result<PathBuf, WebError> {
let relative = Path::new(relative_path);
if relative.is_absolute()
@@ -6884,6 +7065,44 @@ fn local_folder_watch_revision_for_root(
})
}
fn local_folder_watch_revision_for_directory(
root: &Path,
directory: &Path,
root_source_uri: &str,
) -> Result<LocalFolderWatchRevision, WebError> {
let mut hasher = DefaultHasher::new();
let mut entry_count = 0usize;
let mut latest_modified_ms = 0u128;
for entry in read_sorted_entries(directory, root)? {
entry.relative_path.hash(&mut hasher);
entry.file_name.hash(&mut hasher);
entry.is_dir.hash(&mut hasher);
entry.is_symlink.hash(&mut hasher);
entry.is_readonly.hash(&mut hasher);
if let Ok(meta) = fs::symlink_metadata(&entry.path) {
meta.len().hash(&mut hasher);
if let Ok(modified) = meta.modified() {
if let Ok(delta) = modified.duration_since(UNIX_EPOCH) {
let modified_ms = delta.as_millis();
modified_ms.hash(&mut hasher);
if modified_ms > latest_modified_ms {
latest_modified_ms = modified_ms;
}
}
}
}
entry_count += 1;
}
Ok(LocalFolderWatchRevision {
root_uri: root_source_uri.to_string(),
revision: format!("{:016x}", hasher.finish()),
entry_count,
latest_modified_ms,
})
}
fn file_uri_for_path(path: &Path) -> String {
format!("file://{}", path.display())
}
@@ -8244,7 +8463,8 @@ mod tests {
ensure_local_workspace_write_access_with_state, execute_local_tree_command,
execute_local_tree_command_with_sort, get_local_access_policy, get_share_grants,
get_share_links, get_user_access_policy, initialize_local_page_id,
initialize_local_workspace_for_actor, load_local_folder_file_tree_snapshot,
initialize_local_workspace_for_actor, load_local_folder_file_tree_children_snapshot,
load_local_folder_file_tree_snapshot, load_local_folder_file_tree_snapshot_with_reveal,
load_local_folder_page_tree_snapshot, local_folder_watch_revision,
local_markdown_path_page_id, local_resource_write_editor_blocks, local_workspace_id,
open_local_file, read_local_resource, record_shared_cache, record_sync_pending_change,
@@ -10285,6 +10505,38 @@ fn main() {}
let _ = std::fs::remove_dir_all(&root);
}
#[test]
fn local_file_tree_snapshot_loads_root_level_only() {
let root = temp_root("mnote-local-file-tree-root-only");
init_workspace(&root);
std::fs::create_dir_all(root.join("docs").join("nested")).expect("create nested dirs");
std::fs::write(root.join("docs").join("README.md"), "# README\n").expect("write readme");
std::fs::write(root.join("docs").join("nested").join("deep.md"), "# Deep\n")
.expect("write deep");
std::fs::write(root.join("root.md"), "# Root\n").expect("write root");
let root_uri = format!("file://{}", root.display());
let snapshot = load_local_folder_file_tree_snapshot(&root_uri).expect("load file tree");
let items = snapshot.projection["items"].as_array().expect("items");
let titles = items
.iter()
.filter_map(|item| item["title"].as_str())
.collect::<Vec<_>>();
assert!(titles.contains(&"docs"));
assert!(titles.contains(&"root.md"));
assert!(!titles.contains(&"README.md"));
assert!(!titles.contains(&"deep.md"));
let docs = items
.iter()
.find(|item| item["title"].as_str() == Some("docs"))
.expect("docs row");
assert_eq!(docs["parentNodeId"].as_str(), None);
assert_eq!(docs["expandable"].as_bool(), Some(true));
let _ = std::fs::remove_dir_all(&root);
}
#[test]
fn local_tree_command_refreshes_search_index_after_rename() {
let root = temp_root("mnote-local-tree-refresh-search-index");
@@ -11333,7 +11585,8 @@ fn main() {}
std::fs::write(root.join("Page").join("report.docx"), b"docx").expect("write docx");
let root_uri = format!("file://{}", root.display());
let snapshot = load_local_folder_file_tree_snapshot(&root_uri).expect("file tree");
let snapshot =
load_local_folder_file_tree_children_snapshot(&root_uri, "Page").expect("file tree");
let items = snapshot.projection["items"].as_array().expect("items");
let mindmap = items
.iter()
@@ -11600,7 +11853,11 @@ fn main() {}
.expect("write Child.md");
// 读取文件树。
let snapshot = load_local_folder_file_tree_snapshot(&root_uri).expect("load file tree");
let snapshot = load_local_folder_file_tree_snapshot_with_reveal(
&root_uri,
Some("local-md:Root~2FChild~2FChild.md"),
)
.expect("load file tree");
let items = snapshot.projection["items"].as_array().expect("items");
// 文件树应保留 Root 文件夹和其中的 Root.md 真实条目。
@@ -11703,7 +11960,8 @@ fn main() {}
.expect("uploaded asset index");
assert!(uploaded_asset_index.contains("docs/README/notes.md"));
let snapshot = load_local_folder_file_tree_snapshot(&root_uri).expect("file tree");
let snapshot = load_local_folder_file_tree_children_snapshot(&root_uri, "docs/README")
.expect("file tree");
let items = snapshot.projection["items"].as_array().expect("items");
let notes_row = items
.iter()
rust/crates/mnote-web/src/routes/mod.rs
+4
View File
@@ -391,6 +391,10 @@ pub fn build_router(state: AppState) -> Router {
)
.route("/api/tree/projections/page", get(kernel::project_tree_page))
.route("/api/tree/projections/file", get(kernel::project_tree_file))
.route(
"/api/tree/projections/file/children",
get(kernel::project_tree_file),
)
.route(
"/api/kernel/projections/sidebar",
get(kernel::project_sidebar),
rust/crates/mnote-web/src/routes/tree.rs
+2 -1
View File
@@ -2741,7 +2741,8 @@ mod tests {
let html = String::from_utf8(body.to_vec()).expect("utf8");
assert!(html.contains("local_folder"));
assert!(html.contains("README.md"));
assert!(html.contains("child.md"));
assert!(html.contains("docs"));
assert!(!html.contains("child.md"));
assert!(html.contains("image.png"));
assert!(html.contains("data-row-kind=\"folder\""));
assert!(html.contains("data-row-kind=\"markdown\""));
rust/crates/mnote-web/src/routes/web_shell.rs
+2 -2
View File
@@ -10,7 +10,7 @@ use crate::routes::documents::{
use crate::routes::gateway::default_workspace_name_for_context;
use crate::routes::local_folder_source::{
ensure_local_workspace_read_access_with_state, is_local_access_policy_admin_context,
load_local_folder_file_tree_snapshot, load_local_folder_page_tree_snapshot,
load_local_folder_file_tree_snapshot_with_reveal, load_local_folder_page_tree_snapshot,
resolve_local_markdown_page_aggregate,
};
use crate::routes::query_support::execute_runtime_query_against_data;
@@ -1594,7 +1594,7 @@ pub(crate) fn render_local_file_tree_html(
active_document_id: Option<&str>,
active_row_id: Option<&str>,
) -> Result<String, WebError> {
let snapshot = load_local_folder_file_tree_snapshot(root_uri)?;
let snapshot = load_local_folder_file_tree_snapshot_with_reveal(root_uri, active_document_id)?;
let rows =
collect_filetree_render_rows(&snapshot.projection, active_document_id, active_row_id);
Ok(render_initial_filetree_html(&FileTreeInitialRenderInput {