feat(rag): replace LiteParse flows with LightRAG provider
This commit is contained in:
lix-2026
@@ -9,7 +9,7 @@ use crate::routes::local_markdown_parser::{
|
||||
file_stem_title, parse_markdown_attachment_refs, parse_markdown_page, split_frontmatter,
|
||||
};
|
||||
use crate::routes::snapshot_support::ProjectionSnapshot;
|
||||
use crate::routes::{local_ocr, local_search_index};
|
||||
use crate::routes::{knowledge_rag, local_ocr, local_search_index};
|
||||
use axum::extract::{Extension, Multipart, Path as AxumPath, Query, State};
|
||||
use axum::http::{header, HeaderMap, HeaderValue, StatusCode};
|
||||
use axum::Json;
|
||||
@@ -307,14 +307,16 @@ struct LocalFolderScanResult {
|
||||
#[derive(Debug, Clone, Default)]
|
||||
struct LocalFileTreeIndexState {
|
||||
indexed_paths: BTreeSet<String>,
|
||||
indexing_paths: BTreeSet<String>,
|
||||
failed_paths: BTreeSet<String>,
|
||||
}
|
||||
|
||||
impl LocalFileTreeIndexState {
|
||||
fn load(root: &Path) -> Self {
|
||||
local_search_index::local_evidence_source_statuses(root)
|
||||
fn load(root: &Path, workspace_id: &str, root_uri: &str) -> Self {
|
||||
knowledge_rag::knowledge_rag_source_statuses(root, workspace_id, root_uri)
|
||||
.map(|statuses| Self {
|
||||
indexed_paths: statuses.indexed_paths,
|
||||
indexing_paths: statuses.indexing_paths,
|
||||
failed_paths: statuses.failed_paths,
|
||||
})
|
||||
.unwrap_or_default()
|
||||
@@ -334,6 +336,9 @@ impl LocalFileTreeIndexState {
|
||||
if self.indexed_paths.contains(path) {
|
||||
return Some("indexed".to_string());
|
||||
}
|
||||
if self.indexing_paths.contains(path) {
|
||||
return Some("indexing".to_string());
|
||||
}
|
||||
None
|
||||
}
|
||||
}
|
||||
@@ -2752,7 +2757,8 @@ fn load_local_folder_file_tree_scope_snapshot(
|
||||
let workspace_id = local_workspace_id(&canonical_root);
|
||||
let root_source_uri = file_uri_for_path(&canonical_root);
|
||||
let metadata = load_local_folder_metadata(&canonical_root)?;
|
||||
let index_state = LocalFileTreeIndexState::load(&canonical_root);
|
||||
let index_state =
|
||||
LocalFileTreeIndexState::load(&canonical_root, &workspace_id, &root_source_uri);
|
||||
let parent_relative_path = parent_relative_path
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty() && *value != ".")
|
||||
@@ -13646,84 +13652,118 @@ fn main() {}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn local_file_tree_marks_indexed_and_failed_source_files() {
|
||||
let root = temp_root("mnote-local-filetree-index-status");
|
||||
fn local_file_tree_marks_lightrag_indexed_indexing_and_failed_source_files() {
|
||||
let root = temp_root("mnote-lightrag-filetree-index-status");
|
||||
init_workspace(&root);
|
||||
std::fs::write(root.join("ok.pdf"), b"%PDF-1.4\nok").expect("ok pdf");
|
||||
std::fs::write(root.join("pending.pdf"), b"%PDF-1.4\npending").expect("pending pdf");
|
||||
std::fs::write(root.join("failed.pdf"), b"%PDF-1.4\nfailed").expect("failed pdf");
|
||||
std::fs::write(root.join("deleting.pdf"), b"%PDF-1.4\ndeleting").expect("deleting pdf");
|
||||
std::fs::write(root.join("removed.pdf"), b"%PDF-1.4\nremoved").expect("removed pdf");
|
||||
std::fs::write(root.join("draft.md"), "# Draft\n").expect("draft");
|
||||
|
||||
let index_dir = root.join(".mnote").join("index");
|
||||
std::fs::create_dir_all(&index_dir).expect("index dir");
|
||||
let evidence_path = index_dir.join("evidence.sqlite");
|
||||
let connection = rusqlite::Connection::open(&evidence_path).expect("evidence sqlite");
|
||||
connection
|
||||
.execute_batch(
|
||||
r#"
|
||||
CREATE TABLE evidence_resource(
|
||||
resource_id TEXT PRIMARY KEY,
|
||||
owner_document_id TEXT NOT NULL,
|
||||
owner_document_path TEXT NOT NULL,
|
||||
source_root_relative_path TEXT NOT NULL,
|
||||
provider TEXT NOT NULL,
|
||||
source_hash TEXT NOT NULL,
|
||||
artifact_root_relative_path TEXT NOT NULL,
|
||||
source_map_root_relative_path TEXT NOT NULL,
|
||||
updated_at_ms INTEGER NOT NULL
|
||||
);
|
||||
"#,
|
||||
)
|
||||
.expect("evidence schema");
|
||||
connection
|
||||
.execute(
|
||||
"INSERT INTO evidence_resource(resource_id, owner_document_id, owner_document_path, source_root_relative_path, provider, source_hash, artifact_root_relative_path, source_map_root_relative_path, updated_at_ms) VALUES(?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)",
|
||||
rusqlite::params![
|
||||
"local-resource:ok.pdf#parse",
|
||||
"local-resource:ok.pdf",
|
||||
"ok.pdf",
|
||||
"ok.pdf",
|
||||
"liteparse",
|
||||
"hash",
|
||||
"ok.ocr/ok.pdf.parse.md",
|
||||
"ok.ocr/ok.pdf.source-map.json",
|
||||
1_i64,
|
||||
],
|
||||
)
|
||||
.expect("insert indexed evidence");
|
||||
connection
|
||||
.execute(
|
||||
"INSERT INTO evidence_resource(resource_id, owner_document_id, owner_document_path, source_root_relative_path, provider, source_hash, artifact_root_relative_path, source_map_root_relative_path, updated_at_ms) VALUES(?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)",
|
||||
rusqlite::params![
|
||||
"local-md:draft.md",
|
||||
"local-md:draft.md",
|
||||
"draft.md",
|
||||
"draft.md",
|
||||
"markdown",
|
||||
"hash",
|
||||
"draft.md",
|
||||
"draft.md.source-map.json",
|
||||
1_i64,
|
||||
],
|
||||
)
|
||||
.expect("insert markdown evidence");
|
||||
let root_uri = format!("file://{}", root.display());
|
||||
let search_index = json!({
|
||||
"version": 1,
|
||||
"builtAt": 1,
|
||||
let workspace_id = local_workspace_id(&root);
|
||||
let registry = json!({
|
||||
"schema": "mnote.knowledge_rag.source_registry.v1",
|
||||
"workspaceId": workspace_id,
|
||||
"rootUri": root_uri,
|
||||
"workspaceId": "local-filetree-index-status",
|
||||
"indexedPaths": ["."],
|
||||
"documents": [],
|
||||
"resources": [
|
||||
{"resourceId": "local-resource:ok.pdf", "resourceType": "pdf", "title": "ok", "path": "ok.pdf", "updatedAt": 1},
|
||||
{"resourceId": "local-resource:failed.pdf", "resourceType": "pdf", "title": "failed", "path": "failed.pdf", "updatedAt": 1}
|
||||
"updatedAtMs": 1,
|
||||
"entries": [
|
||||
{
|
||||
"sourceId": "lightrag-source-ok",
|
||||
"workspaceId": workspace_id,
|
||||
"rootUri": root_uri,
|
||||
"sourcePath": root.join("ok.pdf").display().to_string(),
|
||||
"sourceRootRelativePath": "ok.pdf",
|
||||
"sourceHash": "hash-ok",
|
||||
"lightRagDocId": "doc-ok",
|
||||
"lightRagStatus": "processed",
|
||||
"lightRagFilePath": "ok.pdf",
|
||||
"symlinkPath": "/tmp/ok.pdf",
|
||||
"parserHint": null,
|
||||
"indexedAtMs": 2,
|
||||
"deletedAtMs": null,
|
||||
"stale": false,
|
||||
"updatedAtMs": 2
|
||||
},
|
||||
{
|
||||
"sourceId": "lightrag-source-pending",
|
||||
"workspaceId": workspace_id,
|
||||
"rootUri": root_uri,
|
||||
"sourcePath": root.join("pending.pdf").display().to_string(),
|
||||
"sourceRootRelativePath": "pending.pdf",
|
||||
"sourceHash": "hash-pending",
|
||||
"lightRagDocId": null,
|
||||
"lightRagStatus": "submitted",
|
||||
"lightRagFilePath": "pending.pdf",
|
||||
"symlinkPath": "/tmp/pending.pdf",
|
||||
"parserHint": null,
|
||||
"indexedAtMs": null,
|
||||
"deletedAtMs": null,
|
||||
"stale": false,
|
||||
"updatedAtMs": 2
|
||||
},
|
||||
{
|
||||
"sourceId": "lightrag-source-failed",
|
||||
"workspaceId": workspace_id,
|
||||
"rootUri": root_uri,
|
||||
"sourcePath": root.join("failed.pdf").display().to_string(),
|
||||
"sourceRootRelativePath": "failed.pdf",
|
||||
"sourceHash": "hash-failed",
|
||||
"lightRagDocId": null,
|
||||
"lightRagStatus": "failed",
|
||||
"lightRagFilePath": "failed.pdf",
|
||||
"symlinkPath": "/tmp/failed.pdf",
|
||||
"parserHint": null,
|
||||
"indexedAtMs": null,
|
||||
"deletedAtMs": 3,
|
||||
"stale": true,
|
||||
"updatedAtMs": 3
|
||||
},
|
||||
{
|
||||
"sourceId": "lightrag-source-deleting",
|
||||
"workspaceId": workspace_id,
|
||||
"rootUri": root_uri,
|
||||
"sourcePath": root.join("deleting.pdf").display().to_string(),
|
||||
"sourceRootRelativePath": "deleting.pdf",
|
||||
"sourceHash": "hash-deleting",
|
||||
"lightRagDocId": "doc-deleting",
|
||||
"lightRagStatus": "delete_submitted",
|
||||
"lightRagFilePath": "deleting.pdf",
|
||||
"symlinkPath": "/tmp/deleting.pdf",
|
||||
"parserHint": null,
|
||||
"indexedAtMs": 2,
|
||||
"deletedAtMs": 3,
|
||||
"stale": true,
|
||||
"updatedAtMs": 3
|
||||
},
|
||||
{
|
||||
"sourceId": "lightrag-source-removed",
|
||||
"workspaceId": workspace_id,
|
||||
"rootUri": root_uri,
|
||||
"sourcePath": root.join("removed.pdf").display().to_string(),
|
||||
"sourceRootRelativePath": "removed.pdf",
|
||||
"sourceHash": "hash-removed",
|
||||
"lightRagDocId": null,
|
||||
"lightRagStatus": "delete_completed",
|
||||
"lightRagFilePath": "removed.pdf",
|
||||
"symlinkPath": "/tmp/removed.pdf",
|
||||
"parserHint": null,
|
||||
"indexedAtMs": null,
|
||||
"deletedAtMs": 3,
|
||||
"stale": true,
|
||||
"updatedAtMs": 3
|
||||
}
|
||||
]
|
||||
});
|
||||
std::fs::write(
|
||||
index_dir.join("search-index.json"),
|
||||
format!("{}\n", serde_json::to_string_pretty(&search_index).unwrap()),
|
||||
index_dir.join("lightrag-source-registry.json"),
|
||||
format!("{}\n", serde_json::to_string_pretty(®istry).unwrap()),
|
||||
)
|
||||
.expect("search index");
|
||||
.expect("lightrag registry");
|
||||
|
||||
let file_tree = load_local_folder_file_tree_snapshot(&format!("file://{}", root.display()))
|
||||
.expect("file tree");
|
||||
@@ -13737,7 +13777,10 @@ CREATE TABLE evidence_resource(
|
||||
.and_then(|item| item["indexStatus"].as_str())
|
||||
};
|
||||
assert_eq!(status_for("ok.pdf"), Some("indexed"));
|
||||
assert_eq!(status_for("pending.pdf"), Some("indexing"));
|
||||
assert_eq!(status_for("failed.pdf"), Some("failed"));
|
||||
assert_eq!(status_for("deleting.pdf"), Some("indexing"));
|
||||
assert_eq!(status_for("removed.pdf"), None);
|
||||
assert_eq!(status_for("draft.md"), None);
|
||||
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
|
||||
Reference in New Issue
Block a user