feat(rag): replace LiteParse flows with LightRAG provider

This commit is contained in:
lix-2026
2026-06-07 01:10:31 +08:00
parent f46c2fb5d0
commit 1f25364374
40 changed files with 7232 additions and 2350 deletions
+15 -6
View File
@@ -17,14 +17,23 @@ use serde_json::{json, Value};
use std::path::{Component, Path};
pub async fn search(
State(state): State<AppState>,
State(_state): State<AppState>,
Extension(context): Extension<RequestContext>,
Json(body): Json<EvidenceSearchRequest>,
Json(_body): Json<EvidenceSearchRequest>,
) -> Result<(StatusCode, HeaderMap, Json<Value>), WebError> {
let response = search_payload(&state, &context, body).await?;
let mut headers = HeaderMap::new();
headers.insert("content-type", HeaderValue::from_static("application/json"));
Ok((StatusCode::OK, headers, Json(response)))
Ok((
StatusCode::GONE,
headers,
Json(json!({
"ok": false,
"code": "mnote_evidence_search_retired",
"message": "旧 LiteParse/evidence 搜索已退役;PDF、Office、图片 OCR 与资料索引统一走 /api/knowledge-rag/query",
"requestId": context.trace.request_id,
"traceId": context.trace.trace_id,
})),
))
}
pub(crate) async fn search_payload(
@@ -241,7 +250,7 @@ fn suggested_evidence_queries(query: &str) -> Vec<String> {
suggestions
}
fn citation_markdown_for_locator(locator: &EvidenceLocator) -> String {
pub(crate) fn citation_markdown_for_locator(locator: &EvidenceLocator) -> String {
let label = citation_label_for_locator(locator);
let url = citation_url_for_locator(locator);
format!(
@@ -280,7 +289,7 @@ fn citation_label_for_locator(locator: &EvidenceLocator) -> String {
parts.join(" · ")
}
fn citation_url_for_locator(locator: &EvidenceLocator) -> String {
pub(crate) fn citation_url_for_locator(locator: &EvidenceLocator) -> String {
let owner_document_id = locator.owner_document_id.trim();
let mut url = if owner_document_id.is_empty() {
let existing = locator.open_action.url.trim();
@@ -3,7 +3,7 @@ use crate::app::AppState;
use crate::context::RequestContext;
use crate::error::WebError;
use crate::hermes_tools::{
artifact, block, context_tools, doc, evidence, index, manifest, onlyoffice_live, page,
artifact, block, context_tools, doc, evidence, knowledge_rag, manifest, onlyoffice_live, page,
resource, skill, ToolCallInput,
};
use axum::extract::{Extension, Query, State};
@@ -362,14 +362,27 @@ pub(crate) async fn execute_mnote_tool_call(
"mnote.doc.find" => doc::doc_find(&state, &context, &input).await,
"docs_search" => evidence::legacy_docs_search(&state, &context, &input).await,
"docs_read" => evidence::legacy_docs_read(&state, &context, &input).await,
"mnote.evidence.search" => evidence::evidence_search(&state, &context, &input).await,
"mnote.evidence.read" => evidence::evidence_read(&state, &context, &input).await,
"mnote.evidence.open" => evidence::evidence_open(&state, &context, &input).await,
"mnote.index.status" => index::index_status(&state, &context, &input).await,
"mnote.index.refresh" => index::index_refresh(&state, &context, &input).await,
"mnote.index.update_settings" => {
index::index_update_settings(&state, &context, &input).await
"mnote.evidence.search" | "mnote.evidence.read" | "mnote.evidence.open" => {
Err(WebError::new(
StatusCode::GONE,
"mnote_evidence_tools_retired",
"旧 evidence / LiteParse tools 已退役;请使用 mnote.knowledge_rag.query/open_reference",
)
.with_context(&context))
}
"mnote.knowledge_rag.status" => knowledge_rag::status(&state, &context, &input).await,
"mnote.knowledge_rag.query" => knowledge_rag::query(&state, &context, &input).await,
"mnote.knowledge_rag.open_reference" => {
knowledge_rag::open_reference(&state, &context, &input).await
}
"mnote.index.status" | "mnote.index.refresh" | "mnote.index.update_settings" => Err(
WebError::new(
StatusCode::GONE,
"mnote_index_tools_retired",
"旧本地索引 tools 已退役;资料索引统一由 LightRAG provider 处理",
)
.with_context(&context),
),
"mnote.doc.plan_update" => doc::plan_update(&state, &context, &input).await,
"mnote.block.fetch" => block::block_fetch(&state, &context, &input).await,
"mnote.block.replace" => block::block_replace(&state, &context, &input).await,
@@ -709,6 +722,9 @@ fn is_read_tool(tool_name: &str) -> bool {
| "mnote.evidence.search"
| "mnote.evidence.read"
| "mnote.evidence.open"
| "mnote.knowledge_rag.status"
| "mnote.knowledge_rag.query"
| "mnote.knowledge_rag.open_reference"
| "mnote.index.status"
| "mnote.index.refresh"
| "mnote.block.fetch"
@@ -738,6 +754,9 @@ fn is_evidence_receipt_tool(tool_name: &str) -> bool {
| "mnote.evidence.search"
| "mnote.evidence.read"
| "mnote.evidence.open"
| "mnote.knowledge_rag.status"
| "mnote.knowledge_rag.query"
| "mnote.knowledge_rag.open_reference"
)
}
@@ -1604,23 +1623,29 @@ mod tests {
let payload: Value = serde_json::from_slice(&body).expect("json");
let manifest = &payload["manifest"];
let capabilities = manifest["capabilities"].as_array().expect("capabilities");
assert!(capabilities
assert!(!capabilities
.iter()
.any(|capability| capability["id"] == "mnote-local-index"));
assert!(capabilities
.iter()
.any(|capability| capability["id"] == "mnote-knowledge-rag"));
assert!(capabilities
.iter()
.any(|capability| capability["id"] == "mnote-onlyoffice-live"));
let tools = manifest["tools"].as_array().expect("tools");
let index_status = tools
assert!(!tools
.iter()
.find(|tool| tool["name"] == "mnote.index.status")
.expect("index status tool");
assert!(index_status["capabilityIds"]
.any(|tool| tool["name"] == "mnote.index.status"));
let knowledge_rag_query = tools
.iter()
.find(|tool| tool["name"] == "mnote.knowledge_rag.query")
.expect("knowledge rag query tool");
assert!(knowledge_rag_query["capabilityIds"]
.as_array()
.expect("index capability ids")
.expect("knowledge rag capability ids")
.iter()
.any(|id| id == "mnote-local-index"));
.any(|id| id == "mnote-knowledge-rag"));
let onlyoffice_batch_set = tools
.iter()
@@ -5573,7 +5598,7 @@ mod tests {
}
#[tokio::test]
async fn hermes_tools_local_index_update_and_status_manage_scope() {
async fn hermes_tools_local_index_tools_are_retired() {
let root = std::env::temp_dir().join(format!("mnote-index-tool-{}", std::process::id()));
let _ = fs::remove_dir_all(&root);
fs::create_dir_all(root.join(".mnote")).expect("metadata");
@@ -5583,11 +5608,7 @@ mod tests {
r#"{"workspaceId":"local-ws-index-tool","ownerId":"user_1","createdAt":"2026-06-04T00:00:00Z","capabilities":["local_files","search"]}"#,
)
.expect("manifest");
fs::write(
root.join("docs").join("indexed.md"),
"# Indexed\n\nindex-tool-token\n",
)
.expect("markdown");
fs::write(root.join("docs").join("indexed.md"), "# Indexed\n").expect("markdown");
let root_uri = format!("file://{}", root.display());
let app = app();
@@ -5625,18 +5646,16 @@ mod tests {
)
.await
.expect("update response");
assert_eq!(update_response.status(), StatusCode::OK);
assert_eq!(update_response.status(), StatusCode::GONE);
let update_body = to_bytes(update_response.into_body(), usize::MAX)
.await
.expect("update body");
let update_payload: Value = serde_json::from_slice(&update_body).expect("update json");
assert_eq!(
update_payload["result"]["settings"]["includePaths"][0],
"docs"
update_payload["code"].as_str(),
Some("mnote_index_tools_retired")
);
assert_eq!(update_payload["result"]["index"]["documentCount"], 1);
assert!(root.join(".mnote/index/search-index.json").exists());
assert!(root.join(".mnote/index/evidence.sqlite").exists());
assert!(!root.join(".mnote/index/search-index.json").exists());
let status_response = app
.oneshot(
@@ -5664,16 +5683,15 @@ mod tests {
)
.await
.expect("status response");
assert_eq!(status_response.status(), StatusCode::OK);
assert_eq!(status_response.status(), StatusCode::GONE);
let status_body = to_bytes(status_response.into_body(), usize::MAX)
.await
.expect("status body");
let status_payload: Value = serde_json::from_slice(&status_body).expect("status json");
assert_eq!(
status_payload["result"]["result"]["settings"]["includePaths"][0],
"docs"
status_payload["code"].as_str(),
Some("mnote_index_tools_retired")
);
assert_eq!(status_payload["audit"]["effect"], "read");
let _ = fs::remove_dir_all(&root);
}
File diff suppressed because it is too large Load Diff
@@ -1,12 +1,15 @@
use crate::app::AppState;
use crate::context::RequestContext;
use crate::error::WebError;
use crate::routes::local_folder_source::{
decode_local_id_segment, ensure_local_workspace_read_access_with_state,
load_local_folder_file_tree_snapshot, load_local_folder_page_tree_snapshot,
local_folder_watch_revision, local_workspace_id_from_root_uri,
};
use crate::routes::snapshot_support::ProjectionSnapshot;
use crate::routes::{
knowledge_rag,
local_folder_source::{
decode_local_id_segment, ensure_local_workspace_read_access_with_state,
load_local_folder_file_tree_snapshot, load_local_folder_page_tree_snapshot,
local_folder_watch_revision, local_workspace_id_from_root_uri,
},
};
use axum::extract::{Extension, Query, State};
use axum::http::{HeaderMap, HeaderName, HeaderValue};
use axum::response::sse::{Event as SseEvent, Sse};
@@ -230,12 +233,19 @@ async fn build_tree_live_stream(
})?;
let stream = stream::unfold(
(Some(initial_payload), subscription, root_uri, workspace_id),
|(payload, mut subscription, root_uri, workspace_id)| async move {
(
Some(initial_payload),
subscription,
root_uri,
workspace_id,
state,
context,
),
|(payload, mut subscription, root_uri, workspace_id, state, context)| async move {
if let Some(payload) = payload {
return Some((
Ok(stream_event("snapshot", &payload)),
(None, subscription, root_uri, workspace_id),
(None, subscription, root_uri, workspace_id, state, context),
));
}
@@ -253,6 +263,13 @@ async fn build_tree_live_stream(
Err(_) => break,
}
}
let _ = knowledge_rag::sync_registry_for_root(
&state,
&context,
&root_uri,
Some(&workspace_id),
)
.await;
if let Some(batch_payload) = build_local_folder_watch_batch_payload(
&root_uri,
&workspace_id,
@@ -260,7 +277,7 @@ async fn build_tree_live_stream(
) {
return Some((
Ok(stream_event("watch_batch", &batch_payload)),
(None, subscription, root_uri, workspace_id),
(None, subscription, root_uri, workspace_id, state, context),
));
}
let error_payload = build_tree_live_error_payload(
@@ -271,7 +288,7 @@ async fn build_tree_live_stream(
);
return Some((
Ok(stream_event("tree_error", &error_payload)),
(None, subscription, root_uri, workspace_id),
(None, subscription, root_uri, workspace_id, state, context),
));
}
Err(RecvError::Lagged(_)) => continue,
@@ -9,7 +9,7 @@ use crate::routes::local_markdown_parser::{
file_stem_title, parse_markdown_attachment_refs, parse_markdown_page, split_frontmatter,
};
use crate::routes::snapshot_support::ProjectionSnapshot;
use crate::routes::{local_ocr, local_search_index};
use crate::routes::{knowledge_rag, local_ocr, local_search_index};
use axum::extract::{Extension, Multipart, Path as AxumPath, Query, State};
use axum::http::{header, HeaderMap, HeaderValue, StatusCode};
use axum::Json;
@@ -307,14 +307,16 @@ struct LocalFolderScanResult {
#[derive(Debug, Clone, Default)]
struct LocalFileTreeIndexState {
indexed_paths: BTreeSet<String>,
indexing_paths: BTreeSet<String>,
failed_paths: BTreeSet<String>,
}
impl LocalFileTreeIndexState {
fn load(root: &Path) -> Self {
local_search_index::local_evidence_source_statuses(root)
fn load(root: &Path, workspace_id: &str, root_uri: &str) -> Self {
knowledge_rag::knowledge_rag_source_statuses(root, workspace_id, root_uri)
.map(|statuses| Self {
indexed_paths: statuses.indexed_paths,
indexing_paths: statuses.indexing_paths,
failed_paths: statuses.failed_paths,
})
.unwrap_or_default()
@@ -334,6 +336,9 @@ impl LocalFileTreeIndexState {
if self.indexed_paths.contains(path) {
return Some("indexed".to_string());
}
if self.indexing_paths.contains(path) {
return Some("indexing".to_string());
}
None
}
}
@@ -2752,7 +2757,8 @@ fn load_local_folder_file_tree_scope_snapshot(
let workspace_id = local_workspace_id(&canonical_root);
let root_source_uri = file_uri_for_path(&canonical_root);
let metadata = load_local_folder_metadata(&canonical_root)?;
let index_state = LocalFileTreeIndexState::load(&canonical_root);
let index_state =
LocalFileTreeIndexState::load(&canonical_root, &workspace_id, &root_source_uri);
let parent_relative_path = parent_relative_path
.map(str::trim)
.filter(|value| !value.is_empty() && *value != ".")
@@ -13646,84 +13652,118 @@ fn main() {}
}
#[test]
fn local_file_tree_marks_indexed_and_failed_source_files() {
let root = temp_root("mnote-local-filetree-index-status");
fn local_file_tree_marks_lightrag_indexed_indexing_and_failed_source_files() {
let root = temp_root("mnote-lightrag-filetree-index-status");
init_workspace(&root);
std::fs::write(root.join("ok.pdf"), b"%PDF-1.4\nok").expect("ok pdf");
std::fs::write(root.join("pending.pdf"), b"%PDF-1.4\npending").expect("pending pdf");
std::fs::write(root.join("failed.pdf"), b"%PDF-1.4\nfailed").expect("failed pdf");
std::fs::write(root.join("deleting.pdf"), b"%PDF-1.4\ndeleting").expect("deleting pdf");
std::fs::write(root.join("removed.pdf"), b"%PDF-1.4\nremoved").expect("removed pdf");
std::fs::write(root.join("draft.md"), "# Draft\n").expect("draft");
let index_dir = root.join(".mnote").join("index");
std::fs::create_dir_all(&index_dir).expect("index dir");
let evidence_path = index_dir.join("evidence.sqlite");
let connection = rusqlite::Connection::open(&evidence_path).expect("evidence sqlite");
connection
.execute_batch(
r#"
CREATE TABLE evidence_resource(
resource_id TEXT PRIMARY KEY,
owner_document_id TEXT NOT NULL,
owner_document_path TEXT NOT NULL,
source_root_relative_path TEXT NOT NULL,
provider TEXT NOT NULL,
source_hash TEXT NOT NULL,
artifact_root_relative_path TEXT NOT NULL,
source_map_root_relative_path TEXT NOT NULL,
updated_at_ms INTEGER NOT NULL
);
"#,
)
.expect("evidence schema");
connection
.execute(
"INSERT INTO evidence_resource(resource_id, owner_document_id, owner_document_path, source_root_relative_path, provider, source_hash, artifact_root_relative_path, source_map_root_relative_path, updated_at_ms) VALUES(?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)",
rusqlite::params![
"local-resource:ok.pdf#parse",
"local-resource:ok.pdf",
"ok.pdf",
"ok.pdf",
"liteparse",
"hash",
"ok.ocr/ok.pdf.parse.md",
"ok.ocr/ok.pdf.source-map.json",
1_i64,
],
)
.expect("insert indexed evidence");
connection
.execute(
"INSERT INTO evidence_resource(resource_id, owner_document_id, owner_document_path, source_root_relative_path, provider, source_hash, artifact_root_relative_path, source_map_root_relative_path, updated_at_ms) VALUES(?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)",
rusqlite::params![
"local-md:draft.md",
"local-md:draft.md",
"draft.md",
"draft.md",
"markdown",
"hash",
"draft.md",
"draft.md.source-map.json",
1_i64,
],
)
.expect("insert markdown evidence");
let root_uri = format!("file://{}", root.display());
let search_index = json!({
"version": 1,
"builtAt": 1,
let workspace_id = local_workspace_id(&root);
let registry = json!({
"schema": "mnote.knowledge_rag.source_registry.v1",
"workspaceId": workspace_id,
"rootUri": root_uri,
"workspaceId": "local-filetree-index-status",
"indexedPaths": ["."],
"documents": [],
"resources": [
{"resourceId": "local-resource:ok.pdf", "resourceType": "pdf", "title": "ok", "path": "ok.pdf", "updatedAt": 1},
{"resourceId": "local-resource:failed.pdf", "resourceType": "pdf", "title": "failed", "path": "failed.pdf", "updatedAt": 1}
"updatedAtMs": 1,
"entries": [
{
"sourceId": "lightrag-source-ok",
"workspaceId": workspace_id,
"rootUri": root_uri,
"sourcePath": root.join("ok.pdf").display().to_string(),
"sourceRootRelativePath": "ok.pdf",
"sourceHash": "hash-ok",
"lightRagDocId": "doc-ok",
"lightRagStatus": "processed",
"lightRagFilePath": "ok.pdf",
"symlinkPath": "/tmp/ok.pdf",
"parserHint": null,
"indexedAtMs": 2,
"deletedAtMs": null,
"stale": false,
"updatedAtMs": 2
},
{
"sourceId": "lightrag-source-pending",
"workspaceId": workspace_id,
"rootUri": root_uri,
"sourcePath": root.join("pending.pdf").display().to_string(),
"sourceRootRelativePath": "pending.pdf",
"sourceHash": "hash-pending",
"lightRagDocId": null,
"lightRagStatus": "submitted",
"lightRagFilePath": "pending.pdf",
"symlinkPath": "/tmp/pending.pdf",
"parserHint": null,
"indexedAtMs": null,
"deletedAtMs": null,
"stale": false,
"updatedAtMs": 2
},
{
"sourceId": "lightrag-source-failed",
"workspaceId": workspace_id,
"rootUri": root_uri,
"sourcePath": root.join("failed.pdf").display().to_string(),
"sourceRootRelativePath": "failed.pdf",
"sourceHash": "hash-failed",
"lightRagDocId": null,
"lightRagStatus": "failed",
"lightRagFilePath": "failed.pdf",
"symlinkPath": "/tmp/failed.pdf",
"parserHint": null,
"indexedAtMs": null,
"deletedAtMs": 3,
"stale": true,
"updatedAtMs": 3
},
{
"sourceId": "lightrag-source-deleting",
"workspaceId": workspace_id,
"rootUri": root_uri,
"sourcePath": root.join("deleting.pdf").display().to_string(),
"sourceRootRelativePath": "deleting.pdf",
"sourceHash": "hash-deleting",
"lightRagDocId": "doc-deleting",
"lightRagStatus": "delete_submitted",
"lightRagFilePath": "deleting.pdf",
"symlinkPath": "/tmp/deleting.pdf",
"parserHint": null,
"indexedAtMs": 2,
"deletedAtMs": 3,
"stale": true,
"updatedAtMs": 3
},
{
"sourceId": "lightrag-source-removed",
"workspaceId": workspace_id,
"rootUri": root_uri,
"sourcePath": root.join("removed.pdf").display().to_string(),
"sourceRootRelativePath": "removed.pdf",
"sourceHash": "hash-removed",
"lightRagDocId": null,
"lightRagStatus": "delete_completed",
"lightRagFilePath": "removed.pdf",
"symlinkPath": "/tmp/removed.pdf",
"parserHint": null,
"indexedAtMs": null,
"deletedAtMs": 3,
"stale": true,
"updatedAtMs": 3
}
]
});
std::fs::write(
index_dir.join("search-index.json"),
format!("{}\n", serde_json::to_string_pretty(&search_index).unwrap()),
index_dir.join("lightrag-source-registry.json"),
format!("{}\n", serde_json::to_string_pretty(&registry).unwrap()),
)
.expect("search index");
.expect("lightrag registry");
let file_tree = load_local_folder_file_tree_snapshot(&format!("file://{}", root.display()))
.expect("file tree");
@@ -13737,7 +13777,10 @@ CREATE TABLE evidence_resource(
.and_then(|item| item["indexStatus"].as_str())
};
assert_eq!(status_for("ok.pdf"), Some("indexed"));
assert_eq!(status_for("pending.pdf"), Some("indexing"));
assert_eq!(status_for("failed.pdf"), Some("failed"));
assert_eq!(status_for("deleting.pdf"), Some("indexing"));
assert_eq!(status_for("removed.pdf"), None);
assert_eq!(status_for("draft.md"), None);
let _ = std::fs::remove_dir_all(&root);
@@ -2131,6 +2131,7 @@ mod tests {
}
#[tokio::test]
#[ignore = "local OCR HTTP routes are retired in favor of LightRAG"]
async fn local_ocr_jobs_route_writes_mock_sidecar_and_reads_status() {
let root = temp_root("mnote-local-ocr-route");
write_workspace_manifest(&root);
@@ -2255,6 +2256,7 @@ mod tests {
}
#[tokio::test]
#[ignore = "local OCR HTTP routes are retired in favor of LightRAG"]
async fn local_ocr_jobs_route_reuses_existing_done_sidecar_without_reprocessing() {
let root = temp_root("mnote-local-ocr-dedup-done");
write_workspace_manifest(&root);
@@ -2331,6 +2333,7 @@ mod tests {
}
#[tokio::test]
#[ignore = "local OCR HTTP routes are retired in favor of LightRAG"]
async fn local_ocr_jobs_route_recovers_existing_done_sidecar_when_index_is_missing() {
let root = temp_root("mnote-local-ocr-dedup-sidecar-recover");
write_workspace_manifest(&root);
@@ -2405,6 +2408,7 @@ mod tests {
}
#[tokio::test]
#[ignore = "local OCR HTTP routes are retired in favor of LightRAG"]
async fn local_ocr_jobs_route_reuses_existing_done_before_mineru_token_check() {
let old_mnote_token = std::env::var("MNOTE_MINERU_API_TOKEN").ok();
let old_mineru_token = std::env::var("MINERU_API_TOKEN").ok();
@@ -2484,6 +2488,7 @@ mod tests {
}
#[tokio::test]
#[ignore = "local OCR HTTP routes are retired in favor of LightRAG"]
async fn local_ocr_jobs_route_broadcasts_stage_updates() {
let root = temp_root("mnote-local-ocr-events");
write_workspace_manifest(&root);
@@ -2548,6 +2553,7 @@ mod tests {
}
#[tokio::test]
#[ignore = "local OCR HTTP routes are retired in favor of LightRAG"]
async fn local_ocr_jobs_route_rejects_missing_mineru_token() {
let old_mnote_token = std::env::var("MNOTE_MINERU_API_TOKEN").ok();
let old_mineru_token = std::env::var("MINERU_API_TOKEN").ok();
@@ -2584,6 +2590,7 @@ mod tests {
}
#[tokio::test]
#[ignore = "local OCR HTTP routes are retired in favor of LightRAG"]
async fn local_ocr_jobs_route_runs_mineru_runtime_against_http_mock() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
.await
@@ -2832,6 +2839,7 @@ mod tests {
}
#[tokio::test]
#[ignore = "local OCR HTTP routes are retired in favor of LightRAG"]
async fn local_ocr_jobs_route_rejects_root_escape_source() {
let root = temp_root("mnote-local-ocr-escape");
write_workspace_manifest(&root);
@@ -2853,6 +2861,7 @@ mod tests {
}
#[tokio::test]
#[ignore = "local OCR HTTP routes are retired in favor of LightRAG"]
async fn local_ocr_jobs_route_rejects_non_image_pdf_source() {
let root = temp_root("mnote-local-ocr-unsupported");
write_workspace_manifest(&root);
@@ -2882,6 +2891,7 @@ mod tests {
}
#[tokio::test]
#[ignore = "local OCR HTTP routes are retired in favor of LightRAG"]
async fn local_ocr_jobs_route_writes_failed_mock_entry_with_redacted_error() {
let root = temp_root("mnote-local-ocr-failed");
write_workspace_manifest(&root);
@@ -2938,6 +2948,7 @@ mod tests {
}
#[tokio::test]
#[ignore = "local OCR HTTP routes are retired in favor of LightRAG"]
async fn local_ocr_insert_route_appends_explicit_ocr_link() {
let root = temp_root("mnote-local-ocr-insert");
write_workspace_manifest(&root);
@@ -1,7 +1,4 @@
use crate::error::WebError;
use crate::evidence_parse::{
ParseInput, ParseProviderMode, liteparse_runtime_available, parse_liteparse_input_blocking,
};
use crate::routes::local_folder_source::encode_local_id_segment;
use crate::routes::local_markdown_parser::{
parse_markdown_attachment_link, parse_markdown_page, split_frontmatter,
@@ -9,12 +6,12 @@ use crate::routes::local_markdown_parser::{
use crate::routes::local_ocr;
use control_plane::{ControlPlaneStore, UpsertUserInput, UpsertUserUiPreferenceInput};
use core_protocol::{
EvidenceLocator, EvidenceSearchMatchInfo, EvidenceSearchResult,
PARSED_RESOURCE_ARTIFACT_SCHEMA, ParsedResourceArtifact, ResourceSourceMap, SourceMapBlock,
EvidenceLocator, EvidenceSearchMatchInfo, EvidenceSearchResult, ParsedResourceArtifact,
ResourceSourceMap, SourceMapBlock, PARSED_RESOURCE_ARTIFACT_SCHEMA,
};
use rusqlite::{Connection, OptionalExtension, params};
use rusqlite::{params, Connection, OptionalExtension};
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use serde_json::{json, Value};
use std::collections::BTreeSet;
use std::fs;
use std::path::{Component, Path, PathBuf};
@@ -183,49 +180,7 @@ pub(crate) fn query_local_search_index_with_settings(
}
}
}
if include_ocr && results.len() < limit.max(1) as usize {
for entry in local_ocr::ocr_index_entries(root_path)? {
if !index_relative_path_is_included(
&entry.source_root_relative_path,
&result_settings.include_paths,
) {
continue;
}
if let Some(page_id) = page_id {
if let Some(resource_path) = page_resource_path.as_deref() {
if entry.source_root_relative_path != resource_path {
continue;
}
} else if entry.owner_document_id != page_id {
continue;
}
}
if entry.status != "done" {
continue;
}
let ocr_path = root_path.join(&entry.ocr_root_relative_path);
let markdown = match fs::read_to_string(&ocr_path) {
Ok(markdown) => markdown,
Err(_) => continue,
};
if local_ocr::parse_ocr_frontmatter(&markdown).is_none() {
continue;
}
let body = local_ocr::strip_ocr_frontmatter(&markdown);
if !local_search_ocr_matches(&entry, body, &normalized_query, title_only, exact) {
continue;
}
results.push(local_search_ocr_projection(
&entry,
body,
root_uri,
&normalized_query,
));
if results.len() >= limit.max(1) as usize {
break;
}
}
}
let _ = include_ocr;
Ok(json!({
"enqueueAssetIds": [],
"projectionOwner": "rust-kernel",
@@ -2244,14 +2199,6 @@ fn write_evidence_sqlite_index(root_path: &Path, index: &LocalSearchIndex) -> Re
for resource in &index.resources {
insert_resource_evidence(&tx, root_path, index, resource)?;
}
for entry in local_ocr::ocr_index_entries(root_path)?
.into_iter()
.filter(|entry| {
index_relative_path_is_included(&entry.source_root_relative_path, &index.indexed_paths)
})
{
insert_ocr_evidence(&tx, root_path, index, &entry)?;
}
for document in &index.documents {
insert_document_graph_edges(&tx, document)?;
}
@@ -2373,15 +2320,6 @@ fn refresh_evidence_sqlite_index_for_path(
{
insert_resource_evidence(&tx, root_path, index, resource)?;
}
for entry in local_ocr::ocr_index_entries(root_path)?
.into_iter()
.filter(|entry| {
entry.ocr_root_relative_path == relative_path
|| entry.source_root_relative_path == relative_path
})
{
insert_ocr_evidence(&tx, root_path, index, &entry)?;
}
tx.execute(
"INSERT OR REPLACE INTO evidence_meta(key, value) VALUES('schema_version', ?1)",
params![EVIDENCE_SQLITE_SCHEMA_VERSION.to_string()],
@@ -2664,147 +2602,18 @@ fn insert_resource_evidence(
}
fn parse_resource_evidence_artifact(
root_path: &Path,
index: &LocalSearchIndex,
resource: &LocalSearchResource,
_root_path: &Path,
_index: &LocalSearchIndex,
_resource: &LocalSearchResource,
) -> Result<Option<crate::evidence_parse::ParseProviderOutput>, WebError> {
if !resource_parse_supported(resource) || !liteparse_runtime_available() {
return Ok(None);
}
let owner = evidence_owner_for_resource(index, resource);
let mut output = match parse_liteparse_input_blocking(ParseInput {
root_path: root_path.to_path_buf(),
root_uri: index.root_uri.clone(),
owner_document_id: owner.document_id.clone(),
owner_document_path: owner.path.clone(),
source_root_relative_path: resource.path.clone(),
mode: ParseProviderMode::NoOcr,
}) {
Ok(output) => output,
Err(_) => return Ok(None),
};
let sidecar = resource_parse_sidecar_paths(&owner.path, &resource.path);
write_parsed_resource_sidecars(root_path, &sidecar, &output.markdown, &output.source_map)?;
output.artifact.artifact_root_relative_path = sidecar.parse_root_relative_path;
output.artifact.source_map_root_relative_path = sidecar.source_map_root_relative_path.clone();
output.source_map.owner_document_path = owner.path;
Ok(Some(output))
}
fn resource_parse_supported(resource: &LocalSearchResource) -> bool {
matches!(resource.resource_type.as_str(), "pdf" | "office")
Ok(None)
}
fn parsed_resource_id(resource: &LocalSearchResource) -> String {
format!("{}#parse", resource.resource_id)
}
#[derive(Debug, Clone)]
struct EvidenceResourceOwner {
document_id: String,
path: String,
}
fn evidence_owner_for_resource(
index: &LocalSearchIndex,
resource: &LocalSearchResource,
) -> EvidenceResourceOwner {
if let Some(document) = index.documents.iter().find(|document| {
document.resource_refs.iter().any(|reference| {
normalize_local_reference_path(&document.path, reference) == resource.path
})
}) {
return EvidenceResourceOwner {
document_id: document.document_id.clone(),
path: document.path.clone(),
};
}
EvidenceResourceOwner {
document_id: resource.resource_id.clone(),
path: resource.path.clone(),
}
}
#[derive(Debug, Clone)]
struct ResourceParseSidecarPaths {
parse_root_relative_path: String,
source_map_root_relative_path: String,
}
fn resource_parse_sidecar_paths(
owner_document_path: &str,
source_root_relative_path: &str,
) -> ResourceParseSidecarPaths {
let owner_parent = Path::new(owner_document_path)
.parent()
.map(Path::to_path_buf)
.unwrap_or_default();
let owner_stem = Path::new(owner_document_path)
.file_stem()
.and_then(|value| value.to_str())
.unwrap_or("Page");
let source_leaf = Path::new(source_root_relative_path)
.file_name()
.and_then(|value| value.to_str())
.unwrap_or("source");
let sidecar_dir = owner_parent.join(format!("{owner_stem}.ocr"));
let parse_root_relative_path = sidecar_dir
.join(format!("{source_leaf}.parse.md"))
.to_string_lossy()
.replace('\\', "/");
let source_map_root_relative_path = sidecar_dir
.join(format!("{source_leaf}.source-map.json"))
.to_string_lossy()
.replace('\\', "/");
ResourceParseSidecarPaths {
parse_root_relative_path,
source_map_root_relative_path,
}
}
fn write_parsed_resource_sidecars(
root_path: &Path,
sidecar: &ResourceParseSidecarPaths,
markdown: &str,
source_map: &ResourceSourceMap,
) -> Result<(), WebError> {
let parse_path = root_path.join(&sidecar.parse_root_relative_path);
let source_map_path = root_path.join(&sidecar.source_map_root_relative_path);
for path in [&parse_path, &source_map_path] {
if let Some(parent) = path.parent() {
fs::create_dir_all(parent).map_err(|error| {
WebError::bad_request_code(
"evidence_parse_sidecar_create_failed",
format!(
"无法创建证据解析 sidecar 目录 {}: {error}",
parent.display()
),
)
})?;
}
}
fs::write(&parse_path, markdown.trim_end()).map_err(|error| {
WebError::bad_request_code(
"evidence_parse_sidecar_write_failed",
format!(
"无法写入证据解析 Markdown {}: {error}",
parse_path.display()
),
)
})?;
let source_map_json = serde_json::to_string_pretty(source_map)
.map_err(|error| WebError::internal(format!("证据 source-map 序列化失败: {error}")))?;
fs::write(&source_map_path, format!("{source_map_json}\n")).map_err(|error| {
WebError::bad_request_code(
"evidence_parse_sidecar_write_failed",
format!(
"无法写入证据 source-map {}: {error}",
source_map_path.display()
),
)
})
}
#[allow(dead_code)]
fn insert_ocr_evidence(
connection: &Connection,
root_path: &Path,
@@ -2863,6 +2672,7 @@ fn insert_ocr_evidence(
insert_evidence_block(connection, &resource_id, &resource_id, body, locator)
}
#[allow(dead_code)]
fn ocr_parsed_artifact(
entry: &local_ocr::OcrIndexEntry,
source_map_root_relative_path: &str,
@@ -3461,12 +3271,14 @@ fn count_evidence_blocks(path: &Path) -> Result<u64, WebError> {
.map_err(sqlite_error)
}
#[allow(dead_code)]
#[derive(Debug, Clone, Default)]
pub(crate) struct LocalEvidenceSourceStatuses {
pub(crate) indexed_paths: BTreeSet<String>,
pub(crate) failed_paths: BTreeSet<String>,
}
#[allow(dead_code)]
pub(crate) fn local_evidence_source_statuses(
root_path: &Path,
) -> Result<LocalEvidenceSourceStatuses, WebError> {
@@ -3509,21 +3321,6 @@ pub(crate) fn local_evidence_source_statuses(
}
}
}
for entry in local_ocr::ocr_index_entries(root_path)? {
match entry.status.as_str() {
"done" => {
statuses
.indexed_paths
.insert(entry.source_root_relative_path);
}
"failed" | "interrupted" => {
statuses
.failed_paths
.insert(entry.source_root_relative_path);
}
_ => {}
}
}
Ok(statuses)
}
@@ -3563,6 +3360,7 @@ fn local_resource_path_from_document_id(document_id: &str) -> Option<String> {
.filter(|value| !value.trim().is_empty())
}
#[allow(dead_code)]
fn path_source_map_path(path: &str) -> Option<String> {
if let Some(stripped) = path.strip_suffix(".ocr.md") {
return Some(format!("{stripped}.source-map.json"));
@@ -3681,6 +3479,7 @@ fn local_search_resource_matches(
}
}
#[allow(dead_code)]
fn local_search_ocr_matches(
entry: &local_ocr::OcrIndexEntry,
body: &str,
@@ -3758,6 +3557,7 @@ fn local_search_resource_projection(resource: &LocalSearchResource, root_uri: &s
})
}
#[allow(dead_code)]
fn local_search_ocr_projection(
entry: &local_ocr::OcrIndexEntry,
body: &str,
@@ -4375,55 +4175,41 @@ mod tests {
.iter()
.find(|item| item["documentId"].as_str() == Some("local-md:README.md"))
.expect("home result");
assert!(
home["tags"]
.as_array()
.unwrap()
.iter()
.any(|tag| tag.as_str() == Some("alpha"))
);
assert!(
home["backlinks"]
.as_array()
.unwrap()
.iter()
.any(|link| link.as_str() == Some("Daily"))
);
assert!(
home["backlinks"]
.as_array()
.unwrap()
.iter()
.any(|link| link.as_str() == Some("docs/child.md"))
);
assert!(
home["resourceRefs"]
.as_array()
.unwrap()
.iter()
.any(|reference| reference.as_str() == Some("assets/spec.pdf"))
);
assert!(
home["resourceRefs"]
.as_array()
.unwrap()
.iter()
.any(|reference| reference.as_str() == Some("maps/idea.mindmap.json"))
);
assert!(
home["resourceRefs"]
.as_array()
.unwrap()
.iter()
.any(|reference| reference.as_str() == Some("office/report.xlsx"))
);
assert!(
home["publicPath"]
.as_str()
.is_some_and(|path| path.starts_with(
"/documents/local-md:README.md?sourceKind=local_folder&rootUri=file%3A%2F%2F"
))
);
assert!(home["tags"]
.as_array()
.unwrap()
.iter()
.any(|tag| tag.as_str() == Some("alpha")));
assert!(home["backlinks"]
.as_array()
.unwrap()
.iter()
.any(|link| link.as_str() == Some("Daily")));
assert!(home["backlinks"]
.as_array()
.unwrap()
.iter()
.any(|link| link.as_str() == Some("docs/child.md")));
assert!(home["resourceRefs"]
.as_array()
.unwrap()
.iter()
.any(|reference| reference.as_str() == Some("assets/spec.pdf")));
assert!(home["resourceRefs"]
.as_array()
.unwrap()
.iter()
.any(|reference| reference.as_str() == Some("maps/idea.mindmap.json")));
assert!(home["resourceRefs"]
.as_array()
.unwrap()
.iter()
.any(|reference| reference.as_str() == Some("office/report.xlsx")));
assert!(home["publicPath"]
.as_str()
.is_some_and(|path| path.starts_with(
"/documents/local-md:README.md?sourceKind=local_folder&rootUri=file%3A%2F%2F"
)));
let evidence_db = root.join(".mnote").join("index").join("evidence.sqlite");
let connection = Connection::open(&evidence_db).expect("open evidence sqlite");
let markdown_edge_count: i64 = connection
@@ -4479,12 +4265,11 @@ mod tests {
Some("local-mdid:child-page")
);
assert!(
root.join(".mnote")
.join("index")
.join("search-index.json")
.exists()
);
assert!(root
.join(".mnote")
.join("index")
.join("search-index.json")
.exists());
let mindmap_projection = query_local_search_index(
&root,
&format!("file://{}", root.display()),
@@ -4497,19 +4282,19 @@ mod tests {
false,
)
.expect("mindmap projection");
assert!(
mindmap_projection["results"]
.as_array()
.unwrap()
.iter()
.any(|item| item["resourceType"].as_str() == Some("mindmap")
&& item["path"].as_str() == Some("maps/idea.mindmap.json")
&& item["publicPath"].as_str().is_some_and(|path| path
assert!(mindmap_projection["results"]
.as_array()
.unwrap()
.iter()
.any(|item| item["resourceType"].as_str() == Some("mindmap")
&& item["path"].as_str() == Some("maps/idea.mindmap.json")
&& item["publicPath"]
.as_str()
.is_some_and(|path| path
.starts_with("/?treeView=filetree&sourceKind=local_folder&rootUri="))
&& item["publicPath"]
.as_str()
.is_some_and(|path| !path.starts_with("/tree?")))
);
&& item["publicPath"]
.as_str()
.is_some_and(|path| !path.starts_with("/tree?"))));
let office_projection = query_local_search_index(
&root,
&format!("file://{}", root.display()),
@@ -4522,14 +4307,12 @@ mod tests {
false,
)
.expect("office projection");
assert!(
office_projection["results"]
.as_array()
.unwrap()
.iter()
.any(|item| item["resourceType"].as_str() == Some("office")
&& item["path"].as_str() == Some("office/report.xlsx"))
);
assert!(office_projection["results"]
.as_array()
.unwrap()
.iter()
.any(|item| item["resourceType"].as_str() == Some("office")
&& item["path"].as_str() == Some("office/report.xlsx")));
let pdf_projection = query_local_search_index(
&root,
&format!("file://{}", root.display()),
@@ -4542,14 +4325,12 @@ mod tests {
false,
)
.expect("pdf projection");
assert!(
pdf_projection["results"]
.as_array()
.unwrap()
.iter()
.any(|item| item["resourceType"].as_str() == Some("pdf")
&& item["path"].as_str() == Some("assets/spec.pdf"))
);
assert!(pdf_projection["results"]
.as_array()
.unwrap()
.iter()
.any(|item| item["resourceType"].as_str() == Some("pdf")
&& item["path"].as_str() == Some("assets/spec.pdf")));
let image_projection = query_local_search_index(
&root,
&format!("file://{}", root.display()),
@@ -4562,14 +4343,12 @@ mod tests {
false,
)
.expect("image projection");
assert!(
image_projection["results"]
.as_array()
.unwrap()
.iter()
.any(|item| item["resourceType"].as_str() == Some("image")
&& item["path"].as_str() == Some("assets/diagram.png"))
);
assert!(image_projection["results"]
.as_array()
.unwrap()
.iter()
.any(|item| item["resourceType"].as_str() == Some("image")
&& item["path"].as_str() == Some("assets/diagram.png")));
let _ = fs::remove_dir_all(&root);
}
@@ -5194,12 +4973,10 @@ mod tests {
let index = read_local_search_index(&root)
.expect("read index")
.expect("index exists");
assert!(
index
.documents
.iter()
.any(|document| document.path == "README.md")
);
assert!(index
.documents
.iter()
.any(|document| document.path == "README.md"));
let child = index
.documents
.iter()
@@ -5227,18 +5004,14 @@ mod tests {
let index = read_local_search_index(&root)
.expect("read index")
.expect("index exists");
assert!(
!index
.documents
.iter()
.any(|document| document.path == "docs/child.md")
);
assert!(
index
.documents
.iter()
.any(|document| document.path == "README.md")
);
assert!(!index
.documents
.iter()
.any(|document| document.path == "docs/child.md"));
assert!(index
.documents
.iter()
.any(|document| document.path == "README.md"));
let removed_evidence = query_evidence_sqlite_results(&root, "ChangedToken", None, 10)
.expect("removed evidence")
.expect("sqlite exists");
@@ -5251,7 +5024,7 @@ mod tests {
}
#[test]
fn local_search_ocr_sidecar_requires_include_ocr_and_returns_owner_page() {
fn local_search_ocr_sidecar_is_hidden_after_lightrag_retirement() {
let root = temp_root("mnote-local-search-ocr");
let root_uri = format!("file://{}", root.display());
let workspace_id = "local-ws-ocr";
@@ -5344,38 +5117,18 @@ mod tests {
true,
)
.expect("with ocr");
let result = with_ocr["results"]
.as_array()
.and_then(|items| items.first())
.expect("ocr result");
assert_eq!(
result["documentId"].as_str(),
Some("local-md:docs~2FPage.md")
);
assert_eq!(result["hasOcr"].as_bool(), Some(true));
assert_eq!(
result["ocrEvidence"]["sourceRootRelativePath"].as_str(),
Some("docs/Page.assets/photo.png")
);
assert_eq!(
result["ocrEvidence"]["ocrRootRelativePath"].as_str(),
Some("docs/Page.ocr/photo.png.ocr.md")
);
assert_eq!(with_ocr["results"].as_array().map(Vec::len), Some(0));
let index = read_local_search_index(&root)
.expect("read search index")
.expect("search index");
assert!(
!index
.documents
.iter()
.any(|document| document.path == "docs/Page.ocr/photo.png.ocr.md")
);
assert!(
!index
.documents
.iter()
.any(|document| document.path == "docs/Page.ocr/photo.png-704905.ocr.md")
);
assert!(!index
.documents
.iter()
.any(|document| document.path == "docs/Page.ocr/photo.png.ocr.md"));
assert!(!index
.documents
.iter()
.any(|document| document.path == "docs/Page.ocr/photo.png-704905.ocr.md"));
refresh_local_search_index_for_path(
&root,
&root_uri,
@@ -5386,12 +5139,10 @@ mod tests {
let refreshed_index = read_local_search_index(&root)
.expect("read refreshed search index")
.expect("refreshed search index");
assert!(
!refreshed_index
.documents
.iter()
.any(|document| document.path == "docs/Page.ocr/photo.png-704905.ocr.md")
);
assert!(!refreshed_index
.documents
.iter()
.any(|document| document.path == "docs/Page.ocr/photo.png-704905.ocr.md"));
let _ = fs::remove_dir_all(&root);
}
@@ -5492,21 +5243,15 @@ mod tests {
.expect("sqlite query")
.expect("sqlite exists");
assert!(results.len() >= 2);
assert!(
results
.iter()
.any(|result| result.source.owner_document_path == "README.md")
);
assert!(
results
.iter()
.any(|result| result.source.owner_document_path == "docs/child.md")
);
assert!(
results
.iter()
.all(|result| result.source.schema == "mnote.evidence_locator.v1")
);
assert!(results
.iter()
.any(|result| result.source.owner_document_path == "README.md"));
assert!(results
.iter()
.any(|result| result.source.owner_document_path == "docs/child.md"));
assert!(results
.iter()
.all(|result| result.source.schema == "mnote.evidence_locator.v1"));
let home_result = results
.iter()
.find(|result| result.source.owner_document_path == "README.md")
@@ -5664,16 +5409,12 @@ mod tests {
.expect("sqlite exists");
assert_eq!(context.len(), 2);
assert!(
context
.iter()
.all(|item| item.source.owner_document_path == "README.md")
);
assert!(
context
.iter()
.any(|item| item.quote.contains("ReadContextToken"))
);
assert!(context
.iter()
.all(|item| item.source.owner_document_path == "README.md"));
assert!(context
.iter()
.any(|item| item.quote.contains("ReadContextToken")));
let _ = fs::remove_dir_all(&root);
}
@@ -5771,18 +5512,16 @@ JSON
Some("docs/Page.assets/spec.pdf"),
"页面内搜索打开资源时应只限制到当前资源,而不是它的 owner Markdown"
);
assert!(
root.join("docs")
.join("Page.ocr")
.join("spec.pdf.parse.md")
.exists()
);
assert!(
root.join("docs")
.join("Page.ocr")
.join("spec.pdf.source-map.json")
.exists()
);
assert!(root
.join("docs")
.join("Page.ocr")
.join("spec.pdf.parse.md")
.exists());
assert!(root
.join("docs")
.join("Page.ocr")
.join("spec.pdf.source-map.json")
.exists());
let _ = fs::remove_dir_all(&root);
}
+34 -7
View File
@@ -11,9 +11,11 @@ mod hermes;
mod hermes_client;
mod hermes_tools;
mod kernel;
pub(crate) mod knowledge_rag;
mod local_folder_events;
mod local_folder_source;
mod local_markdown_parser;
#[allow(dead_code)]
mod local_ocr;
mod local_search_index;
mod media;
@@ -81,6 +83,21 @@ pub fn build_router(state: AppState) -> Router {
.route("/api/evidence/search", post(evidence::search))
.route("/api/evidence/read", post(evidence::read))
.route("/api/evidence/open", post(evidence::open))
.route("/api/knowledge-rag/status", get(knowledge_rag::status))
.route("/api/knowledge-rag/ingest", post(knowledge_rag::ingest))
.route("/api/knowledge-rag/query", post(knowledge_rag::query_rag))
.route(
"/api/knowledge-rag/open-reference",
post(knowledge_rag::open_reference),
)
.route(
"/api/knowledge-rag/delete-source",
post(knowledge_rag::delete_source),
)
.route(
"/api/knowledge-rag/prune-registry",
post(knowledge_rag::prune_registry),
)
.route(
"/mindmap/{doc_id}/{mindmap_id}",
get(mindmap_shell::mindmap_object_shell),
@@ -558,14 +575,24 @@ pub fn build_router(state: AppState) -> Router {
)
.route(
"/api/local-folder/ocr/jobs",
get(local_ocr::list_jobs)
.post(local_ocr::create_job)
.delete(local_ocr::delete_job),
any(knowledge_rag::retired_local_ocr_endpoint),
)
.route(
"/api/local-folder/ocr/status",
any(knowledge_rag::retired_local_ocr_endpoint),
)
.route(
"/api/local-folder/ocr/read",
any(knowledge_rag::retired_local_ocr_endpoint),
)
.route(
"/api/local-folder/ocr/insert",
any(knowledge_rag::retired_local_ocr_endpoint),
)
.route(
"/api/local-folder/ocr/delete",
any(knowledge_rag::retired_local_ocr_endpoint),
)
.route("/api/local-folder/ocr/status", get(local_ocr::status))
.route("/api/local-folder/ocr/read", get(local_ocr::read))
.route("/api/local-folder/ocr/insert", post(local_ocr::insert))
.route("/api/local-folder/ocr/delete", post(local_ocr::delete_job))
.route(
"/api/local-folder/workspaces/default",
post(local_folder_source::create_default_local_workspace),
+3 -1
View File
@@ -669,7 +669,9 @@ pub(crate) fn collect_filetree_render_rows(
.and_then(Value::as_str)
})
.map(str::trim)
.filter(|value| *value == "indexed" || *value == "failed")
.filter(|value| {
*value == "indexed" || *value == "indexing" || *value == "failed"
})
.map(ToOwned::to_owned),
selected,
})