Improve evidence search ranking and diagnostics

This commit is contained in:
lix-2026
2026-06-06 00:33:02 +08:00
parent 4dbd9a978b
commit 5157a5960c
6 changed files with 891 additions and 235 deletions
+18
View File
@@ -235,6 +235,8 @@ pub struct EvidenceSearchResult {
pub score: f64, pub score: f64,
pub source: EvidenceLocator, pub source: EvidenceLocator,
#[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
pub match_info: Option<EvidenceSearchMatchInfo>,
#[serde(skip_serializing_if = "Option::is_none")]
pub citation_url: Option<String>, pub citation_url: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
pub citation_label: Option<String>, pub citation_label: Option<String>,
@@ -242,12 +244,28 @@ pub struct EvidenceSearchResult {
pub citation_markdown: Option<String>, pub citation_markdown: Option<String>,
} }
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[serde(rename_all = "camelCase")]
pub struct EvidenceSearchMatchInfo {
#[serde(skip_serializing_if = "Option::is_none")]
pub rank: Option<u32>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub matched_terms: Vec<String>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub missing_terms: Vec<String>,
pub match_mode: String,
pub match_scope: String,
pub strong: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
pub struct EvidenceSearchResponse { pub struct EvidenceSearchResponse {
pub ok: bool, pub ok: bool,
#[serde(default)] #[serde(default)]
pub results: Vec<EvidenceSearchResult>, pub results: Vec<EvidenceSearchResult>,
#[serde(skip_serializing_if = "Option::is_none")]
pub diagnostics: Option<Value>,
} }
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+5 -4
View File
@@ -41,10 +41,11 @@ pub use editor::{
pub use evidence::{ pub use evidence::{
EvidenceBBox, EvidenceEdge, EvidenceLocator, EvidenceOpenAction, EvidenceOpenRequest, EvidenceBBox, EvidenceEdge, EvidenceLocator, EvidenceOpenAction, EvidenceOpenRequest,
EvidenceRange, EvidenceReadContext, EvidenceReadRequest, EvidenceReadResponse, EvidenceRange, EvidenceReadContext, EvidenceReadRequest, EvidenceReadResponse,
EvidenceResourceKind, EvidenceSearchMode, EvidenceSearchRequest, EvidenceSearchResponse, EvidenceResourceKind, EvidenceSearchMatchInfo, EvidenceSearchMode, EvidenceSearchRequest,
EvidenceSearchResult, EvidenceSearchScope, ParsedResourceArtifact, ResourceSourceMap, EvidenceSearchResponse, EvidenceSearchResult, EvidenceSearchScope, ParsedResourceArtifact,
SourceMapBlock, SourceMapBlockKind, SourceMapPage, SourceMapSection, SourceMapTextItem, ResourceSourceMap, SourceMapBlock, SourceMapBlockKind, SourceMapPage, SourceMapSection,
EVIDENCE_LOCATOR_SCHEMA, PARSED_RESOURCE_ARTIFACT_SCHEMA, RESOURCE_SOURCE_MAP_SCHEMA, SourceMapTextItem, EVIDENCE_LOCATOR_SCHEMA, PARSED_RESOURCE_ARTIFACT_SCHEMA,
RESOURCE_SOURCE_MAP_SCHEMA,
}; };
pub use kernel::{ pub use kernel::{
DocBufferDirtyState, DocumentBuffer, DocumentContentResult, DocumentReadEvidenceItem, DocBufferDirtyState, DocumentBuffer, DocumentContentResult, DocumentReadEvidenceItem,
@@ -2224,7 +2224,18 @@ import { createSidebarPageSettingsRuntime } from './sidebar-page-settings-runtim
return button instanceof HTMLElement && button.getAttribute('aria-checked') === 'true'; return button instanceof HTMLElement && button.getAttribute('aria-checked') === 'true';
} }
function highlightSearchText(value, query, exact) { function searchHighlightTerms(item, query) {
var info = item && item.matchInfo || item && item.evidence && item.evidence.matchInfo || null;
var terms = info && Array.isArray(info.matchedTerms) ? info.matchedTerms : [];
terms = terms.map(searchText).filter(Boolean);
if (!terms.length) {
terms = searchText(query).split(/[\s,;:,;:。、()[\]{}]+/).map(searchText).filter(Boolean);
}
terms.sort(function(left, right) { return right.length - left.length; });
return terms.filter(function(term, index) { return terms.indexOf(term) === index; });
}
function highlightSearchText(value, query, exact, terms) {
var text = searchText(value); var text = searchText(value);
var cleanQuery = searchText(query); var cleanQuery = searchText(query);
if (!text || !cleanQuery) return escapeHtml(text); if (!text || !cleanQuery) return escapeHtml(text);
@@ -2237,19 +2248,30 @@ import { createSidebarPageSettingsRuntime } from './sidebar-page-settings-runtim
'<mark>' + escapeHtml(text.slice(exactIndex, exactIndex + cleanQuery.length)) + '</mark>' + '<mark>' + escapeHtml(text.slice(exactIndex, exactIndex + cleanQuery.length)) + '</mark>' +
escapeHtml(text.slice(exactIndex + cleanQuery.length)); escapeHtml(text.slice(exactIndex + cleanQuery.length));
} }
var chars = Array.from(cleanQuery.replace(/\s+/g, '').toLowerCase()); terms = Array.isArray(terms) ? terms.map(searchText).filter(Boolean) : [];
if (!chars.length) return escapeHtml(text); if (!terms.length) return escapeHtml(text);
var next = 0; var ranges = [];
var html = ''; terms.forEach(function(term) {
Array.from(text).forEach(function(ch) { var lowerTerm = term.toLowerCase();
if (next < chars.length && ch.toLowerCase() === chars[next]) { var start = lower.indexOf(lowerTerm);
html += '<mark>' + escapeHtml(ch) + '</mark>'; while (start >= 0) {
next += 1; var end = start + lowerTerm.length;
} else { var overlaps = ranges.some(function(range) { return start < range.end && end > range.start; });
html += escapeHtml(ch); if (!overlaps) ranges.push({ start: start, end: end });
start = lower.indexOf(lowerTerm, end);
} }
}); });
return next >= chars.length ? html : escapeHtml(text); if (!ranges.length) return escapeHtml(text);
ranges.sort(function(left, right) { return left.start - right.start; });
var html = '';
var cursor = 0;
ranges.forEach(function(range) {
if (range.start > cursor) html += escapeHtml(text.slice(cursor, range.start));
html += '<mark>' + escapeHtml(text.slice(range.start, range.end)) + '</mark>';
cursor = range.end;
});
if (cursor < text.length) html += escapeHtml(text.slice(cursor));
return html;
} }
function searchResultEvidenceLocator(item) { function searchResultEvidenceLocator(item) {
@@ -2436,10 +2458,11 @@ import { createSidebarPageSettingsRuntime } from './sidebar-page-settings-runtim
var locatorAttr = locator ? escapeHtml(JSON.stringify(locator)) : ''; var locatorAttr = locator ? escapeHtml(JSON.stringify(locator)) : '';
var index = items.indexOf(item); var index = items.indexOf(item);
var exact = searchSwitchValue(overlay, 'exact'); var exact = searchSwitchValue(overlay, 'exact');
var highlightTerms = searchHighlightTerms(item, query);
return '<button type="button" class="wolai-search-result-row" data-testid="wolai-search-result-row" data-search-result-owner="rust-kernel" data-search-result-index="' + String(index) + '" data-document-id="' + escapeHtml(item.documentId || item.nodeId || item.id || '') + '" data-resource-type="' + escapeHtml(resourceType) + '"' + (locatorAttr ? ' data-evidence-locator="' + locatorAttr + '"' : '') + '>' + return '<button type="button" class="wolai-search-result-row" data-testid="wolai-search-result-row" data-search-result-owner="rust-kernel" data-search-result-index="' + String(index) + '" data-document-id="' + escapeHtml(item.documentId || item.nodeId || item.id || '') + '" data-resource-type="' + escapeHtml(resourceType) + '"' + (locatorAttr ? ' data-evidence-locator="' + locatorAttr + '"' : '') + '>' +
'<span class="material-symbols-outlined wolai-search-result-icon" data-icon="article" aria-hidden="true"></span>' + '<span class="material-symbols-outlined wolai-search-result-icon" data-icon="article" aria-hidden="true"></span>' +
'<span class="wolai-search-result-main"><span class="wolai-search-result-title">' + highlightSearchText(title, query, exact) + '</span>' + '<span class="wolai-search-result-main"><span class="wolai-search-result-title">' + highlightSearchText(title, query, exact, highlightTerms) + '</span>' +
'<span class="wolai-search-result-snippet">' + (snippet ? highlightSearchText(snippet, query, exact) : escapeHtml(path)) + '</span>' + '<span class="wolai-search-result-snippet">' + (snippet ? highlightSearchText(snippet, query, exact, highlightTerms) : escapeHtml(path)) + '</span>' +
'<span class="wolai-search-result-path"><span>' + escapeHtml(path) + '</span><span class="wolai-search-result-type">' + escapeHtml(resourceType) + '</span></span></span>' + '<span class="wolai-search-result-path"><span>' + escapeHtml(path) + '</span><span class="wolai-search-result-type">' + escapeHtml(resourceType) + '</span></span></span>' +
'</button>'; '</button>';
}).join(''); }).join('');
+138 -3
View File
@@ -61,19 +61,33 @@ pub(crate) async fn search_payload(
body.top_k, body.top_k,
)? { )? {
enrich_citation_links(&mut results); enrich_citation_links(&mut results);
return Ok(json!(EvidenceSearchResponse { ok: true, results })); return Ok(json!(EvidenceSearchResponse {
ok: true,
results,
diagnostics: None
}));
} }
} }
if let Some(mut results) = local_search_index::query_evidence_sqlite_results( if let Some(mut results) = local_search_index::query_evidence_sqlite_results_with_mode(
&root_path, &root_path,
&query, &query,
evidence_owner_filter, evidence_owner_filter,
body.top_k, body.top_k,
false,
)? { )? {
enrich_sqlite_evidence_results(&mut results, &root_path, &query); enrich_sqlite_evidence_results(&mut results, &root_path, &query);
enrich_citation_links(&mut results); enrich_citation_links(&mut results);
if !results.is_empty() || !query.is_empty() { if !results.is_empty() || !query.is_empty() {
let response = EvidenceSearchResponse { ok: true, results }; let diagnostics = Some(evidence_search_diagnostics(
&query,
&results,
"evidence_sqlite",
));
let response = EvidenceSearchResponse {
ok: true,
results,
diagnostics,
};
return Ok(json!(response)); return Ok(json!(response));
} }
} }
@@ -97,6 +111,7 @@ pub(crate) async fn search_payload(
enrich_citation_links(&mut results); enrich_citation_links(&mut results);
results results
}, },
diagnostics: None,
}; };
Ok(json!(response)) Ok(json!(response))
} }
@@ -154,6 +169,78 @@ fn enrich_citation_links(results: &mut [EvidenceSearchResult]) {
} }
} }
fn evidence_search_diagnostics(
query: &str,
results: &[EvidenceSearchResult],
backend: &str,
) -> Value {
let query_tokens = evidence_query_tokens(query);
let strong_match_count = results
.iter()
.filter(|result| result.match_info.as_ref().is_some_and(|info| info.strong))
.count();
let weak_match_count = results.len().saturating_sub(strong_match_count);
let reason = if results.is_empty() {
"no_evidence_match"
} else if strong_match_count == 0 {
"only_weak_partial_matches"
} else {
"ranked_evidence_matches"
};
json!({
"backend": backend,
"queryTokens": query_tokens,
"resultCount": results.len(),
"strongMatchCount": strong_match_count,
"weakMatchCount": weak_match_count,
"reason": reason,
"suggestedQueries": suggested_evidence_queries(query),
})
}
fn evidence_query_tokens(query: &str) -> Vec<String> {
query
.split(|ch: char| {
ch.is_whitespace()
|| matches!(
ch,
',' | ';'
| ':'
| ''
| ''
| ''
| '。'
| '、'
| '('
| ')'
| '['
| ']'
| '{'
| '}'
)
})
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
.collect()
}
fn suggested_evidence_queries(query: &str) -> Vec<String> {
let tokens = evidence_query_tokens(query);
let mut suggestions = Vec::new();
if tokens.len() > 1 {
suggestions.push(tokens.join(" "));
}
for token in tokens {
if token.chars().count() >= 3 {
suggestions.push(token.chars().take(2).collect::<String>());
}
}
suggestions.sort();
suggestions.dedup();
suggestions
}
fn citation_markdown_for_locator(locator: &EvidenceLocator) -> String { fn citation_markdown_for_locator(locator: &EvidenceLocator) -> String {
let label = citation_label_for_locator(locator); let label = citation_label_for_locator(locator);
let url = citation_url_for_locator(locator); let url = citation_url_for_locator(locator);
@@ -537,6 +624,7 @@ fn source_map_block_result(
quote: block.text.clone(), quote: block.text.clone(),
score, score,
source, source,
match_info: None,
citation_url: None, citation_url: None,
citation_label: None, citation_label: None,
citation_markdown: None, citation_markdown: None,
@@ -747,6 +835,11 @@ pub(crate) fn evidence_results_from_local_search(
.to_string(), .to_string(),
score: result.get("score").and_then(Value::as_f64).unwrap_or(0.0), score: result.get("score").and_then(Value::as_f64).unwrap_or(0.0),
source: locator, source: locator,
match_info: result
.get("evidence")
.and_then(|value| value.get("matchInfo"))
.cloned()
.and_then(|value| serde_json::from_value(value).ok()),
citation_url: None, citation_url: None,
citation_label: None, citation_label: None,
citation_markdown: None, citation_markdown: None,
@@ -1251,6 +1344,7 @@ mod tests {
quote: "NeedleToken 原文定位".into(), quote: "NeedleToken 原文定位".into(),
score: 1.0, score: 1.0,
source: locator, source: locator,
match_info: None,
citation_url: None, citation_url: None,
citation_label: None, citation_label: None,
citation_markdown: None, citation_markdown: None,
@@ -1285,6 +1379,47 @@ mod tests {
.contains("spec.pdf")); .contains("spec.pdf"));
} }
#[test]
fn evidence_search_diagnostics_marks_partial_matches() {
let results = vec![EvidenceSearchResult {
evidence_id: "ev1".into(),
quote: "羧酸需要被保护".into(),
score: 0.7,
source: EvidenceLocator::new(
"file:///workspace",
"local-md:docs~2FPage.md",
"docs/Page.md",
EvidenceResourceKind::Markdown,
EvidenceOpenAction {
action_type: "mnote.open_resource_locator".into(),
url: "/documents/local-md:docs~2FPage.md".into(),
params: json!({}),
},
),
match_info: Some(core_protocol::EvidenceSearchMatchInfo {
rank: Some(1),
matched_terms: vec!["羧酸".into(), "保护".into()],
missing_terms: vec!["保护基".into()],
match_mode: "cjk_partial".into(),
match_scope: "block".into(),
strong: false,
}),
citation_url: None,
citation_label: None,
citation_markdown: None,
}];
let diagnostics = evidence_search_diagnostics("羧酸 保护基", &results, "evidence_sqlite");
assert_eq!(diagnostics["backend"].as_str(), Some("evidence_sqlite"));
assert_eq!(diagnostics["queryTokens"].as_array().map(Vec::len), Some(2));
assert_eq!(diagnostics["strongMatchCount"].as_u64(), Some(0));
assert_eq!(diagnostics["weakMatchCount"].as_u64(), Some(1));
assert_eq!(
diagnostics["reason"].as_str(),
Some("only_weak_partial_matches")
);
}
#[test] #[test]
fn evidence_read_uses_source_map_context_and_section_blocks() { fn evidence_read_uses_source_map_context_and_section_blocks() {
let root = std::env::temp_dir().join(format!( let root = std::env::temp_dir().join(format!(
@@ -1,6 +1,6 @@
use crate::error::WebError; use crate::error::WebError;
use crate::evidence_parse::{ use crate::evidence_parse::{
liteparse_runtime_available, parse_liteparse_input_blocking, ParseInput, ParseProviderMode, ParseInput, ParseProviderMode, liteparse_runtime_available, parse_liteparse_input_blocking,
}; };
use crate::routes::local_folder_source::encode_local_id_segment; use crate::routes::local_folder_source::encode_local_id_segment;
use crate::routes::local_markdown_parser::{ use crate::routes::local_markdown_parser::{
@@ -9,12 +9,12 @@ use crate::routes::local_markdown_parser::{
use crate::routes::local_ocr; use crate::routes::local_ocr;
use control_plane::{ControlPlaneStore, UpsertUserInput, UpsertUserUiPreferenceInput}; use control_plane::{ControlPlaneStore, UpsertUserInput, UpsertUserUiPreferenceInput};
use core_protocol::{ use core_protocol::{
EvidenceLocator, EvidenceSearchResult, ParsedResourceArtifact, ResourceSourceMap, EvidenceLocator, EvidenceSearchMatchInfo, EvidenceSearchResult,
SourceMapBlock, PARSED_RESOURCE_ARTIFACT_SCHEMA, PARSED_RESOURCE_ARTIFACT_SCHEMA, ParsedResourceArtifact, ResourceSourceMap, SourceMapBlock,
}; };
use rusqlite::{params, Connection, OptionalExtension}; use rusqlite::{Connection, OptionalExtension, params};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use serde_json::{json, Value}; use serde_json::{Value, json};
use std::collections::BTreeSet; use std::collections::BTreeSet;
use std::fs; use std::fs;
use std::path::{Component, Path, PathBuf}; use std::path::{Component, Path, PathBuf};
@@ -671,6 +671,7 @@ pub(crate) fn refresh_local_search_index_if_scheduled_due_with_store(
.map(Some) .map(Some)
} }
#[cfg(test)]
pub(crate) fn query_evidence_sqlite_results( pub(crate) fn query_evidence_sqlite_results(
root_path: &Path, root_path: &Path,
query: &str, query: &str,
@@ -812,6 +813,7 @@ pub(crate) fn read_evidence_sqlite_context(
0.8 0.8
}, },
source: source.clone(), source: source.clone(),
match_info: None,
citation_url: None, citation_url: None,
citation_label: None, citation_label: None,
citation_markdown: None, citation_markdown: None,
@@ -901,6 +903,7 @@ fn graph_edge_row(row: &rusqlite::Row<'_>) -> rusqlite::Result<EvidenceSearchRes
block_id: source.block_id.or(Some(source_block_id)), block_id: source.block_id.or(Some(source_block_id)),
..source ..source
}, },
match_info: None,
citation_url: None, citation_url: None,
citation_label: None, citation_label: None,
citation_markdown: None, citation_markdown: None,
@@ -1059,43 +1062,76 @@ fn query_evidence_sqlite_fuzzy(
let scope_value = owner_document_id.or(resource_path); let scope_value = owner_document_id.or(resource_path);
let rows = match (scope_value, first_char_like.as_deref()) { let rows = match (scope_value, first_char_like.as_deref()) {
(Some(scope_value), Some(first_char_like)) => statement (Some(scope_value), Some(first_char_like)) => statement
.query_map(params![scope_value, first_char_like, scan_limit], |row| { .query_map(
evidence_result_from_sqlite_row(row, query) params![scope_value, first_char_like, scan_limit],
}) evidence_sqlite_row_parts,
)
.map_err(sqlite_error)? .map_err(sqlite_error)?
.collect::<Result<Vec<_>, _>>(), .collect::<Result<Vec<_>, _>>(),
(Some(scope_value), None) => statement (Some(scope_value), None) => statement
.query_map(params![scope_value, scan_limit], |row| { .query_map(params![scope_value, scan_limit], evidence_sqlite_row_parts)
evidence_result_from_sqlite_row(row, query)
})
.map_err(sqlite_error)? .map_err(sqlite_error)?
.collect::<Result<Vec<_>, _>>(), .collect::<Result<Vec<_>, _>>(),
(None, Some(first_char_like)) => statement (None, Some(first_char_like)) => statement
.query_map(params![first_char_like, scan_limit], |row| { .query_map(
evidence_result_from_sqlite_row(row, query) params![first_char_like, scan_limit],
}) evidence_sqlite_row_parts,
)
.map_err(sqlite_error)? .map_err(sqlite_error)?
.collect::<Result<Vec<_>, _>>(), .collect::<Result<Vec<_>, _>>(),
(None, None) => statement (None, None) => statement
.query_map(params![scan_limit], |row| { .query_map(params![scan_limit], evidence_sqlite_row_parts)
evidence_result_from_sqlite_row(row, query)
})
.map_err(sqlite_error)? .map_err(sqlite_error)?
.collect::<Result<Vec<_>, _>>(), .collect::<Result<Vec<_>, _>>(),
} }
.map_err(sqlite_error)?; .map_err(sqlite_error)?;
Ok(rows let mut scored = rows
.into_iter() .into_iter()
.filter(|result| { .filter_map(|(block_id, text, source)| {
fuzzy_search_match( let text_match = score_evidence_text_match(&text, query)?;
&normalize_search_text(&result.quote), Some(EvidenceSearchResult {
&normalize_search_text(query), evidence_id: block_id,
) quote: ocr_search_snippet_for_terms(&text, query, &text_match.matched_terms),
score: text_match.score,
source,
match_info: Some(text_match.into_match_info(None)),
citation_url: None,
citation_label: None,
citation_markdown: None,
})
})
.collect::<Vec<_>>();
scored.sort_by(|left, right| {
right
.score
.partial_cmp(&left.score)
.unwrap_or(std::cmp::Ordering::Equal)
});
Ok(scored
.into_iter()
.enumerate()
.map(|(index, mut result)| {
if let Some(match_info) = result.match_info.as_mut() {
match_info.rank = Some((index + 1) as u32);
}
result
}) })
.take(limit.max(1) as usize) .take(limit.max(1) as usize)
.collect()) .collect())
} }
fn evidence_sqlite_row_parts(
row: &rusqlite::Row<'_>,
) -> rusqlite::Result<(String, String, EvidenceLocator)> {
let block_id: String = row.get(0)?;
let text: String = row.get(1)?;
let locator_json: String = row.get(2)?;
let source = serde_json::from_str::<EvidenceLocator>(&locator_json).map_err(|error| {
rusqlite::Error::FromSqlConversionFailure(2, rusqlite::types::Type::Text, Box::new(error))
})?;
Ok((block_id, text, source))
}
fn evidence_result_from_sqlite_row( fn evidence_result_from_sqlite_row(
row: &rusqlite::Row<'_>, row: &rusqlite::Row<'_>,
query: &str, query: &str,
@@ -1107,15 +1143,20 @@ fn evidence_result_from_sqlite_row(
let source = serde_json::from_str::<EvidenceLocator>(&locator_json).map_err(|error| { let source = serde_json::from_str::<EvidenceLocator>(&locator_json).map_err(|error| {
rusqlite::Error::FromSqlConversionFailure(2, rusqlite::types::Type::Text, Box::new(error)) rusqlite::Error::FromSqlConversionFailure(2, rusqlite::types::Type::Text, Box::new(error))
})?; })?;
let text_match = score_evidence_text_match(&text, query);
Ok(EvidenceSearchResult { Ok(EvidenceSearchResult {
evidence_id: block_id, evidence_id: block_id,
quote: ocr_search_snippet(&text, query), quote: text_match
.as_ref()
.map(|text_match| ocr_search_snippet_for_terms(&text, query, &text_match.matched_terms))
.unwrap_or_else(|| ocr_search_snippet(&text, query)),
score: if rank == 0.0 { score: if rank == 0.0 {
1.0 1.0
} else { } else {
1.0 / (1.0 + rank.abs()) 1.0 / (1.0 + rank.abs())
}, },
source, source,
match_info: text_match.map(|text_match| text_match.into_match_info(None)),
citation_url: None, citation_url: None,
citation_label: None, citation_label: None,
citation_markdown: None, citation_markdown: None,
@@ -1952,6 +1993,13 @@ fn index_relative_path_is_included(relative_path: &str, include_paths: &[String]
}) })
} }
pub(crate) fn local_index_relative_path_is_included(
relative_path: &str,
settings: &LocalIndexSettings,
) -> bool {
index_relative_path_is_included(relative_path, &settings.include_paths)
}
fn collect_markdown_documents( fn collect_markdown_documents(
root_path: &Path, root_path: &Path,
current: &Path, current: &Path,
@@ -3605,7 +3653,7 @@ fn local_search_document_matches(
if exact { if exact {
haystack.contains(query) haystack.contains(query)
} else { } else {
fuzzy_search_match(&haystack, query) token_search_match(&haystack, query)
} }
} }
@@ -3629,7 +3677,7 @@ fn local_search_resource_matches(
if exact { if exact {
haystack.contains(query) haystack.contains(query)
} else { } else {
fuzzy_search_match(&haystack, query) token_search_match(&haystack, query)
} }
} }
@@ -3657,7 +3705,7 @@ fn local_search_ocr_matches(
if exact { if exact {
haystack.contains(query) haystack.contains(query)
} else { } else {
fuzzy_search_match(&haystack, query) token_search_match(&haystack, query)
} }
} }
@@ -3929,6 +3977,203 @@ fn search_document_hit(document: &LocalSearchDocument, query: &str) -> SearchDoc
} }
} }
#[derive(Debug, Clone)]
struct EvidenceQueryTerm {
display: String,
normalized: String,
alternatives: Vec<String>,
}
#[derive(Debug, Clone)]
struct EvidenceTextMatch {
score: f64,
matched_terms: Vec<String>,
missing_terms: Vec<String>,
match_mode: String,
strong: bool,
}
impl EvidenceTextMatch {
fn into_match_info(self, rank: Option<u32>) -> EvidenceSearchMatchInfo {
EvidenceSearchMatchInfo {
rank,
matched_terms: self.matched_terms,
missing_terms: self.missing_terms,
match_mode: self.match_mode,
match_scope: "block".into(),
strong: self.strong,
}
}
}
fn score_evidence_text_match(text: &str, query: &str) -> Option<EvidenceTextMatch> {
let normalized_text = normalize_search_text(text);
let normalized_query = normalize_search_text(query);
if normalized_query.is_empty() {
return None;
}
let terms = evidence_query_terms(&normalized_query);
if terms.is_empty() {
return None;
}
let phrase_exact = normalized_text.contains(&normalized_query);
let mut matched_terms = Vec::new();
let mut missing_terms = Vec::new();
let mut exact_count = 0usize;
let mut partial_count = 0usize;
let mut positions = Vec::new();
for term in &terms {
if let Some(index) = normalized_text.find(&term.normalized) {
exact_count += 1;
positions.push(index);
push_unique(&mut matched_terms, term.display.clone());
continue;
}
let partial_matches = term
.alternatives
.iter()
.filter_map(|alternative| {
normalized_text
.find(alternative)
.map(|index| (alternative.clone(), index))
})
.collect::<Vec<_>>();
let partial_allowed = if terms.len() > 1 {
!partial_matches.is_empty()
} else {
partial_matches.len() >= 2 || term.normalized.chars().count() <= 2
};
if partial_allowed {
partial_count += 1;
if let Some((_, index)) = partial_matches.first() {
positions.push(*index);
}
for (alternative, _) in partial_matches {
push_unique(&mut matched_terms, alternative);
}
push_unique(&mut missing_terms, term.display.clone());
} else {
return None;
}
}
let term_count = terms.len().max(1) as f64;
let coverage = (exact_count as f64 + partial_count as f64 * 0.45) / term_count;
let proximity = if positions.len() > 1 {
let min = positions.iter().min().copied().unwrap_or(0);
let max = positions.iter().max().copied().unwrap_or(min);
let span = max.saturating_sub(min);
if span <= 80 {
0.1
} else if span <= 240 {
0.05
} else {
0.0
}
} else {
0.0
};
let score = if phrase_exact {
1.0
} else {
(coverage * 0.85 + proximity).min(0.95)
};
let match_mode = if phrase_exact {
"exact_phrase"
} else if partial_count == 0 {
"all_terms"
} else {
"cjk_partial"
};
Some(EvidenceTextMatch {
score,
matched_terms,
missing_terms,
match_mode: match_mode.into(),
strong: phrase_exact || partial_count == 0 && exact_count == terms.len(),
})
}
fn evidence_query_terms(query: &str) -> Vec<EvidenceQueryTerm> {
split_search_query_tokens(query)
.into_iter()
.filter(|token| !token.is_empty())
.map(|token| {
let alternatives = cjk_token_bigrams(&token)
.into_iter()
.filter(|alternative| alternative != &token)
.collect::<Vec<_>>();
EvidenceQueryTerm {
display: token.clone(),
normalized: token,
alternatives,
}
})
.collect()
}
fn split_search_query_tokens(query: &str) -> Vec<String> {
let mut tokens = Vec::new();
let mut current = String::new();
for ch in query.chars() {
if ch.is_whitespace() || is_search_separator(ch) {
if !current.is_empty() {
tokens.push(std::mem::take(&mut current));
}
} else {
current.push(ch);
}
}
if !current.is_empty() {
tokens.push(current);
}
tokens
}
fn is_search_separator(ch: char) -> bool {
matches!(
ch,
',' | ';' | ':' | '' | '' | '' | '。' | '、' | '(' | ')' | '[' | ']' | '{' | '}'
)
}
fn cjk_token_bigrams(token: &str) -> Vec<String> {
let chars = token.chars().collect::<Vec<_>>();
if chars.len() < 3 || !chars.iter().any(|ch| is_cjk_char(*ch)) {
return Vec::new();
}
chars
.windows(2)
.filter_map(|window| {
if window.iter().any(|ch| is_cjk_stop_char(*ch)) {
return None;
}
Some(window.iter().collect::<String>())
})
.collect()
}
fn is_cjk_char(ch: char) -> bool {
matches!(
ch as u32,
0x3400..=0x4DBF | 0x4E00..=0x9FFF | 0xF900..=0xFAFF
)
}
fn is_cjk_stop_char(ch: char) -> bool {
matches!(
ch,
'的' | '了' | '和' | '与' | '及' | '在' | '是' | '为' | '对' | '中'
)
}
fn push_unique(values: &mut Vec<String>, value: String) {
if !value.is_empty() && !values.iter().any(|existing| existing == &value) {
values.push(value);
}
}
fn ocr_search_snippet(body: &str, query: &str) -> String { fn ocr_search_snippet(body: &str, query: &str) -> String {
let normalized_body = body.replace('\n', " "); let normalized_body = body.replace('\n', " ");
let normalized_query = query.trim(); let normalized_query = query.trim();
@@ -3958,26 +4203,37 @@ fn ocr_search_snippet(body: &str, query: &str) -> String {
} }
} }
fn fuzzy_search_match(haystack: &str, query: &str) -> bool { fn ocr_search_snippet_for_terms(body: &str, query: &str, terms: &[String]) -> String {
if query.is_empty() { let normalized_body = body.replace('\n', " ");
return false; let normalized_query = query.trim();
} if !normalized_query.is_empty() {
if haystack.contains(query) { let lower = normalized_body.to_lowercase();
return true; let lower_query = normalized_query.to_lowercase();
} if let Some(byte_index) = lower.find(&lower_query) {
let mut query_chars = query.chars().filter(|ch| !ch.is_whitespace()); return snippet_from_byte_index(&normalized_body, byte_index, 180);
let Some(mut wanted) = query_chars.next() else {
return false;
};
for ch in haystack.chars().filter(|ch| !ch.is_whitespace()) {
if ch == wanted {
match query_chars.next() {
Some(next) => wanted = next,
None => return true,
}
} }
} }
false let lower = normalized_body.to_lowercase();
for term in terms {
let term = term.trim().to_lowercase();
if term.is_empty() {
continue;
}
if let Some(byte_index) = lower.find(&term) {
return snippet_from_byte_index(&normalized_body, byte_index, 180);
}
}
ocr_search_snippet(body, query)
}
fn snippet_from_byte_index(body: &str, byte_index: usize, len: usize) -> String {
let start = body[..byte_index]
.char_indices()
.rev()
.nth(40)
.map(|(idx, _)| idx)
.unwrap_or(0);
body[start..].chars().take(len).collect()
} }
fn fuzzy_search_start_byte(haystack: &str, query: &str) -> Option<usize> { fn fuzzy_search_start_byte(haystack: &str, query: &str) -> Option<usize> {
@@ -4012,6 +4268,10 @@ fn fuzzy_search_start_byte(haystack: &str, query: &str) -> Option<usize> {
None None
} }
fn token_search_match(haystack: &str, query: &str) -> bool {
score_evidence_text_match(haystack, query).is_some()
}
fn is_markdown_path(path: &Path) -> bool { fn is_markdown_path(path: &Path) -> bool {
path.extension() path.extension()
.and_then(|value| value.to_str()) .and_then(|value| value.to_str())
@@ -4034,6 +4294,7 @@ fn resource_type_from_path(path: &Path) -> Option<&'static str> {
.unwrap_or_default() .unwrap_or_default()
.to_lowercase(); .to_lowercase();
match extension.as_str() { match extension.as_str() {
"png" | "jpg" | "jpeg" | "webp" | "bmp" | "gif" | "tif" | "tiff" | "svg" => Some("image"),
"pdf" => Some("pdf"), "pdf" => Some("pdf"),
"doc" | "docx" | "odt" | "ppt" | "pptx" | "odp" | "xls" | "xlsx" | "ods" => Some("office"), "doc" | "docx" | "odt" | "ppt" | "pptx" | "odp" | "xls" | "xlsx" | "ods" => Some("office"),
_ => None, _ => None,
@@ -4092,7 +4353,10 @@ mod tests {
.expect("write mindmap"); .expect("write mindmap");
fs::create_dir_all(root.join("assets")).expect("create assets"); fs::create_dir_all(root.join("assets")).expect("create assets");
fs::write(root.join("assets").join("spec.pdf"), b"%PDF-1.4\n").expect("write pdf"); fs::write(root.join("assets").join("spec.pdf"), b"%PDF-1.4\n").expect("write pdf");
fs::write(root.join("assets").join("diagram.png"), b"png").expect("write image");
fs::write(root.join("office").join("report.xlsx"), b"office bytes").expect("write office"); fs::write(root.join("office").join("report.xlsx"), b"office bytes").expect("write office");
write_local_index_settings(&root, &[String::from(".")], None, None, None, None)
.expect("settings");
let projection = query_local_search_index( let projection = query_local_search_index(
&root, &root,
@@ -4111,41 +4375,55 @@ mod tests {
.iter() .iter()
.find(|item| item["documentId"].as_str() == Some("local-md:README.md")) .find(|item| item["documentId"].as_str() == Some("local-md:README.md"))
.expect("home result"); .expect("home result");
assert!(home["tags"] assert!(
.as_array() home["tags"]
.unwrap() .as_array()
.iter() .unwrap()
.any(|tag| tag.as_str() == Some("alpha"))); .iter()
assert!(home["backlinks"] .any(|tag| tag.as_str() == Some("alpha"))
.as_array() );
.unwrap() assert!(
.iter() home["backlinks"]
.any(|link| link.as_str() == Some("Daily"))); .as_array()
assert!(home["backlinks"] .unwrap()
.as_array() .iter()
.unwrap() .any(|link| link.as_str() == Some("Daily"))
.iter() );
.any(|link| link.as_str() == Some("docs/child.md"))); assert!(
assert!(home["resourceRefs"] home["backlinks"]
.as_array() .as_array()
.unwrap() .unwrap()
.iter() .iter()
.any(|reference| reference.as_str() == Some("assets/spec.pdf"))); .any(|link| link.as_str() == Some("docs/child.md"))
assert!(home["resourceRefs"] );
.as_array() assert!(
.unwrap() home["resourceRefs"]
.iter() .as_array()
.any(|reference| reference.as_str() == Some("maps/idea.mindmap.json"))); .unwrap()
assert!(home["resourceRefs"] .iter()
.as_array() .any(|reference| reference.as_str() == Some("assets/spec.pdf"))
.unwrap() );
.iter() assert!(
.any(|reference| reference.as_str() == Some("office/report.xlsx"))); home["resourceRefs"]
assert!(home["publicPath"] .as_array()
.as_str() .unwrap()
.is_some_and(|path| path.starts_with( .iter()
"/documents/local-md:README.md?sourceKind=local_folder&rootUri=file%3A%2F%2F" .any(|reference| reference.as_str() == Some("maps/idea.mindmap.json"))
))); );
assert!(
home["resourceRefs"]
.as_array()
.unwrap()
.iter()
.any(|reference| reference.as_str() == Some("office/report.xlsx"))
);
assert!(
home["publicPath"]
.as_str()
.is_some_and(|path| path.starts_with(
"/documents/local-md:README.md?sourceKind=local_folder&rootUri=file%3A%2F%2F"
))
);
let evidence_db = root.join(".mnote").join("index").join("evidence.sqlite"); let evidence_db = root.join(".mnote").join("index").join("evidence.sqlite");
let connection = Connection::open(&evidence_db).expect("open evidence sqlite"); let connection = Connection::open(&evidence_db).expect("open evidence sqlite");
let markdown_edge_count: i64 = connection let markdown_edge_count: i64 = connection
@@ -4201,11 +4479,12 @@ mod tests {
Some("local-mdid:child-page") Some("local-mdid:child-page")
); );
assert!(root assert!(
.join(".mnote") root.join(".mnote")
.join("index") .join("index")
.join("search-index.json") .join("search-index.json")
.exists()); .exists()
);
let mindmap_projection = query_local_search_index( let mindmap_projection = query_local_search_index(
&root, &root,
&format!("file://{}", root.display()), &format!("file://{}", root.display()),
@@ -4218,19 +4497,19 @@ mod tests {
false, false,
) )
.expect("mindmap projection"); .expect("mindmap projection");
assert!(mindmap_projection["results"] assert!(
.as_array() mindmap_projection["results"]
.unwrap() .as_array()
.iter() .unwrap()
.any(|item| item["resourceType"].as_str() == Some("mindmap") .iter()
&& item["path"].as_str() == Some("maps/idea.mindmap.json") .any(|item| item["resourceType"].as_str() == Some("mindmap")
&& item["publicPath"] && item["path"].as_str() == Some("maps/idea.mindmap.json")
.as_str() && item["publicPath"].as_str().is_some_and(|path| path
.is_some_and(|path| path
.starts_with("/?treeView=filetree&sourceKind=local_folder&rootUri=")) .starts_with("/?treeView=filetree&sourceKind=local_folder&rootUri="))
&& item["publicPath"] && item["publicPath"]
.as_str() .as_str()
.is_some_and(|path| !path.starts_with("/tree?")))); .is_some_and(|path| !path.starts_with("/tree?")))
);
let office_projection = query_local_search_index( let office_projection = query_local_search_index(
&root, &root,
&format!("file://{}", root.display()), &format!("file://{}", root.display()),
@@ -4243,12 +4522,14 @@ mod tests {
false, false,
) )
.expect("office projection"); .expect("office projection");
assert!(office_projection["results"] assert!(
.as_array() office_projection["results"]
.unwrap() .as_array()
.iter() .unwrap()
.any(|item| item["resourceType"].as_str() == Some("office") .iter()
&& item["path"].as_str() == Some("office/report.xlsx"))); .any(|item| item["resourceType"].as_str() == Some("office")
&& item["path"].as_str() == Some("office/report.xlsx"))
);
let pdf_projection = query_local_search_index( let pdf_projection = query_local_search_index(
&root, &root,
&format!("file://{}", root.display()), &format!("file://{}", root.display()),
@@ -4261,12 +4542,34 @@ mod tests {
false, false,
) )
.expect("pdf projection"); .expect("pdf projection");
assert!(pdf_projection["results"] assert!(
.as_array() pdf_projection["results"]
.unwrap() .as_array()
.iter() .unwrap()
.any(|item| item["resourceType"].as_str() == Some("pdf") .iter()
&& item["path"].as_str() == Some("assets/spec.pdf"))); .any(|item| item["resourceType"].as_str() == Some("pdf")
&& item["path"].as_str() == Some("assets/spec.pdf"))
);
let image_projection = query_local_search_index(
&root,
&format!("file://{}", root.display()),
"local-ws-test",
"diagram.png",
None,
10,
false,
false,
false,
)
.expect("image projection");
assert!(
image_projection["results"]
.as_array()
.unwrap()
.iter()
.any(|item| item["resourceType"].as_str() == Some("image")
&& item["path"].as_str() == Some("assets/diagram.png"))
);
let _ = fs::remove_dir_all(&root); let _ = fs::remove_dir_all(&root);
} }
@@ -4891,10 +5194,12 @@ mod tests {
let index = read_local_search_index(&root) let index = read_local_search_index(&root)
.expect("read index") .expect("read index")
.expect("index exists"); .expect("index exists");
assert!(index assert!(
.documents index
.iter() .documents
.any(|document| document.path == "README.md")); .iter()
.any(|document| document.path == "README.md")
);
let child = index let child = index
.documents .documents
.iter() .iter()
@@ -4922,14 +5227,18 @@ mod tests {
let index = read_local_search_index(&root) let index = read_local_search_index(&root)
.expect("read index") .expect("read index")
.expect("index exists"); .expect("index exists");
assert!(!index assert!(
.documents !index
.iter() .documents
.any(|document| document.path == "docs/child.md")); .iter()
assert!(index .any(|document| document.path == "docs/child.md")
.documents );
.iter() assert!(
.any(|document| document.path == "README.md")); index
.documents
.iter()
.any(|document| document.path == "README.md")
);
let removed_evidence = query_evidence_sqlite_results(&root, "ChangedToken", None, 10) let removed_evidence = query_evidence_sqlite_results(&root, "ChangedToken", None, 10)
.expect("removed evidence") .expect("removed evidence")
.expect("sqlite exists"); .expect("sqlite exists");
@@ -4990,6 +5299,8 @@ mod tests {
.expect("serialize index"), .expect("serialize index"),
) )
.expect("ocr index"); .expect("ocr index");
write_local_index_settings(&root, &[String::from(".")], None, None, None, None)
.expect("settings");
let without_ocr = query_local_search_index( let without_ocr = query_local_search_index(
&root, &root,
@@ -5053,14 +5364,18 @@ mod tests {
let index = read_local_search_index(&root) let index = read_local_search_index(&root)
.expect("read search index") .expect("read search index")
.expect("search index"); .expect("search index");
assert!(!index assert!(
.documents !index
.iter() .documents
.any(|document| document.path == "docs/Page.ocr/photo.png.ocr.md")); .iter()
assert!(!index .any(|document| document.path == "docs/Page.ocr/photo.png.ocr.md")
.documents );
.iter() assert!(
.any(|document| document.path == "docs/Page.ocr/photo.png-704905.ocr.md")); !index
.documents
.iter()
.any(|document| document.path == "docs/Page.ocr/photo.png-704905.ocr.md")
);
refresh_local_search_index_for_path( refresh_local_search_index_for_path(
&root, &root,
&root_uri, &root_uri,
@@ -5071,10 +5386,12 @@ mod tests {
let refreshed_index = read_local_search_index(&root) let refreshed_index = read_local_search_index(&root)
.expect("read refreshed search index") .expect("read refreshed search index")
.expect("refreshed search index"); .expect("refreshed search index");
assert!(!refreshed_index assert!(
.documents !refreshed_index
.iter() .documents
.any(|document| document.path == "docs/Page.ocr/photo.png-704905.ocr.md")); .iter()
.any(|document| document.path == "docs/Page.ocr/photo.png-704905.ocr.md")
);
let _ = fs::remove_dir_all(&root); let _ = fs::remove_dir_all(&root);
} }
@@ -5175,15 +5492,21 @@ mod tests {
.expect("sqlite query") .expect("sqlite query")
.expect("sqlite exists"); .expect("sqlite exists");
assert!(results.len() >= 2); assert!(results.len() >= 2);
assert!(results assert!(
.iter() results
.any(|result| result.source.owner_document_path == "README.md")); .iter()
assert!(results .any(|result| result.source.owner_document_path == "README.md")
.iter() );
.any(|result| result.source.owner_document_path == "docs/child.md")); assert!(
assert!(results results
.iter() .iter()
.all(|result| result.source.schema == "mnote.evidence_locator.v1")); .any(|result| result.source.owner_document_path == "docs/child.md")
);
assert!(
results
.iter()
.all(|result| result.source.schema == "mnote.evidence_locator.v1")
);
let home_result = results let home_result = results
.iter() .iter()
.find(|result| result.source.owner_document_path == "README.md") .find(|result| result.source.owner_document_path == "README.md")
@@ -5265,6 +5588,55 @@ mod tests {
let _ = fs::remove_dir_all(&root); let _ = fs::remove_dir_all(&root);
} }
#[test]
fn evidence_sqlite_fuzzy_uses_cjk_terms_without_overrequiring_suffix() {
let root = temp_root("mnote-evidence-sqlite-cjk-fuzzy");
let root_uri = format!("file://{}", root.display());
let workspace_id = "local-ws-evidence-cjk";
fs::write(
root.join("README.md"),
"# 有机合成\n羧酸需要被保护有许多原因,常见做法是形成稳定酯类后再脱保护。\n",
)
.expect("write home");
write_local_index_settings(&root, &[String::from(".")], None, None, None, None)
.expect("settings");
refresh_local_search_index(&root, &root_uri, workspace_id).expect("refresh");
let exact = query_evidence_sqlite_results_with_mode(&root, "羧酸 保护基", None, 10, true)
.expect("exact sqlite query")
.expect("sqlite exists");
assert!(exact.is_empty(), "精确短语不应伪造不存在的同串命中");
let fuzzy = query_evidence_sqlite_results_with_mode(&root, "羧酸 保护基", None, 10, false)
.expect("fuzzy sqlite query")
.expect("sqlite exists");
assert_eq!(fuzzy.len(), 1);
let info = fuzzy[0].match_info.as_ref().expect("match info");
assert_eq!(info.match_mode, "cjk_partial");
assert!(!info.strong);
assert!(info.matched_terms.iter().any(|term| term == "羧酸"));
assert!(info.matched_terms.iter().any(|term| term == "保护"));
assert!(info.missing_terms.iter().any(|term| term == "保护基"));
assert!(fuzzy[0].quote.contains("羧酸"));
let projection = query_local_search_index(
&root,
&root_uri,
workspace_id,
"羧酸 保护基",
None,
10,
false,
false,
false,
)
.expect("local search projection");
assert_eq!(projection["results"].as_array().map(Vec::len), Some(1));
let _ = fs::remove_dir_all(&root);
}
#[test] #[test]
fn evidence_sqlite_read_context_returns_anchor_block() { fn evidence_sqlite_read_context_returns_anchor_block() {
let root = temp_root("mnote-evidence-sqlite-read-context"); let root = temp_root("mnote-evidence-sqlite-read-context");
@@ -5292,12 +5664,16 @@ mod tests {
.expect("sqlite exists"); .expect("sqlite exists");
assert_eq!(context.len(), 2); assert_eq!(context.len(), 2);
assert!(context assert!(
.iter() context
.all(|item| item.source.owner_document_path == "README.md")); .iter()
assert!(context .all(|item| item.source.owner_document_path == "README.md")
.iter() );
.any(|item| item.quote.contains("ReadContextToken"))); assert!(
context
.iter()
.any(|item| item.quote.contains("ReadContextToken"))
);
let _ = fs::remove_dir_all(&root); let _ = fs::remove_dir_all(&root);
} }
@@ -5395,16 +5771,18 @@ JSON
Some("docs/Page.assets/spec.pdf"), Some("docs/Page.assets/spec.pdf"),
"页面内搜索打开资源时应只限制到当前资源,而不是它的 owner Markdown" "页面内搜索打开资源时应只限制到当前资源,而不是它的 owner Markdown"
); );
assert!(root assert!(
.join("docs") root.join("docs")
.join("Page.ocr") .join("Page.ocr")
.join("spec.pdf.parse.md") .join("spec.pdf.parse.md")
.exists()); .exists()
assert!(root );
.join("docs") assert!(
.join("Page.ocr") root.join("docs")
.join("spec.pdf.source-map.json") .join("Page.ocr")
.exists()); .join("spec.pdf.source-map.json")
.exists()
);
let _ = fs::remove_dir_all(&root); let _ = fs::remove_dir_all(&root);
} }
+157 -56
View File
@@ -9,14 +9,14 @@ use crate::routes::query_support::{
use crate::routes::web_shell::load_sidebar_tree_html; use crate::routes::web_shell::load_sidebar_tree_html;
use crate::routes::{evidence, local_folder_source, local_search_index}; use crate::routes::{evidence, local_folder_source, local_search_index};
use crate::ssr::pages::search::SearchPage; use crate::ssr::pages::search::SearchPage;
use axum::Json;
use axum::extract::{Extension, Query, State}; use axum::extract::{Extension, Query, State};
use axum::http::{HeaderMap, HeaderName, HeaderValue, StatusCode}; use axum::http::{HeaderMap, HeaderName, HeaderValue, StatusCode};
use axum::response::{Html, IntoResponse, Response}; use axum::response::{Html, IntoResponse, Response};
use axum::Json;
use bridge_runtime::RuntimeQueryEnvelopeWire; use bridge_runtime::RuntimeQueryEnvelopeWire;
use core_protocol::{EvidenceSearchMode, EvidenceSearchResult}; use core_protocol::{EvidenceSearchMode, EvidenceSearchResult};
use serde::Deserialize; use serde::Deserialize;
use serde_json::{json, Value}; use serde_json::{Value, json};
const HEADER_MNOTE_WEB_OWNER: &str = "x-mnote-web-owner"; const HEADER_MNOTE_WEB_OWNER: &str = "x-mnote-web-owner";
const HEADER_QUERY_NAME: &str = "x-query-name"; const HEADER_QUERY_NAME: &str = "x-query-name";
@@ -239,6 +239,16 @@ pub async fn documents(
filters.exact.unwrap_or(false), filters.exact.unwrap_or(false),
)? )?
.unwrap_or_default() .unwrap_or_default()
.into_iter()
.filter(|evidence| {
let path = evidence
.source
.resource_path
.as_deref()
.unwrap_or(evidence.source.owner_document_path.as_str());
local_search_index::local_index_relative_path_is_included(path, &user_settings)
})
.collect::<Vec<_>>()
} else { } else {
Vec::new() Vec::new()
}; };
@@ -438,12 +448,6 @@ pub async fn update_local_index_settings(
&effective_workspace_id, &effective_workspace_id,
&root_path, &root_path,
)?; )?;
let refreshed = local_search_index::refresh_local_search_index_with_settings(
&root_path,
root_uri,
&effective_workspace_id,
&effective_settings,
)?;
let status = local_search_index::local_index_status_with_settings( let status = local_search_index::local_index_status_with_settings(
&root_path, &root_path,
root_uri, root_uri,
@@ -459,7 +463,6 @@ pub async fn update_local_index_settings(
Json(json!({ Json(json!({
"ok": true, "ok": true,
"settings": settings, "settings": settings,
"index": refreshed,
"result": status, "result": status,
"meta": { "meta": {
"owner": "mnote-web", "owner": "mnote-web",
@@ -481,26 +484,72 @@ fn merge_local_search_with_evidence_results(
if direct_evidence_results.is_empty() { if direct_evidence_results.is_empty() {
return (result, evidence_results); return (result, evidence_results);
} }
let mut result_items = result let original_items = result
.get("results") .get("results")
.and_then(Value::as_array) .and_then(Value::as_array)
.cloned() .cloned()
.unwrap_or_default(); .unwrap_or_default();
let mut seen_evidence_ids = evidence_results let original_evidence_results = std::mem::take(&mut evidence_results);
.iter() let mut result_items = Vec::new();
.map(|item| item.evidence_id.clone()) let mut merged_evidence_results = Vec::new();
.collect::<std::collections::HashSet<_>>(); let mut seen_result_ids = std::collections::HashSet::new();
let mut seen_evidence_ids = std::collections::HashSet::new();
let mut seen_paths = std::collections::HashSet::new();
for evidence in direct_evidence_results { for evidence in direct_evidence_results {
if result_items.len() >= limit || !seen_evidence_ids.insert(evidence.evidence_id.clone()) { if result_items.len() >= limit || !seen_evidence_ids.insert(evidence.evidence_id.clone()) {
continue; continue;
} }
result_items.push(search_result_from_evidence(&evidence)); let item = search_result_from_evidence(&evidence);
evidence_results.push(evidence); if let Some(id) = item.get("id").and_then(Value::as_str) {
seen_result_ids.insert(id.to_string());
}
if let Some(path) = search_result_dedupe_path(&item) {
seen_paths.insert(path);
}
result_items.push(item);
merged_evidence_results.push(evidence);
}
for (index, item) in original_items.into_iter().enumerate() {
if result_items.len() >= limit {
break;
}
let item_id = item
.get("id")
.and_then(Value::as_str)
.map(ToOwned::to_owned)
.unwrap_or_default();
if !item_id.is_empty() && !seen_result_ids.insert(item_id) {
continue;
}
if let Some(path) = search_result_dedupe_path(&item) {
if !seen_paths.insert(path) {
continue;
}
}
if let Some(evidence) = original_evidence_results.get(index).cloned() {
merged_evidence_results.push(evidence);
}
result_items.push(item);
} }
if let Some(map) = result.as_object_mut() { if let Some(map) = result.as_object_mut() {
map.insert("results".into(), Value::Array(result_items)); map.insert("results".into(), Value::Array(result_items));
} }
(result, evidence_results) (result, merged_evidence_results)
}
fn search_result_dedupe_path(item: &Value) -> Option<String> {
let source_kind = item
.get("sourceKind")
.and_then(Value::as_str)
.unwrap_or_default();
if source_kind != "local_folder" {
return None;
}
item.get("path")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
} }
fn search_result_from_evidence(evidence: &EvidenceSearchResult) -> Value { fn search_result_from_evidence(evidence: &EvidenceSearchResult) -> Value {
@@ -533,6 +582,7 @@ fn search_result_from_evidence(evidence: &EvidenceSearchResult) -> Value {
"rootUri": source.root_uri, "rootUri": source.root_uri,
"snippet": evidence.quote, "snippet": evidence.quote,
"score": evidence.score, "score": evidence.score,
"matchInfo": evidence.match_info,
"publicPath": source.open_action.url, "publicPath": source.open_action.url,
"evidence": evidence_value, "evidence": evidence_value,
"source": { "source": {
@@ -890,12 +940,12 @@ fn stamp_search_headers(headers: &mut HeaderMap) {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::app::{build_app, AppConfig, AppState}; use crate::app::{AppConfig, AppState, build_app};
use crate::routes::local_search_index; use crate::routes::local_search_index;
use axum::body::{to_bytes, Body}; use axum::body::{Body, to_bytes};
use axum::http::{Request, StatusCode}; use axum::http::{Request, StatusCode};
use control_plane::{DirectoryGrantInput, UpsertUserInput}; use control_plane::{DirectoryGrantInput, UpsertUserInput};
use serde_json::{json, Value}; use serde_json::{Value, json};
use std::fs; use std::fs;
use tower::util::ServiceExt; use tower::util::ServiceExt;
@@ -1152,26 +1202,33 @@ mod tests {
Some("README.md") Some("README.md")
); );
assert!(!payload["evidence"].as_array().expect("evidence").is_empty()); assert!(!payload["evidence"].as_array().expect("evidence").is_empty());
assert!(home["tags"] assert!(
.as_array() home["tags"]
.unwrap() .as_array()
.iter() .unwrap()
.any(|tag| tag.as_str() == Some("alpha"))); .iter()
assert!(home["backlinks"] .any(|tag| tag.as_str() == Some("alpha"))
.as_array() );
.unwrap() assert!(
.iter() home["backlinks"]
.any(|link| link.as_str() == Some("Daily"))); .as_array()
assert!(home["resourceRefs"] .unwrap()
.as_array() .iter()
.unwrap() .any(|link| link.as_str() == Some("Daily"))
.iter() );
.any(|reference| reference.as_str() == Some("assets/spec.pdf"))); assert!(
assert!(root home["resourceRefs"]
.join(".mnote") .as_array()
.join("index") .unwrap()
.join("search-index.json") .iter()
.exists()); .any(|reference| reference.as_str() == Some("assets/spec.pdf"))
);
assert!(
root.join(".mnote")
.join("index")
.join("search-index.json")
.exists()
);
let evidence_db = root.join(".mnote").join("index").join("evidence.sqlite"); let evidence_db = root.join(".mnote").join("index").join("evidence.sqlite");
assert!(evidence_db.exists(), "evidence sqlite should be built"); assert!(evidence_db.exists(), "evidence sqlite should be built");
let connection = rusqlite::Connection::open(&evidence_db).expect("open evidence sqlite"); let connection = rusqlite::Connection::open(&evidence_db).expect("open evidence sqlite");
@@ -1201,11 +1258,13 @@ mod tests {
locator["schema"].as_str(), locator["schema"].as_str(),
Some("mnote.evidence_locator.v1") Some("mnote.evidence_locator.v1")
); );
assert!(payload["recent"] assert!(
.as_array() payload["recent"]
.unwrap() .as_array()
.iter() .unwrap()
.any(|item| item["documentId"].as_str() == Some("local-md:README.md"))); .iter()
.any(|item| item["documentId"].as_str() == Some("local-md:README.md"))
);
let _ = fs::remove_dir_all(&root); let _ = fs::remove_dir_all(&root);
} }
@@ -1409,10 +1468,32 @@ mod tests {
.await .await
.expect("create settings response"); .expect("create settings response");
assert_eq!(create_response.status(), StatusCode::OK); assert_eq!(create_response.status(), StatusCode::OK);
let create_refresh_response = app
.clone()
.oneshot(
Request::builder()
.method("POST")
.uri("/api/search/local-index/refresh")
.header("content-type", "application/json")
.header("x-mnote-actor-id", "user_test")
.header("x-mnote-actor-type", "user")
.body(Body::from(
json!({
"workspaceId": "local-ws-settings-delete",
"rootUri": root_uri
})
.to_string(),
))
.expect("create refresh request"),
)
.await
.expect("create refresh response");
assert_eq!(create_refresh_response.status(), StatusCode::OK);
assert!(root.join(".mnote/index/search-index.json").exists()); assert!(root.join(".mnote/index/search-index.json").exists());
assert!(root.join(".mnote/index/evidence.sqlite").exists()); assert!(root.join(".mnote/index/evidence.sqlite").exists());
let delete_response = app let delete_response = app
.clone()
.oneshot( .oneshot(
Request::builder() Request::builder()
.method("PUT") .method("PUT")
@@ -1440,18 +1521,36 @@ mod tests {
.await .await
.expect("delete body"); .expect("delete body");
let delete_payload: Value = serde_json::from_slice(&delete_body).expect("delete json"); let delete_payload: Value = serde_json::from_slice(&delete_body).expect("delete json");
assert_eq!(
delete_payload["index"]["indexedPaths"]
.as_array()
.map(Vec::len),
Some(0)
);
assert_eq!( assert_eq!(
delete_payload["result"]["settings"]["includePaths"] delete_payload["result"]["settings"]["includePaths"]
.as_array() .as_array()
.map(Vec::len), .map(Vec::len),
Some(0) Some(0)
); );
assert!(root.join(".mnote/index/search-index.json").exists());
assert!(root.join(".mnote/index/evidence.sqlite").exists());
let delete_refresh_response = app
.clone()
.oneshot(
Request::builder()
.method("POST")
.uri("/api/search/local-index/refresh")
.header("content-type", "application/json")
.header("x-mnote-actor-id", "user_test")
.header("x-mnote-actor-type", "user")
.body(Body::from(
json!({
"workspaceId": "local-ws-settings-delete",
"rootUri": root_uri
})
.to_string(),
))
.expect("delete refresh request"),
)
.await
.expect("delete refresh response");
assert_eq!(delete_refresh_response.status(), StatusCode::OK);
assert!(!root.join(".mnote/index/search-index.json").exists()); assert!(!root.join(".mnote/index/search-index.json").exists());
assert!(!root.join(".mnote/index/evidence.sqlite").exists()); assert!(!root.join(".mnote/index/evidence.sqlite").exists());
@@ -1508,11 +1607,13 @@ mod tests {
backlinks_payload["meta"]["queryName"].as_str(), backlinks_payload["meta"]["queryName"].as_str(),
Some("search.local_index.backlinks") Some("search.local_index.backlinks")
); );
assert!(backlinks_payload["result"]["backlinks"] assert!(
.as_array() backlinks_payload["result"]["backlinks"]
.unwrap() .as_array()
.iter() .unwrap()
.any(|item| item["documentId"].as_str() == Some("local-md:README.md"))); .iter()
.any(|item| item["documentId"].as_str() == Some("local-md:README.md"))
);
let tags_response = app() let tags_response = app()
.oneshot( .oneshot(