Improve evidence search ranking and diagnostics
This commit is contained in:
@@ -235,6 +235,8 @@ pub struct EvidenceSearchResult {
|
|||||||
pub score: f64,
|
pub score: f64,
|
||||||
pub source: EvidenceLocator,
|
pub source: EvidenceLocator,
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub match_info: Option<EvidenceSearchMatchInfo>,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
pub citation_url: Option<String>,
|
pub citation_url: Option<String>,
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
pub citation_label: Option<String>,
|
pub citation_label: Option<String>,
|
||||||
@@ -242,12 +244,28 @@ pub struct EvidenceSearchResult {
|
|||||||
pub citation_markdown: Option<String>,
|
pub citation_markdown: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
pub struct EvidenceSearchMatchInfo {
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub rank: Option<u32>,
|
||||||
|
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||||
|
pub matched_terms: Vec<String>,
|
||||||
|
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||||
|
pub missing_terms: Vec<String>,
|
||||||
|
pub match_mode: String,
|
||||||
|
pub match_scope: String,
|
||||||
|
pub strong: bool,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||||
#[serde(rename_all = "camelCase")]
|
#[serde(rename_all = "camelCase")]
|
||||||
pub struct EvidenceSearchResponse {
|
pub struct EvidenceSearchResponse {
|
||||||
pub ok: bool,
|
pub ok: bool,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub results: Vec<EvidenceSearchResult>,
|
pub results: Vec<EvidenceSearchResult>,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub diagnostics: Option<Value>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||||
|
|||||||
@@ -41,10 +41,11 @@ pub use editor::{
|
|||||||
pub use evidence::{
|
pub use evidence::{
|
||||||
EvidenceBBox, EvidenceEdge, EvidenceLocator, EvidenceOpenAction, EvidenceOpenRequest,
|
EvidenceBBox, EvidenceEdge, EvidenceLocator, EvidenceOpenAction, EvidenceOpenRequest,
|
||||||
EvidenceRange, EvidenceReadContext, EvidenceReadRequest, EvidenceReadResponse,
|
EvidenceRange, EvidenceReadContext, EvidenceReadRequest, EvidenceReadResponse,
|
||||||
EvidenceResourceKind, EvidenceSearchMode, EvidenceSearchRequest, EvidenceSearchResponse,
|
EvidenceResourceKind, EvidenceSearchMatchInfo, EvidenceSearchMode, EvidenceSearchRequest,
|
||||||
EvidenceSearchResult, EvidenceSearchScope, ParsedResourceArtifact, ResourceSourceMap,
|
EvidenceSearchResponse, EvidenceSearchResult, EvidenceSearchScope, ParsedResourceArtifact,
|
||||||
SourceMapBlock, SourceMapBlockKind, SourceMapPage, SourceMapSection, SourceMapTextItem,
|
ResourceSourceMap, SourceMapBlock, SourceMapBlockKind, SourceMapPage, SourceMapSection,
|
||||||
EVIDENCE_LOCATOR_SCHEMA, PARSED_RESOURCE_ARTIFACT_SCHEMA, RESOURCE_SOURCE_MAP_SCHEMA,
|
SourceMapTextItem, EVIDENCE_LOCATOR_SCHEMA, PARSED_RESOURCE_ARTIFACT_SCHEMA,
|
||||||
|
RESOURCE_SOURCE_MAP_SCHEMA,
|
||||||
};
|
};
|
||||||
pub use kernel::{
|
pub use kernel::{
|
||||||
DocBufferDirtyState, DocumentBuffer, DocumentContentResult, DocumentReadEvidenceItem,
|
DocBufferDirtyState, DocumentBuffer, DocumentContentResult, DocumentReadEvidenceItem,
|
||||||
|
|||||||
@@ -2224,7 +2224,18 @@ import { createSidebarPageSettingsRuntime } from './sidebar-page-settings-runtim
|
|||||||
return button instanceof HTMLElement && button.getAttribute('aria-checked') === 'true';
|
return button instanceof HTMLElement && button.getAttribute('aria-checked') === 'true';
|
||||||
}
|
}
|
||||||
|
|
||||||
function highlightSearchText(value, query, exact) {
|
function searchHighlightTerms(item, query) {
|
||||||
|
var info = item && item.matchInfo || item && item.evidence && item.evidence.matchInfo || null;
|
||||||
|
var terms = info && Array.isArray(info.matchedTerms) ? info.matchedTerms : [];
|
||||||
|
terms = terms.map(searchText).filter(Boolean);
|
||||||
|
if (!terms.length) {
|
||||||
|
terms = searchText(query).split(/[\s,;:,;:。、()[\]{}]+/).map(searchText).filter(Boolean);
|
||||||
|
}
|
||||||
|
terms.sort(function(left, right) { return right.length - left.length; });
|
||||||
|
return terms.filter(function(term, index) { return terms.indexOf(term) === index; });
|
||||||
|
}
|
||||||
|
|
||||||
|
function highlightSearchText(value, query, exact, terms) {
|
||||||
var text = searchText(value);
|
var text = searchText(value);
|
||||||
var cleanQuery = searchText(query);
|
var cleanQuery = searchText(query);
|
||||||
if (!text || !cleanQuery) return escapeHtml(text);
|
if (!text || !cleanQuery) return escapeHtml(text);
|
||||||
@@ -2237,19 +2248,30 @@ import { createSidebarPageSettingsRuntime } from './sidebar-page-settings-runtim
|
|||||||
'<mark>' + escapeHtml(text.slice(exactIndex, exactIndex + cleanQuery.length)) + '</mark>' +
|
'<mark>' + escapeHtml(text.slice(exactIndex, exactIndex + cleanQuery.length)) + '</mark>' +
|
||||||
escapeHtml(text.slice(exactIndex + cleanQuery.length));
|
escapeHtml(text.slice(exactIndex + cleanQuery.length));
|
||||||
}
|
}
|
||||||
var chars = Array.from(cleanQuery.replace(/\s+/g, '').toLowerCase());
|
terms = Array.isArray(terms) ? terms.map(searchText).filter(Boolean) : [];
|
||||||
if (!chars.length) return escapeHtml(text);
|
if (!terms.length) return escapeHtml(text);
|
||||||
var next = 0;
|
var ranges = [];
|
||||||
var html = '';
|
terms.forEach(function(term) {
|
||||||
Array.from(text).forEach(function(ch) {
|
var lowerTerm = term.toLowerCase();
|
||||||
if (next < chars.length && ch.toLowerCase() === chars[next]) {
|
var start = lower.indexOf(lowerTerm);
|
||||||
html += '<mark>' + escapeHtml(ch) + '</mark>';
|
while (start >= 0) {
|
||||||
next += 1;
|
var end = start + lowerTerm.length;
|
||||||
} else {
|
var overlaps = ranges.some(function(range) { return start < range.end && end > range.start; });
|
||||||
html += escapeHtml(ch);
|
if (!overlaps) ranges.push({ start: start, end: end });
|
||||||
|
start = lower.indexOf(lowerTerm, end);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
return next >= chars.length ? html : escapeHtml(text);
|
if (!ranges.length) return escapeHtml(text);
|
||||||
|
ranges.sort(function(left, right) { return left.start - right.start; });
|
||||||
|
var html = '';
|
||||||
|
var cursor = 0;
|
||||||
|
ranges.forEach(function(range) {
|
||||||
|
if (range.start > cursor) html += escapeHtml(text.slice(cursor, range.start));
|
||||||
|
html += '<mark>' + escapeHtml(text.slice(range.start, range.end)) + '</mark>';
|
||||||
|
cursor = range.end;
|
||||||
|
});
|
||||||
|
if (cursor < text.length) html += escapeHtml(text.slice(cursor));
|
||||||
|
return html;
|
||||||
}
|
}
|
||||||
|
|
||||||
function searchResultEvidenceLocator(item) {
|
function searchResultEvidenceLocator(item) {
|
||||||
@@ -2436,10 +2458,11 @@ import { createSidebarPageSettingsRuntime } from './sidebar-page-settings-runtim
|
|||||||
var locatorAttr = locator ? escapeHtml(JSON.stringify(locator)) : '';
|
var locatorAttr = locator ? escapeHtml(JSON.stringify(locator)) : '';
|
||||||
var index = items.indexOf(item);
|
var index = items.indexOf(item);
|
||||||
var exact = searchSwitchValue(overlay, 'exact');
|
var exact = searchSwitchValue(overlay, 'exact');
|
||||||
|
var highlightTerms = searchHighlightTerms(item, query);
|
||||||
return '<button type="button" class="wolai-search-result-row" data-testid="wolai-search-result-row" data-search-result-owner="rust-kernel" data-search-result-index="' + String(index) + '" data-document-id="' + escapeHtml(item.documentId || item.nodeId || item.id || '') + '" data-resource-type="' + escapeHtml(resourceType) + '"' + (locatorAttr ? ' data-evidence-locator="' + locatorAttr + '"' : '') + '>' +
|
return '<button type="button" class="wolai-search-result-row" data-testid="wolai-search-result-row" data-search-result-owner="rust-kernel" data-search-result-index="' + String(index) + '" data-document-id="' + escapeHtml(item.documentId || item.nodeId || item.id || '') + '" data-resource-type="' + escapeHtml(resourceType) + '"' + (locatorAttr ? ' data-evidence-locator="' + locatorAttr + '"' : '') + '>' +
|
||||||
'<span class="material-symbols-outlined wolai-search-result-icon" data-icon="article" aria-hidden="true"></span>' +
|
'<span class="material-symbols-outlined wolai-search-result-icon" data-icon="article" aria-hidden="true"></span>' +
|
||||||
'<span class="wolai-search-result-main"><span class="wolai-search-result-title">' + highlightSearchText(title, query, exact) + '</span>' +
|
'<span class="wolai-search-result-main"><span class="wolai-search-result-title">' + highlightSearchText(title, query, exact, highlightTerms) + '</span>' +
|
||||||
'<span class="wolai-search-result-snippet">' + (snippet ? highlightSearchText(snippet, query, exact) : escapeHtml(path)) + '</span>' +
|
'<span class="wolai-search-result-snippet">' + (snippet ? highlightSearchText(snippet, query, exact, highlightTerms) : escapeHtml(path)) + '</span>' +
|
||||||
'<span class="wolai-search-result-path"><span>' + escapeHtml(path) + '</span><span class="wolai-search-result-type">' + escapeHtml(resourceType) + '</span></span></span>' +
|
'<span class="wolai-search-result-path"><span>' + escapeHtml(path) + '</span><span class="wolai-search-result-type">' + escapeHtml(resourceType) + '</span></span></span>' +
|
||||||
'</button>';
|
'</button>';
|
||||||
}).join('');
|
}).join('');
|
||||||
|
|||||||
@@ -61,19 +61,33 @@ pub(crate) async fn search_payload(
|
|||||||
body.top_k,
|
body.top_k,
|
||||||
)? {
|
)? {
|
||||||
enrich_citation_links(&mut results);
|
enrich_citation_links(&mut results);
|
||||||
return Ok(json!(EvidenceSearchResponse { ok: true, results }));
|
return Ok(json!(EvidenceSearchResponse {
|
||||||
|
ok: true,
|
||||||
|
results,
|
||||||
|
diagnostics: None
|
||||||
|
}));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if let Some(mut results) = local_search_index::query_evidence_sqlite_results(
|
if let Some(mut results) = local_search_index::query_evidence_sqlite_results_with_mode(
|
||||||
&root_path,
|
&root_path,
|
||||||
&query,
|
&query,
|
||||||
evidence_owner_filter,
|
evidence_owner_filter,
|
||||||
body.top_k,
|
body.top_k,
|
||||||
|
false,
|
||||||
)? {
|
)? {
|
||||||
enrich_sqlite_evidence_results(&mut results, &root_path, &query);
|
enrich_sqlite_evidence_results(&mut results, &root_path, &query);
|
||||||
enrich_citation_links(&mut results);
|
enrich_citation_links(&mut results);
|
||||||
if !results.is_empty() || !query.is_empty() {
|
if !results.is_empty() || !query.is_empty() {
|
||||||
let response = EvidenceSearchResponse { ok: true, results };
|
let diagnostics = Some(evidence_search_diagnostics(
|
||||||
|
&query,
|
||||||
|
&results,
|
||||||
|
"evidence_sqlite",
|
||||||
|
));
|
||||||
|
let response = EvidenceSearchResponse {
|
||||||
|
ok: true,
|
||||||
|
results,
|
||||||
|
diagnostics,
|
||||||
|
};
|
||||||
return Ok(json!(response));
|
return Ok(json!(response));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -97,6 +111,7 @@ pub(crate) async fn search_payload(
|
|||||||
enrich_citation_links(&mut results);
|
enrich_citation_links(&mut results);
|
||||||
results
|
results
|
||||||
},
|
},
|
||||||
|
diagnostics: None,
|
||||||
};
|
};
|
||||||
Ok(json!(response))
|
Ok(json!(response))
|
||||||
}
|
}
|
||||||
@@ -154,6 +169,78 @@ fn enrich_citation_links(results: &mut [EvidenceSearchResult]) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn evidence_search_diagnostics(
|
||||||
|
query: &str,
|
||||||
|
results: &[EvidenceSearchResult],
|
||||||
|
backend: &str,
|
||||||
|
) -> Value {
|
||||||
|
let query_tokens = evidence_query_tokens(query);
|
||||||
|
let strong_match_count = results
|
||||||
|
.iter()
|
||||||
|
.filter(|result| result.match_info.as_ref().is_some_and(|info| info.strong))
|
||||||
|
.count();
|
||||||
|
let weak_match_count = results.len().saturating_sub(strong_match_count);
|
||||||
|
let reason = if results.is_empty() {
|
||||||
|
"no_evidence_match"
|
||||||
|
} else if strong_match_count == 0 {
|
||||||
|
"only_weak_partial_matches"
|
||||||
|
} else {
|
||||||
|
"ranked_evidence_matches"
|
||||||
|
};
|
||||||
|
json!({
|
||||||
|
"backend": backend,
|
||||||
|
"queryTokens": query_tokens,
|
||||||
|
"resultCount": results.len(),
|
||||||
|
"strongMatchCount": strong_match_count,
|
||||||
|
"weakMatchCount": weak_match_count,
|
||||||
|
"reason": reason,
|
||||||
|
"suggestedQueries": suggested_evidence_queries(query),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn evidence_query_tokens(query: &str) -> Vec<String> {
|
||||||
|
query
|
||||||
|
.split(|ch: char| {
|
||||||
|
ch.is_whitespace()
|
||||||
|
|| matches!(
|
||||||
|
ch,
|
||||||
|
',' | ';'
|
||||||
|
| ':'
|
||||||
|
| ','
|
||||||
|
| ';'
|
||||||
|
| ':'
|
||||||
|
| '。'
|
||||||
|
| '、'
|
||||||
|
| '('
|
||||||
|
| ')'
|
||||||
|
| '['
|
||||||
|
| ']'
|
||||||
|
| '{'
|
||||||
|
| '}'
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|value| !value.is_empty())
|
||||||
|
.map(ToOwned::to_owned)
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn suggested_evidence_queries(query: &str) -> Vec<String> {
|
||||||
|
let tokens = evidence_query_tokens(query);
|
||||||
|
let mut suggestions = Vec::new();
|
||||||
|
if tokens.len() > 1 {
|
||||||
|
suggestions.push(tokens.join(" "));
|
||||||
|
}
|
||||||
|
for token in tokens {
|
||||||
|
if token.chars().count() >= 3 {
|
||||||
|
suggestions.push(token.chars().take(2).collect::<String>());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
suggestions.sort();
|
||||||
|
suggestions.dedup();
|
||||||
|
suggestions
|
||||||
|
}
|
||||||
|
|
||||||
fn citation_markdown_for_locator(locator: &EvidenceLocator) -> String {
|
fn citation_markdown_for_locator(locator: &EvidenceLocator) -> String {
|
||||||
let label = citation_label_for_locator(locator);
|
let label = citation_label_for_locator(locator);
|
||||||
let url = citation_url_for_locator(locator);
|
let url = citation_url_for_locator(locator);
|
||||||
@@ -537,6 +624,7 @@ fn source_map_block_result(
|
|||||||
quote: block.text.clone(),
|
quote: block.text.clone(),
|
||||||
score,
|
score,
|
||||||
source,
|
source,
|
||||||
|
match_info: None,
|
||||||
citation_url: None,
|
citation_url: None,
|
||||||
citation_label: None,
|
citation_label: None,
|
||||||
citation_markdown: None,
|
citation_markdown: None,
|
||||||
@@ -747,6 +835,11 @@ pub(crate) fn evidence_results_from_local_search(
|
|||||||
.to_string(),
|
.to_string(),
|
||||||
score: result.get("score").and_then(Value::as_f64).unwrap_or(0.0),
|
score: result.get("score").and_then(Value::as_f64).unwrap_or(0.0),
|
||||||
source: locator,
|
source: locator,
|
||||||
|
match_info: result
|
||||||
|
.get("evidence")
|
||||||
|
.and_then(|value| value.get("matchInfo"))
|
||||||
|
.cloned()
|
||||||
|
.and_then(|value| serde_json::from_value(value).ok()),
|
||||||
citation_url: None,
|
citation_url: None,
|
||||||
citation_label: None,
|
citation_label: None,
|
||||||
citation_markdown: None,
|
citation_markdown: None,
|
||||||
@@ -1251,6 +1344,7 @@ mod tests {
|
|||||||
quote: "NeedleToken 原文定位".into(),
|
quote: "NeedleToken 原文定位".into(),
|
||||||
score: 1.0,
|
score: 1.0,
|
||||||
source: locator,
|
source: locator,
|
||||||
|
match_info: None,
|
||||||
citation_url: None,
|
citation_url: None,
|
||||||
citation_label: None,
|
citation_label: None,
|
||||||
citation_markdown: None,
|
citation_markdown: None,
|
||||||
@@ -1285,6 +1379,47 @@ mod tests {
|
|||||||
.contains("spec.pdf"));
|
.contains("spec.pdf"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn evidence_search_diagnostics_marks_partial_matches() {
|
||||||
|
let results = vec![EvidenceSearchResult {
|
||||||
|
evidence_id: "ev1".into(),
|
||||||
|
quote: "羧酸需要被保护".into(),
|
||||||
|
score: 0.7,
|
||||||
|
source: EvidenceLocator::new(
|
||||||
|
"file:///workspace",
|
||||||
|
"local-md:docs~2FPage.md",
|
||||||
|
"docs/Page.md",
|
||||||
|
EvidenceResourceKind::Markdown,
|
||||||
|
EvidenceOpenAction {
|
||||||
|
action_type: "mnote.open_resource_locator".into(),
|
||||||
|
url: "/documents/local-md:docs~2FPage.md".into(),
|
||||||
|
params: json!({}),
|
||||||
|
},
|
||||||
|
),
|
||||||
|
match_info: Some(core_protocol::EvidenceSearchMatchInfo {
|
||||||
|
rank: Some(1),
|
||||||
|
matched_terms: vec!["羧酸".into(), "保护".into()],
|
||||||
|
missing_terms: vec!["保护基".into()],
|
||||||
|
match_mode: "cjk_partial".into(),
|
||||||
|
match_scope: "block".into(),
|
||||||
|
strong: false,
|
||||||
|
}),
|
||||||
|
citation_url: None,
|
||||||
|
citation_label: None,
|
||||||
|
citation_markdown: None,
|
||||||
|
}];
|
||||||
|
|
||||||
|
let diagnostics = evidence_search_diagnostics("羧酸 保护基", &results, "evidence_sqlite");
|
||||||
|
assert_eq!(diagnostics["backend"].as_str(), Some("evidence_sqlite"));
|
||||||
|
assert_eq!(diagnostics["queryTokens"].as_array().map(Vec::len), Some(2));
|
||||||
|
assert_eq!(diagnostics["strongMatchCount"].as_u64(), Some(0));
|
||||||
|
assert_eq!(diagnostics["weakMatchCount"].as_u64(), Some(1));
|
||||||
|
assert_eq!(
|
||||||
|
diagnostics["reason"].as_str(),
|
||||||
|
Some("only_weak_partial_matches")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn evidence_read_uses_source_map_context_and_section_blocks() {
|
fn evidence_read_uses_source_map_context_and_section_blocks() {
|
||||||
let root = std::env::temp_dir().join(format!(
|
let root = std::env::temp_dir().join(format!(
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
use crate::error::WebError;
|
use crate::error::WebError;
|
||||||
use crate::evidence_parse::{
|
use crate::evidence_parse::{
|
||||||
liteparse_runtime_available, parse_liteparse_input_blocking, ParseInput, ParseProviderMode,
|
ParseInput, ParseProviderMode, liteparse_runtime_available, parse_liteparse_input_blocking,
|
||||||
};
|
};
|
||||||
use crate::routes::local_folder_source::encode_local_id_segment;
|
use crate::routes::local_folder_source::encode_local_id_segment;
|
||||||
use crate::routes::local_markdown_parser::{
|
use crate::routes::local_markdown_parser::{
|
||||||
@@ -9,12 +9,12 @@ use crate::routes::local_markdown_parser::{
|
|||||||
use crate::routes::local_ocr;
|
use crate::routes::local_ocr;
|
||||||
use control_plane::{ControlPlaneStore, UpsertUserInput, UpsertUserUiPreferenceInput};
|
use control_plane::{ControlPlaneStore, UpsertUserInput, UpsertUserUiPreferenceInput};
|
||||||
use core_protocol::{
|
use core_protocol::{
|
||||||
EvidenceLocator, EvidenceSearchResult, ParsedResourceArtifact, ResourceSourceMap,
|
EvidenceLocator, EvidenceSearchMatchInfo, EvidenceSearchResult,
|
||||||
SourceMapBlock, PARSED_RESOURCE_ARTIFACT_SCHEMA,
|
PARSED_RESOURCE_ARTIFACT_SCHEMA, ParsedResourceArtifact, ResourceSourceMap, SourceMapBlock,
|
||||||
};
|
};
|
||||||
use rusqlite::{params, Connection, OptionalExtension};
|
use rusqlite::{Connection, OptionalExtension, params};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use serde_json::{json, Value};
|
use serde_json::{Value, json};
|
||||||
use std::collections::BTreeSet;
|
use std::collections::BTreeSet;
|
||||||
use std::fs;
|
use std::fs;
|
||||||
use std::path::{Component, Path, PathBuf};
|
use std::path::{Component, Path, PathBuf};
|
||||||
@@ -671,6 +671,7 @@ pub(crate) fn refresh_local_search_index_if_scheduled_due_with_store(
|
|||||||
.map(Some)
|
.map(Some)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
pub(crate) fn query_evidence_sqlite_results(
|
pub(crate) fn query_evidence_sqlite_results(
|
||||||
root_path: &Path,
|
root_path: &Path,
|
||||||
query: &str,
|
query: &str,
|
||||||
@@ -812,6 +813,7 @@ pub(crate) fn read_evidence_sqlite_context(
|
|||||||
0.8
|
0.8
|
||||||
},
|
},
|
||||||
source: source.clone(),
|
source: source.clone(),
|
||||||
|
match_info: None,
|
||||||
citation_url: None,
|
citation_url: None,
|
||||||
citation_label: None,
|
citation_label: None,
|
||||||
citation_markdown: None,
|
citation_markdown: None,
|
||||||
@@ -901,6 +903,7 @@ fn graph_edge_row(row: &rusqlite::Row<'_>) -> rusqlite::Result<EvidenceSearchRes
|
|||||||
block_id: source.block_id.or(Some(source_block_id)),
|
block_id: source.block_id.or(Some(source_block_id)),
|
||||||
..source
|
..source
|
||||||
},
|
},
|
||||||
|
match_info: None,
|
||||||
citation_url: None,
|
citation_url: None,
|
||||||
citation_label: None,
|
citation_label: None,
|
||||||
citation_markdown: None,
|
citation_markdown: None,
|
||||||
@@ -1059,43 +1062,76 @@ fn query_evidence_sqlite_fuzzy(
|
|||||||
let scope_value = owner_document_id.or(resource_path);
|
let scope_value = owner_document_id.or(resource_path);
|
||||||
let rows = match (scope_value, first_char_like.as_deref()) {
|
let rows = match (scope_value, first_char_like.as_deref()) {
|
||||||
(Some(scope_value), Some(first_char_like)) => statement
|
(Some(scope_value), Some(first_char_like)) => statement
|
||||||
.query_map(params![scope_value, first_char_like, scan_limit], |row| {
|
.query_map(
|
||||||
evidence_result_from_sqlite_row(row, query)
|
params![scope_value, first_char_like, scan_limit],
|
||||||
})
|
evidence_sqlite_row_parts,
|
||||||
|
)
|
||||||
.map_err(sqlite_error)?
|
.map_err(sqlite_error)?
|
||||||
.collect::<Result<Vec<_>, _>>(),
|
.collect::<Result<Vec<_>, _>>(),
|
||||||
(Some(scope_value), None) => statement
|
(Some(scope_value), None) => statement
|
||||||
.query_map(params![scope_value, scan_limit], |row| {
|
.query_map(params![scope_value, scan_limit], evidence_sqlite_row_parts)
|
||||||
evidence_result_from_sqlite_row(row, query)
|
|
||||||
})
|
|
||||||
.map_err(sqlite_error)?
|
.map_err(sqlite_error)?
|
||||||
.collect::<Result<Vec<_>, _>>(),
|
.collect::<Result<Vec<_>, _>>(),
|
||||||
(None, Some(first_char_like)) => statement
|
(None, Some(first_char_like)) => statement
|
||||||
.query_map(params![first_char_like, scan_limit], |row| {
|
.query_map(
|
||||||
evidence_result_from_sqlite_row(row, query)
|
params![first_char_like, scan_limit],
|
||||||
})
|
evidence_sqlite_row_parts,
|
||||||
|
)
|
||||||
.map_err(sqlite_error)?
|
.map_err(sqlite_error)?
|
||||||
.collect::<Result<Vec<_>, _>>(),
|
.collect::<Result<Vec<_>, _>>(),
|
||||||
(None, None) => statement
|
(None, None) => statement
|
||||||
.query_map(params![scan_limit], |row| {
|
.query_map(params![scan_limit], evidence_sqlite_row_parts)
|
||||||
evidence_result_from_sqlite_row(row, query)
|
|
||||||
})
|
|
||||||
.map_err(sqlite_error)?
|
.map_err(sqlite_error)?
|
||||||
.collect::<Result<Vec<_>, _>>(),
|
.collect::<Result<Vec<_>, _>>(),
|
||||||
}
|
}
|
||||||
.map_err(sqlite_error)?;
|
.map_err(sqlite_error)?;
|
||||||
Ok(rows
|
let mut scored = rows
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.filter(|result| {
|
.filter_map(|(block_id, text, source)| {
|
||||||
fuzzy_search_match(
|
let text_match = score_evidence_text_match(&text, query)?;
|
||||||
&normalize_search_text(&result.quote),
|
Some(EvidenceSearchResult {
|
||||||
&normalize_search_text(query),
|
evidence_id: block_id,
|
||||||
)
|
quote: ocr_search_snippet_for_terms(&text, query, &text_match.matched_terms),
|
||||||
|
score: text_match.score,
|
||||||
|
source,
|
||||||
|
match_info: Some(text_match.into_match_info(None)),
|
||||||
|
citation_url: None,
|
||||||
|
citation_label: None,
|
||||||
|
citation_markdown: None,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
scored.sort_by(|left, right| {
|
||||||
|
right
|
||||||
|
.score
|
||||||
|
.partial_cmp(&left.score)
|
||||||
|
.unwrap_or(std::cmp::Ordering::Equal)
|
||||||
|
});
|
||||||
|
Ok(scored
|
||||||
|
.into_iter()
|
||||||
|
.enumerate()
|
||||||
|
.map(|(index, mut result)| {
|
||||||
|
if let Some(match_info) = result.match_info.as_mut() {
|
||||||
|
match_info.rank = Some((index + 1) as u32);
|
||||||
|
}
|
||||||
|
result
|
||||||
})
|
})
|
||||||
.take(limit.max(1) as usize)
|
.take(limit.max(1) as usize)
|
||||||
.collect())
|
.collect())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn evidence_sqlite_row_parts(
|
||||||
|
row: &rusqlite::Row<'_>,
|
||||||
|
) -> rusqlite::Result<(String, String, EvidenceLocator)> {
|
||||||
|
let block_id: String = row.get(0)?;
|
||||||
|
let text: String = row.get(1)?;
|
||||||
|
let locator_json: String = row.get(2)?;
|
||||||
|
let source = serde_json::from_str::<EvidenceLocator>(&locator_json).map_err(|error| {
|
||||||
|
rusqlite::Error::FromSqlConversionFailure(2, rusqlite::types::Type::Text, Box::new(error))
|
||||||
|
})?;
|
||||||
|
Ok((block_id, text, source))
|
||||||
|
}
|
||||||
|
|
||||||
fn evidence_result_from_sqlite_row(
|
fn evidence_result_from_sqlite_row(
|
||||||
row: &rusqlite::Row<'_>,
|
row: &rusqlite::Row<'_>,
|
||||||
query: &str,
|
query: &str,
|
||||||
@@ -1107,15 +1143,20 @@ fn evidence_result_from_sqlite_row(
|
|||||||
let source = serde_json::from_str::<EvidenceLocator>(&locator_json).map_err(|error| {
|
let source = serde_json::from_str::<EvidenceLocator>(&locator_json).map_err(|error| {
|
||||||
rusqlite::Error::FromSqlConversionFailure(2, rusqlite::types::Type::Text, Box::new(error))
|
rusqlite::Error::FromSqlConversionFailure(2, rusqlite::types::Type::Text, Box::new(error))
|
||||||
})?;
|
})?;
|
||||||
|
let text_match = score_evidence_text_match(&text, query);
|
||||||
Ok(EvidenceSearchResult {
|
Ok(EvidenceSearchResult {
|
||||||
evidence_id: block_id,
|
evidence_id: block_id,
|
||||||
quote: ocr_search_snippet(&text, query),
|
quote: text_match
|
||||||
|
.as_ref()
|
||||||
|
.map(|text_match| ocr_search_snippet_for_terms(&text, query, &text_match.matched_terms))
|
||||||
|
.unwrap_or_else(|| ocr_search_snippet(&text, query)),
|
||||||
score: if rank == 0.0 {
|
score: if rank == 0.0 {
|
||||||
1.0
|
1.0
|
||||||
} else {
|
} else {
|
||||||
1.0 / (1.0 + rank.abs())
|
1.0 / (1.0 + rank.abs())
|
||||||
},
|
},
|
||||||
source,
|
source,
|
||||||
|
match_info: text_match.map(|text_match| text_match.into_match_info(None)),
|
||||||
citation_url: None,
|
citation_url: None,
|
||||||
citation_label: None,
|
citation_label: None,
|
||||||
citation_markdown: None,
|
citation_markdown: None,
|
||||||
@@ -1952,6 +1993,13 @@ fn index_relative_path_is_included(relative_path: &str, include_paths: &[String]
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn local_index_relative_path_is_included(
|
||||||
|
relative_path: &str,
|
||||||
|
settings: &LocalIndexSettings,
|
||||||
|
) -> bool {
|
||||||
|
index_relative_path_is_included(relative_path, &settings.include_paths)
|
||||||
|
}
|
||||||
|
|
||||||
fn collect_markdown_documents(
|
fn collect_markdown_documents(
|
||||||
root_path: &Path,
|
root_path: &Path,
|
||||||
current: &Path,
|
current: &Path,
|
||||||
@@ -3605,7 +3653,7 @@ fn local_search_document_matches(
|
|||||||
if exact {
|
if exact {
|
||||||
haystack.contains(query)
|
haystack.contains(query)
|
||||||
} else {
|
} else {
|
||||||
fuzzy_search_match(&haystack, query)
|
token_search_match(&haystack, query)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -3629,7 +3677,7 @@ fn local_search_resource_matches(
|
|||||||
if exact {
|
if exact {
|
||||||
haystack.contains(query)
|
haystack.contains(query)
|
||||||
} else {
|
} else {
|
||||||
fuzzy_search_match(&haystack, query)
|
token_search_match(&haystack, query)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -3657,7 +3705,7 @@ fn local_search_ocr_matches(
|
|||||||
if exact {
|
if exact {
|
||||||
haystack.contains(query)
|
haystack.contains(query)
|
||||||
} else {
|
} else {
|
||||||
fuzzy_search_match(&haystack, query)
|
token_search_match(&haystack, query)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -3929,6 +3977,203 @@ fn search_document_hit(document: &LocalSearchDocument, query: &str) -> SearchDoc
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
struct EvidenceQueryTerm {
|
||||||
|
display: String,
|
||||||
|
normalized: String,
|
||||||
|
alternatives: Vec<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
struct EvidenceTextMatch {
|
||||||
|
score: f64,
|
||||||
|
matched_terms: Vec<String>,
|
||||||
|
missing_terms: Vec<String>,
|
||||||
|
match_mode: String,
|
||||||
|
strong: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl EvidenceTextMatch {
|
||||||
|
fn into_match_info(self, rank: Option<u32>) -> EvidenceSearchMatchInfo {
|
||||||
|
EvidenceSearchMatchInfo {
|
||||||
|
rank,
|
||||||
|
matched_terms: self.matched_terms,
|
||||||
|
missing_terms: self.missing_terms,
|
||||||
|
match_mode: self.match_mode,
|
||||||
|
match_scope: "block".into(),
|
||||||
|
strong: self.strong,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn score_evidence_text_match(text: &str, query: &str) -> Option<EvidenceTextMatch> {
|
||||||
|
let normalized_text = normalize_search_text(text);
|
||||||
|
let normalized_query = normalize_search_text(query);
|
||||||
|
if normalized_query.is_empty() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let terms = evidence_query_terms(&normalized_query);
|
||||||
|
if terms.is_empty() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let phrase_exact = normalized_text.contains(&normalized_query);
|
||||||
|
let mut matched_terms = Vec::new();
|
||||||
|
let mut missing_terms = Vec::new();
|
||||||
|
let mut exact_count = 0usize;
|
||||||
|
let mut partial_count = 0usize;
|
||||||
|
let mut positions = Vec::new();
|
||||||
|
|
||||||
|
for term in &terms {
|
||||||
|
if let Some(index) = normalized_text.find(&term.normalized) {
|
||||||
|
exact_count += 1;
|
||||||
|
positions.push(index);
|
||||||
|
push_unique(&mut matched_terms, term.display.clone());
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let partial_matches = term
|
||||||
|
.alternatives
|
||||||
|
.iter()
|
||||||
|
.filter_map(|alternative| {
|
||||||
|
normalized_text
|
||||||
|
.find(alternative)
|
||||||
|
.map(|index| (alternative.clone(), index))
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
let partial_allowed = if terms.len() > 1 {
|
||||||
|
!partial_matches.is_empty()
|
||||||
|
} else {
|
||||||
|
partial_matches.len() >= 2 || term.normalized.chars().count() <= 2
|
||||||
|
};
|
||||||
|
if partial_allowed {
|
||||||
|
partial_count += 1;
|
||||||
|
if let Some((_, index)) = partial_matches.first() {
|
||||||
|
positions.push(*index);
|
||||||
|
}
|
||||||
|
for (alternative, _) in partial_matches {
|
||||||
|
push_unique(&mut matched_terms, alternative);
|
||||||
|
}
|
||||||
|
push_unique(&mut missing_terms, term.display.clone());
|
||||||
|
} else {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let term_count = terms.len().max(1) as f64;
|
||||||
|
let coverage = (exact_count as f64 + partial_count as f64 * 0.45) / term_count;
|
||||||
|
let proximity = if positions.len() > 1 {
|
||||||
|
let min = positions.iter().min().copied().unwrap_or(0);
|
||||||
|
let max = positions.iter().max().copied().unwrap_or(min);
|
||||||
|
let span = max.saturating_sub(min);
|
||||||
|
if span <= 80 {
|
||||||
|
0.1
|
||||||
|
} else if span <= 240 {
|
||||||
|
0.05
|
||||||
|
} else {
|
||||||
|
0.0
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
0.0
|
||||||
|
};
|
||||||
|
let score = if phrase_exact {
|
||||||
|
1.0
|
||||||
|
} else {
|
||||||
|
(coverage * 0.85 + proximity).min(0.95)
|
||||||
|
};
|
||||||
|
let match_mode = if phrase_exact {
|
||||||
|
"exact_phrase"
|
||||||
|
} else if partial_count == 0 {
|
||||||
|
"all_terms"
|
||||||
|
} else {
|
||||||
|
"cjk_partial"
|
||||||
|
};
|
||||||
|
Some(EvidenceTextMatch {
|
||||||
|
score,
|
||||||
|
matched_terms,
|
||||||
|
missing_terms,
|
||||||
|
match_mode: match_mode.into(),
|
||||||
|
strong: phrase_exact || partial_count == 0 && exact_count == terms.len(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn evidence_query_terms(query: &str) -> Vec<EvidenceQueryTerm> {
|
||||||
|
split_search_query_tokens(query)
|
||||||
|
.into_iter()
|
||||||
|
.filter(|token| !token.is_empty())
|
||||||
|
.map(|token| {
|
||||||
|
let alternatives = cjk_token_bigrams(&token)
|
||||||
|
.into_iter()
|
||||||
|
.filter(|alternative| alternative != &token)
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
EvidenceQueryTerm {
|
||||||
|
display: token.clone(),
|
||||||
|
normalized: token,
|
||||||
|
alternatives,
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn split_search_query_tokens(query: &str) -> Vec<String> {
|
||||||
|
let mut tokens = Vec::new();
|
||||||
|
let mut current = String::new();
|
||||||
|
for ch in query.chars() {
|
||||||
|
if ch.is_whitespace() || is_search_separator(ch) {
|
||||||
|
if !current.is_empty() {
|
||||||
|
tokens.push(std::mem::take(&mut current));
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
current.push(ch);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !current.is_empty() {
|
||||||
|
tokens.push(current);
|
||||||
|
}
|
||||||
|
tokens
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_search_separator(ch: char) -> bool {
|
||||||
|
matches!(
|
||||||
|
ch,
|
||||||
|
',' | ';' | ':' | ',' | ';' | ':' | '。' | '、' | '(' | ')' | '[' | ']' | '{' | '}'
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn cjk_token_bigrams(token: &str) -> Vec<String> {
|
||||||
|
let chars = token.chars().collect::<Vec<_>>();
|
||||||
|
if chars.len() < 3 || !chars.iter().any(|ch| is_cjk_char(*ch)) {
|
||||||
|
return Vec::new();
|
||||||
|
}
|
||||||
|
chars
|
||||||
|
.windows(2)
|
||||||
|
.filter_map(|window| {
|
||||||
|
if window.iter().any(|ch| is_cjk_stop_char(*ch)) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Some(window.iter().collect::<String>())
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_cjk_char(ch: char) -> bool {
|
||||||
|
matches!(
|
||||||
|
ch as u32,
|
||||||
|
0x3400..=0x4DBF | 0x4E00..=0x9FFF | 0xF900..=0xFAFF
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_cjk_stop_char(ch: char) -> bool {
|
||||||
|
matches!(
|
||||||
|
ch,
|
||||||
|
'的' | '了' | '和' | '与' | '及' | '在' | '是' | '为' | '对' | '中'
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn push_unique(values: &mut Vec<String>, value: String) {
|
||||||
|
if !value.is_empty() && !values.iter().any(|existing| existing == &value) {
|
||||||
|
values.push(value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn ocr_search_snippet(body: &str, query: &str) -> String {
|
fn ocr_search_snippet(body: &str, query: &str) -> String {
|
||||||
let normalized_body = body.replace('\n', " ");
|
let normalized_body = body.replace('\n', " ");
|
||||||
let normalized_query = query.trim();
|
let normalized_query = query.trim();
|
||||||
@@ -3958,26 +4203,37 @@ fn ocr_search_snippet(body: &str, query: &str) -> String {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn fuzzy_search_match(haystack: &str, query: &str) -> bool {
|
fn ocr_search_snippet_for_terms(body: &str, query: &str, terms: &[String]) -> String {
|
||||||
if query.is_empty() {
|
let normalized_body = body.replace('\n', " ");
|
||||||
return false;
|
let normalized_query = query.trim();
|
||||||
}
|
if !normalized_query.is_empty() {
|
||||||
if haystack.contains(query) {
|
let lower = normalized_body.to_lowercase();
|
||||||
return true;
|
let lower_query = normalized_query.to_lowercase();
|
||||||
}
|
if let Some(byte_index) = lower.find(&lower_query) {
|
||||||
let mut query_chars = query.chars().filter(|ch| !ch.is_whitespace());
|
return snippet_from_byte_index(&normalized_body, byte_index, 180);
|
||||||
let Some(mut wanted) = query_chars.next() else {
|
|
||||||
return false;
|
|
||||||
};
|
|
||||||
for ch in haystack.chars().filter(|ch| !ch.is_whitespace()) {
|
|
||||||
if ch == wanted {
|
|
||||||
match query_chars.next() {
|
|
||||||
Some(next) => wanted = next,
|
|
||||||
None => return true,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
let lower = normalized_body.to_lowercase();
|
||||||
|
for term in terms {
|
||||||
|
let term = term.trim().to_lowercase();
|
||||||
|
if term.is_empty() {
|
||||||
|
continue;
|
||||||
}
|
}
|
||||||
false
|
if let Some(byte_index) = lower.find(&term) {
|
||||||
|
return snippet_from_byte_index(&normalized_body, byte_index, 180);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ocr_search_snippet(body, query)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn snippet_from_byte_index(body: &str, byte_index: usize, len: usize) -> String {
|
||||||
|
let start = body[..byte_index]
|
||||||
|
.char_indices()
|
||||||
|
.rev()
|
||||||
|
.nth(40)
|
||||||
|
.map(|(idx, _)| idx)
|
||||||
|
.unwrap_or(0);
|
||||||
|
body[start..].chars().take(len).collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
fn fuzzy_search_start_byte(haystack: &str, query: &str) -> Option<usize> {
|
fn fuzzy_search_start_byte(haystack: &str, query: &str) -> Option<usize> {
|
||||||
@@ -4012,6 +4268,10 @@ fn fuzzy_search_start_byte(haystack: &str, query: &str) -> Option<usize> {
|
|||||||
None
|
None
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn token_search_match(haystack: &str, query: &str) -> bool {
|
||||||
|
score_evidence_text_match(haystack, query).is_some()
|
||||||
|
}
|
||||||
|
|
||||||
fn is_markdown_path(path: &Path) -> bool {
|
fn is_markdown_path(path: &Path) -> bool {
|
||||||
path.extension()
|
path.extension()
|
||||||
.and_then(|value| value.to_str())
|
.and_then(|value| value.to_str())
|
||||||
@@ -4034,6 +4294,7 @@ fn resource_type_from_path(path: &Path) -> Option<&'static str> {
|
|||||||
.unwrap_or_default()
|
.unwrap_or_default()
|
||||||
.to_lowercase();
|
.to_lowercase();
|
||||||
match extension.as_str() {
|
match extension.as_str() {
|
||||||
|
"png" | "jpg" | "jpeg" | "webp" | "bmp" | "gif" | "tif" | "tiff" | "svg" => Some("image"),
|
||||||
"pdf" => Some("pdf"),
|
"pdf" => Some("pdf"),
|
||||||
"doc" | "docx" | "odt" | "ppt" | "pptx" | "odp" | "xls" | "xlsx" | "ods" => Some("office"),
|
"doc" | "docx" | "odt" | "ppt" | "pptx" | "odp" | "xls" | "xlsx" | "ods" => Some("office"),
|
||||||
_ => None,
|
_ => None,
|
||||||
@@ -4092,7 +4353,10 @@ mod tests {
|
|||||||
.expect("write mindmap");
|
.expect("write mindmap");
|
||||||
fs::create_dir_all(root.join("assets")).expect("create assets");
|
fs::create_dir_all(root.join("assets")).expect("create assets");
|
||||||
fs::write(root.join("assets").join("spec.pdf"), b"%PDF-1.4\n").expect("write pdf");
|
fs::write(root.join("assets").join("spec.pdf"), b"%PDF-1.4\n").expect("write pdf");
|
||||||
|
fs::write(root.join("assets").join("diagram.png"), b"png").expect("write image");
|
||||||
fs::write(root.join("office").join("report.xlsx"), b"office bytes").expect("write office");
|
fs::write(root.join("office").join("report.xlsx"), b"office bytes").expect("write office");
|
||||||
|
write_local_index_settings(&root, &[String::from(".")], None, None, None, None)
|
||||||
|
.expect("settings");
|
||||||
|
|
||||||
let projection = query_local_search_index(
|
let projection = query_local_search_index(
|
||||||
&root,
|
&root,
|
||||||
@@ -4111,41 +4375,55 @@ mod tests {
|
|||||||
.iter()
|
.iter()
|
||||||
.find(|item| item["documentId"].as_str() == Some("local-md:README.md"))
|
.find(|item| item["documentId"].as_str() == Some("local-md:README.md"))
|
||||||
.expect("home result");
|
.expect("home result");
|
||||||
assert!(home["tags"]
|
assert!(
|
||||||
|
home["tags"]
|
||||||
.as_array()
|
.as_array()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.iter()
|
.iter()
|
||||||
.any(|tag| tag.as_str() == Some("alpha")));
|
.any(|tag| tag.as_str() == Some("alpha"))
|
||||||
assert!(home["backlinks"]
|
);
|
||||||
|
assert!(
|
||||||
|
home["backlinks"]
|
||||||
.as_array()
|
.as_array()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.iter()
|
.iter()
|
||||||
.any(|link| link.as_str() == Some("Daily")));
|
.any(|link| link.as_str() == Some("Daily"))
|
||||||
assert!(home["backlinks"]
|
);
|
||||||
|
assert!(
|
||||||
|
home["backlinks"]
|
||||||
.as_array()
|
.as_array()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.iter()
|
.iter()
|
||||||
.any(|link| link.as_str() == Some("docs/child.md")));
|
.any(|link| link.as_str() == Some("docs/child.md"))
|
||||||
assert!(home["resourceRefs"]
|
);
|
||||||
|
assert!(
|
||||||
|
home["resourceRefs"]
|
||||||
.as_array()
|
.as_array()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.iter()
|
.iter()
|
||||||
.any(|reference| reference.as_str() == Some("assets/spec.pdf")));
|
.any(|reference| reference.as_str() == Some("assets/spec.pdf"))
|
||||||
assert!(home["resourceRefs"]
|
);
|
||||||
|
assert!(
|
||||||
|
home["resourceRefs"]
|
||||||
.as_array()
|
.as_array()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.iter()
|
.iter()
|
||||||
.any(|reference| reference.as_str() == Some("maps/idea.mindmap.json")));
|
.any(|reference| reference.as_str() == Some("maps/idea.mindmap.json"))
|
||||||
assert!(home["resourceRefs"]
|
);
|
||||||
|
assert!(
|
||||||
|
home["resourceRefs"]
|
||||||
.as_array()
|
.as_array()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.iter()
|
.iter()
|
||||||
.any(|reference| reference.as_str() == Some("office/report.xlsx")));
|
.any(|reference| reference.as_str() == Some("office/report.xlsx"))
|
||||||
assert!(home["publicPath"]
|
);
|
||||||
|
assert!(
|
||||||
|
home["publicPath"]
|
||||||
.as_str()
|
.as_str()
|
||||||
.is_some_and(|path| path.starts_with(
|
.is_some_and(|path| path.starts_with(
|
||||||
"/documents/local-md:README.md?sourceKind=local_folder&rootUri=file%3A%2F%2F"
|
"/documents/local-md:README.md?sourceKind=local_folder&rootUri=file%3A%2F%2F"
|
||||||
)));
|
))
|
||||||
|
);
|
||||||
let evidence_db = root.join(".mnote").join("index").join("evidence.sqlite");
|
let evidence_db = root.join(".mnote").join("index").join("evidence.sqlite");
|
||||||
let connection = Connection::open(&evidence_db).expect("open evidence sqlite");
|
let connection = Connection::open(&evidence_db).expect("open evidence sqlite");
|
||||||
let markdown_edge_count: i64 = connection
|
let markdown_edge_count: i64 = connection
|
||||||
@@ -4201,11 +4479,12 @@ mod tests {
|
|||||||
Some("local-mdid:child-page")
|
Some("local-mdid:child-page")
|
||||||
);
|
);
|
||||||
|
|
||||||
assert!(root
|
assert!(
|
||||||
.join(".mnote")
|
root.join(".mnote")
|
||||||
.join("index")
|
.join("index")
|
||||||
.join("search-index.json")
|
.join("search-index.json")
|
||||||
.exists());
|
.exists()
|
||||||
|
);
|
||||||
let mindmap_projection = query_local_search_index(
|
let mindmap_projection = query_local_search_index(
|
||||||
&root,
|
&root,
|
||||||
&format!("file://{}", root.display()),
|
&format!("file://{}", root.display()),
|
||||||
@@ -4218,19 +4497,19 @@ mod tests {
|
|||||||
false,
|
false,
|
||||||
)
|
)
|
||||||
.expect("mindmap projection");
|
.expect("mindmap projection");
|
||||||
assert!(mindmap_projection["results"]
|
assert!(
|
||||||
|
mindmap_projection["results"]
|
||||||
.as_array()
|
.as_array()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.iter()
|
.iter()
|
||||||
.any(|item| item["resourceType"].as_str() == Some("mindmap")
|
.any(|item| item["resourceType"].as_str() == Some("mindmap")
|
||||||
&& item["path"].as_str() == Some("maps/idea.mindmap.json")
|
&& item["path"].as_str() == Some("maps/idea.mindmap.json")
|
||||||
&& item["publicPath"]
|
&& item["publicPath"].as_str().is_some_and(|path| path
|
||||||
.as_str()
|
|
||||||
.is_some_and(|path| path
|
|
||||||
.starts_with("/?treeView=filetree&sourceKind=local_folder&rootUri="))
|
.starts_with("/?treeView=filetree&sourceKind=local_folder&rootUri="))
|
||||||
&& item["publicPath"]
|
&& item["publicPath"]
|
||||||
.as_str()
|
.as_str()
|
||||||
.is_some_and(|path| !path.starts_with("/tree?"))));
|
.is_some_and(|path| !path.starts_with("/tree?")))
|
||||||
|
);
|
||||||
let office_projection = query_local_search_index(
|
let office_projection = query_local_search_index(
|
||||||
&root,
|
&root,
|
||||||
&format!("file://{}", root.display()),
|
&format!("file://{}", root.display()),
|
||||||
@@ -4243,12 +4522,14 @@ mod tests {
|
|||||||
false,
|
false,
|
||||||
)
|
)
|
||||||
.expect("office projection");
|
.expect("office projection");
|
||||||
assert!(office_projection["results"]
|
assert!(
|
||||||
|
office_projection["results"]
|
||||||
.as_array()
|
.as_array()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.iter()
|
.iter()
|
||||||
.any(|item| item["resourceType"].as_str() == Some("office")
|
.any(|item| item["resourceType"].as_str() == Some("office")
|
||||||
&& item["path"].as_str() == Some("office/report.xlsx")));
|
&& item["path"].as_str() == Some("office/report.xlsx"))
|
||||||
|
);
|
||||||
let pdf_projection = query_local_search_index(
|
let pdf_projection = query_local_search_index(
|
||||||
&root,
|
&root,
|
||||||
&format!("file://{}", root.display()),
|
&format!("file://{}", root.display()),
|
||||||
@@ -4261,12 +4542,34 @@ mod tests {
|
|||||||
false,
|
false,
|
||||||
)
|
)
|
||||||
.expect("pdf projection");
|
.expect("pdf projection");
|
||||||
assert!(pdf_projection["results"]
|
assert!(
|
||||||
|
pdf_projection["results"]
|
||||||
.as_array()
|
.as_array()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.iter()
|
.iter()
|
||||||
.any(|item| item["resourceType"].as_str() == Some("pdf")
|
.any(|item| item["resourceType"].as_str() == Some("pdf")
|
||||||
&& item["path"].as_str() == Some("assets/spec.pdf")));
|
&& item["path"].as_str() == Some("assets/spec.pdf"))
|
||||||
|
);
|
||||||
|
let image_projection = query_local_search_index(
|
||||||
|
&root,
|
||||||
|
&format!("file://{}", root.display()),
|
||||||
|
"local-ws-test",
|
||||||
|
"diagram.png",
|
||||||
|
None,
|
||||||
|
10,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
.expect("image projection");
|
||||||
|
assert!(
|
||||||
|
image_projection["results"]
|
||||||
|
.as_array()
|
||||||
|
.unwrap()
|
||||||
|
.iter()
|
||||||
|
.any(|item| item["resourceType"].as_str() == Some("image")
|
||||||
|
&& item["path"].as_str() == Some("assets/diagram.png"))
|
||||||
|
);
|
||||||
let _ = fs::remove_dir_all(&root);
|
let _ = fs::remove_dir_all(&root);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -4891,10 +5194,12 @@ mod tests {
|
|||||||
let index = read_local_search_index(&root)
|
let index = read_local_search_index(&root)
|
||||||
.expect("read index")
|
.expect("read index")
|
||||||
.expect("index exists");
|
.expect("index exists");
|
||||||
assert!(index
|
assert!(
|
||||||
|
index
|
||||||
.documents
|
.documents
|
||||||
.iter()
|
.iter()
|
||||||
.any(|document| document.path == "README.md"));
|
.any(|document| document.path == "README.md")
|
||||||
|
);
|
||||||
let child = index
|
let child = index
|
||||||
.documents
|
.documents
|
||||||
.iter()
|
.iter()
|
||||||
@@ -4922,14 +5227,18 @@ mod tests {
|
|||||||
let index = read_local_search_index(&root)
|
let index = read_local_search_index(&root)
|
||||||
.expect("read index")
|
.expect("read index")
|
||||||
.expect("index exists");
|
.expect("index exists");
|
||||||
assert!(!index
|
assert!(
|
||||||
|
!index
|
||||||
.documents
|
.documents
|
||||||
.iter()
|
.iter()
|
||||||
.any(|document| document.path == "docs/child.md"));
|
.any(|document| document.path == "docs/child.md")
|
||||||
assert!(index
|
);
|
||||||
|
assert!(
|
||||||
|
index
|
||||||
.documents
|
.documents
|
||||||
.iter()
|
.iter()
|
||||||
.any(|document| document.path == "README.md"));
|
.any(|document| document.path == "README.md")
|
||||||
|
);
|
||||||
let removed_evidence = query_evidence_sqlite_results(&root, "ChangedToken", None, 10)
|
let removed_evidence = query_evidence_sqlite_results(&root, "ChangedToken", None, 10)
|
||||||
.expect("removed evidence")
|
.expect("removed evidence")
|
||||||
.expect("sqlite exists");
|
.expect("sqlite exists");
|
||||||
@@ -4990,6 +5299,8 @@ mod tests {
|
|||||||
.expect("serialize index"),
|
.expect("serialize index"),
|
||||||
)
|
)
|
||||||
.expect("ocr index");
|
.expect("ocr index");
|
||||||
|
write_local_index_settings(&root, &[String::from(".")], None, None, None, None)
|
||||||
|
.expect("settings");
|
||||||
|
|
||||||
let without_ocr = query_local_search_index(
|
let without_ocr = query_local_search_index(
|
||||||
&root,
|
&root,
|
||||||
@@ -5053,14 +5364,18 @@ mod tests {
|
|||||||
let index = read_local_search_index(&root)
|
let index = read_local_search_index(&root)
|
||||||
.expect("read search index")
|
.expect("read search index")
|
||||||
.expect("search index");
|
.expect("search index");
|
||||||
assert!(!index
|
assert!(
|
||||||
|
!index
|
||||||
.documents
|
.documents
|
||||||
.iter()
|
.iter()
|
||||||
.any(|document| document.path == "docs/Page.ocr/photo.png.ocr.md"));
|
.any(|document| document.path == "docs/Page.ocr/photo.png.ocr.md")
|
||||||
assert!(!index
|
);
|
||||||
|
assert!(
|
||||||
|
!index
|
||||||
.documents
|
.documents
|
||||||
.iter()
|
.iter()
|
||||||
.any(|document| document.path == "docs/Page.ocr/photo.png-704905.ocr.md"));
|
.any(|document| document.path == "docs/Page.ocr/photo.png-704905.ocr.md")
|
||||||
|
);
|
||||||
refresh_local_search_index_for_path(
|
refresh_local_search_index_for_path(
|
||||||
&root,
|
&root,
|
||||||
&root_uri,
|
&root_uri,
|
||||||
@@ -5071,10 +5386,12 @@ mod tests {
|
|||||||
let refreshed_index = read_local_search_index(&root)
|
let refreshed_index = read_local_search_index(&root)
|
||||||
.expect("read refreshed search index")
|
.expect("read refreshed search index")
|
||||||
.expect("refreshed search index");
|
.expect("refreshed search index");
|
||||||
assert!(!refreshed_index
|
assert!(
|
||||||
|
!refreshed_index
|
||||||
.documents
|
.documents
|
||||||
.iter()
|
.iter()
|
||||||
.any(|document| document.path == "docs/Page.ocr/photo.png-704905.ocr.md"));
|
.any(|document| document.path == "docs/Page.ocr/photo.png-704905.ocr.md")
|
||||||
|
);
|
||||||
let _ = fs::remove_dir_all(&root);
|
let _ = fs::remove_dir_all(&root);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -5175,15 +5492,21 @@ mod tests {
|
|||||||
.expect("sqlite query")
|
.expect("sqlite query")
|
||||||
.expect("sqlite exists");
|
.expect("sqlite exists");
|
||||||
assert!(results.len() >= 2);
|
assert!(results.len() >= 2);
|
||||||
assert!(results
|
assert!(
|
||||||
|
results
|
||||||
.iter()
|
.iter()
|
||||||
.any(|result| result.source.owner_document_path == "README.md"));
|
.any(|result| result.source.owner_document_path == "README.md")
|
||||||
assert!(results
|
);
|
||||||
|
assert!(
|
||||||
|
results
|
||||||
.iter()
|
.iter()
|
||||||
.any(|result| result.source.owner_document_path == "docs/child.md"));
|
.any(|result| result.source.owner_document_path == "docs/child.md")
|
||||||
assert!(results
|
);
|
||||||
|
assert!(
|
||||||
|
results
|
||||||
.iter()
|
.iter()
|
||||||
.all(|result| result.source.schema == "mnote.evidence_locator.v1"));
|
.all(|result| result.source.schema == "mnote.evidence_locator.v1")
|
||||||
|
);
|
||||||
let home_result = results
|
let home_result = results
|
||||||
.iter()
|
.iter()
|
||||||
.find(|result| result.source.owner_document_path == "README.md")
|
.find(|result| result.source.owner_document_path == "README.md")
|
||||||
@@ -5265,6 +5588,55 @@ mod tests {
|
|||||||
let _ = fs::remove_dir_all(&root);
|
let _ = fs::remove_dir_all(&root);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn evidence_sqlite_fuzzy_uses_cjk_terms_without_overrequiring_suffix() {
|
||||||
|
let root = temp_root("mnote-evidence-sqlite-cjk-fuzzy");
|
||||||
|
let root_uri = format!("file://{}", root.display());
|
||||||
|
let workspace_id = "local-ws-evidence-cjk";
|
||||||
|
fs::write(
|
||||||
|
root.join("README.md"),
|
||||||
|
"# 有机合成\n羧酸需要被保护有许多原因,常见做法是形成稳定酯类后再脱保护。\n",
|
||||||
|
)
|
||||||
|
.expect("write home");
|
||||||
|
|
||||||
|
write_local_index_settings(&root, &[String::from(".")], None, None, None, None)
|
||||||
|
.expect("settings");
|
||||||
|
refresh_local_search_index(&root, &root_uri, workspace_id).expect("refresh");
|
||||||
|
|
||||||
|
let exact = query_evidence_sqlite_results_with_mode(&root, "羧酸 保护基", None, 10, true)
|
||||||
|
.expect("exact sqlite query")
|
||||||
|
.expect("sqlite exists");
|
||||||
|
assert!(exact.is_empty(), "精确短语不应伪造不存在的同串命中");
|
||||||
|
|
||||||
|
let fuzzy = query_evidence_sqlite_results_with_mode(&root, "羧酸 保护基", None, 10, false)
|
||||||
|
.expect("fuzzy sqlite query")
|
||||||
|
.expect("sqlite exists");
|
||||||
|
assert_eq!(fuzzy.len(), 1);
|
||||||
|
let info = fuzzy[0].match_info.as_ref().expect("match info");
|
||||||
|
assert_eq!(info.match_mode, "cjk_partial");
|
||||||
|
assert!(!info.strong);
|
||||||
|
assert!(info.matched_terms.iter().any(|term| term == "羧酸"));
|
||||||
|
assert!(info.matched_terms.iter().any(|term| term == "保护"));
|
||||||
|
assert!(info.missing_terms.iter().any(|term| term == "保护基"));
|
||||||
|
assert!(fuzzy[0].quote.contains("羧酸"));
|
||||||
|
|
||||||
|
let projection = query_local_search_index(
|
||||||
|
&root,
|
||||||
|
&root_uri,
|
||||||
|
workspace_id,
|
||||||
|
"羧酸 保护基",
|
||||||
|
None,
|
||||||
|
10,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
.expect("local search projection");
|
||||||
|
assert_eq!(projection["results"].as_array().map(Vec::len), Some(1));
|
||||||
|
|
||||||
|
let _ = fs::remove_dir_all(&root);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn evidence_sqlite_read_context_returns_anchor_block() {
|
fn evidence_sqlite_read_context_returns_anchor_block() {
|
||||||
let root = temp_root("mnote-evidence-sqlite-read-context");
|
let root = temp_root("mnote-evidence-sqlite-read-context");
|
||||||
@@ -5292,12 +5664,16 @@ mod tests {
|
|||||||
.expect("sqlite exists");
|
.expect("sqlite exists");
|
||||||
|
|
||||||
assert_eq!(context.len(), 2);
|
assert_eq!(context.len(), 2);
|
||||||
assert!(context
|
assert!(
|
||||||
|
context
|
||||||
.iter()
|
.iter()
|
||||||
.all(|item| item.source.owner_document_path == "README.md"));
|
.all(|item| item.source.owner_document_path == "README.md")
|
||||||
assert!(context
|
);
|
||||||
|
assert!(
|
||||||
|
context
|
||||||
.iter()
|
.iter()
|
||||||
.any(|item| item.quote.contains("ReadContextToken")));
|
.any(|item| item.quote.contains("ReadContextToken"))
|
||||||
|
);
|
||||||
|
|
||||||
let _ = fs::remove_dir_all(&root);
|
let _ = fs::remove_dir_all(&root);
|
||||||
}
|
}
|
||||||
@@ -5395,16 +5771,18 @@ JSON
|
|||||||
Some("docs/Page.assets/spec.pdf"),
|
Some("docs/Page.assets/spec.pdf"),
|
||||||
"页面内搜索打开资源时应只限制到当前资源,而不是它的 owner Markdown"
|
"页面内搜索打开资源时应只限制到当前资源,而不是它的 owner Markdown"
|
||||||
);
|
);
|
||||||
assert!(root
|
assert!(
|
||||||
.join("docs")
|
root.join("docs")
|
||||||
.join("Page.ocr")
|
.join("Page.ocr")
|
||||||
.join("spec.pdf.parse.md")
|
.join("spec.pdf.parse.md")
|
||||||
.exists());
|
.exists()
|
||||||
assert!(root
|
);
|
||||||
.join("docs")
|
assert!(
|
||||||
|
root.join("docs")
|
||||||
.join("Page.ocr")
|
.join("Page.ocr")
|
||||||
.join("spec.pdf.source-map.json")
|
.join("spec.pdf.source-map.json")
|
||||||
.exists());
|
.exists()
|
||||||
|
);
|
||||||
|
|
||||||
let _ = fs::remove_dir_all(&root);
|
let _ = fs::remove_dir_all(&root);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,14 +9,14 @@ use crate::routes::query_support::{
|
|||||||
use crate::routes::web_shell::load_sidebar_tree_html;
|
use crate::routes::web_shell::load_sidebar_tree_html;
|
||||||
use crate::routes::{evidence, local_folder_source, local_search_index};
|
use crate::routes::{evidence, local_folder_source, local_search_index};
|
||||||
use crate::ssr::pages::search::SearchPage;
|
use crate::ssr::pages::search::SearchPage;
|
||||||
|
use axum::Json;
|
||||||
use axum::extract::{Extension, Query, State};
|
use axum::extract::{Extension, Query, State};
|
||||||
use axum::http::{HeaderMap, HeaderName, HeaderValue, StatusCode};
|
use axum::http::{HeaderMap, HeaderName, HeaderValue, StatusCode};
|
||||||
use axum::response::{Html, IntoResponse, Response};
|
use axum::response::{Html, IntoResponse, Response};
|
||||||
use axum::Json;
|
|
||||||
use bridge_runtime::RuntimeQueryEnvelopeWire;
|
use bridge_runtime::RuntimeQueryEnvelopeWire;
|
||||||
use core_protocol::{EvidenceSearchMode, EvidenceSearchResult};
|
use core_protocol::{EvidenceSearchMode, EvidenceSearchResult};
|
||||||
use serde::Deserialize;
|
use serde::Deserialize;
|
||||||
use serde_json::{json, Value};
|
use serde_json::{Value, json};
|
||||||
|
|
||||||
const HEADER_MNOTE_WEB_OWNER: &str = "x-mnote-web-owner";
|
const HEADER_MNOTE_WEB_OWNER: &str = "x-mnote-web-owner";
|
||||||
const HEADER_QUERY_NAME: &str = "x-query-name";
|
const HEADER_QUERY_NAME: &str = "x-query-name";
|
||||||
@@ -239,6 +239,16 @@ pub async fn documents(
|
|||||||
filters.exact.unwrap_or(false),
|
filters.exact.unwrap_or(false),
|
||||||
)?
|
)?
|
||||||
.unwrap_or_default()
|
.unwrap_or_default()
|
||||||
|
.into_iter()
|
||||||
|
.filter(|evidence| {
|
||||||
|
let path = evidence
|
||||||
|
.source
|
||||||
|
.resource_path
|
||||||
|
.as_deref()
|
||||||
|
.unwrap_or(evidence.source.owner_document_path.as_str());
|
||||||
|
local_search_index::local_index_relative_path_is_included(path, &user_settings)
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>()
|
||||||
} else {
|
} else {
|
||||||
Vec::new()
|
Vec::new()
|
||||||
};
|
};
|
||||||
@@ -438,12 +448,6 @@ pub async fn update_local_index_settings(
|
|||||||
&effective_workspace_id,
|
&effective_workspace_id,
|
||||||
&root_path,
|
&root_path,
|
||||||
)?;
|
)?;
|
||||||
let refreshed = local_search_index::refresh_local_search_index_with_settings(
|
|
||||||
&root_path,
|
|
||||||
root_uri,
|
|
||||||
&effective_workspace_id,
|
|
||||||
&effective_settings,
|
|
||||||
)?;
|
|
||||||
let status = local_search_index::local_index_status_with_settings(
|
let status = local_search_index::local_index_status_with_settings(
|
||||||
&root_path,
|
&root_path,
|
||||||
root_uri,
|
root_uri,
|
||||||
@@ -459,7 +463,6 @@ pub async fn update_local_index_settings(
|
|||||||
Json(json!({
|
Json(json!({
|
||||||
"ok": true,
|
"ok": true,
|
||||||
"settings": settings,
|
"settings": settings,
|
||||||
"index": refreshed,
|
|
||||||
"result": status,
|
"result": status,
|
||||||
"meta": {
|
"meta": {
|
||||||
"owner": "mnote-web",
|
"owner": "mnote-web",
|
||||||
@@ -481,26 +484,72 @@ fn merge_local_search_with_evidence_results(
|
|||||||
if direct_evidence_results.is_empty() {
|
if direct_evidence_results.is_empty() {
|
||||||
return (result, evidence_results);
|
return (result, evidence_results);
|
||||||
}
|
}
|
||||||
let mut result_items = result
|
let original_items = result
|
||||||
.get("results")
|
.get("results")
|
||||||
.and_then(Value::as_array)
|
.and_then(Value::as_array)
|
||||||
.cloned()
|
.cloned()
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
let mut seen_evidence_ids = evidence_results
|
let original_evidence_results = std::mem::take(&mut evidence_results);
|
||||||
.iter()
|
let mut result_items = Vec::new();
|
||||||
.map(|item| item.evidence_id.clone())
|
let mut merged_evidence_results = Vec::new();
|
||||||
.collect::<std::collections::HashSet<_>>();
|
let mut seen_result_ids = std::collections::HashSet::new();
|
||||||
|
let mut seen_evidence_ids = std::collections::HashSet::new();
|
||||||
|
let mut seen_paths = std::collections::HashSet::new();
|
||||||
for evidence in direct_evidence_results {
|
for evidence in direct_evidence_results {
|
||||||
if result_items.len() >= limit || !seen_evidence_ids.insert(evidence.evidence_id.clone()) {
|
if result_items.len() >= limit || !seen_evidence_ids.insert(evidence.evidence_id.clone()) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
result_items.push(search_result_from_evidence(&evidence));
|
let item = search_result_from_evidence(&evidence);
|
||||||
evidence_results.push(evidence);
|
if let Some(id) = item.get("id").and_then(Value::as_str) {
|
||||||
|
seen_result_ids.insert(id.to_string());
|
||||||
|
}
|
||||||
|
if let Some(path) = search_result_dedupe_path(&item) {
|
||||||
|
seen_paths.insert(path);
|
||||||
|
}
|
||||||
|
result_items.push(item);
|
||||||
|
merged_evidence_results.push(evidence);
|
||||||
|
}
|
||||||
|
for (index, item) in original_items.into_iter().enumerate() {
|
||||||
|
if result_items.len() >= limit {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
let item_id = item
|
||||||
|
.get("id")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.map(ToOwned::to_owned)
|
||||||
|
.unwrap_or_default();
|
||||||
|
if !item_id.is_empty() && !seen_result_ids.insert(item_id) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if let Some(path) = search_result_dedupe_path(&item) {
|
||||||
|
if !seen_paths.insert(path) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Some(evidence) = original_evidence_results.get(index).cloned() {
|
||||||
|
merged_evidence_results.push(evidence);
|
||||||
|
}
|
||||||
|
result_items.push(item);
|
||||||
}
|
}
|
||||||
if let Some(map) = result.as_object_mut() {
|
if let Some(map) = result.as_object_mut() {
|
||||||
map.insert("results".into(), Value::Array(result_items));
|
map.insert("results".into(), Value::Array(result_items));
|
||||||
}
|
}
|
||||||
(result, evidence_results)
|
(result, merged_evidence_results)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn search_result_dedupe_path(item: &Value) -> Option<String> {
|
||||||
|
let source_kind = item
|
||||||
|
.get("sourceKind")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.unwrap_or_default();
|
||||||
|
if source_kind != "local_folder" {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
item.get("path")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|value| !value.is_empty())
|
||||||
|
.map(ToOwned::to_owned)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn search_result_from_evidence(evidence: &EvidenceSearchResult) -> Value {
|
fn search_result_from_evidence(evidence: &EvidenceSearchResult) -> Value {
|
||||||
@@ -533,6 +582,7 @@ fn search_result_from_evidence(evidence: &EvidenceSearchResult) -> Value {
|
|||||||
"rootUri": source.root_uri,
|
"rootUri": source.root_uri,
|
||||||
"snippet": evidence.quote,
|
"snippet": evidence.quote,
|
||||||
"score": evidence.score,
|
"score": evidence.score,
|
||||||
|
"matchInfo": evidence.match_info,
|
||||||
"publicPath": source.open_action.url,
|
"publicPath": source.open_action.url,
|
||||||
"evidence": evidence_value,
|
"evidence": evidence_value,
|
||||||
"source": {
|
"source": {
|
||||||
@@ -890,12 +940,12 @@ fn stamp_search_headers(headers: &mut HeaderMap) {
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use crate::app::{build_app, AppConfig, AppState};
|
use crate::app::{AppConfig, AppState, build_app};
|
||||||
use crate::routes::local_search_index;
|
use crate::routes::local_search_index;
|
||||||
use axum::body::{to_bytes, Body};
|
use axum::body::{Body, to_bytes};
|
||||||
use axum::http::{Request, StatusCode};
|
use axum::http::{Request, StatusCode};
|
||||||
use control_plane::{DirectoryGrantInput, UpsertUserInput};
|
use control_plane::{DirectoryGrantInput, UpsertUserInput};
|
||||||
use serde_json::{json, Value};
|
use serde_json::{Value, json};
|
||||||
use std::fs;
|
use std::fs;
|
||||||
use tower::util::ServiceExt;
|
use tower::util::ServiceExt;
|
||||||
|
|
||||||
@@ -1152,26 +1202,33 @@ mod tests {
|
|||||||
Some("README.md")
|
Some("README.md")
|
||||||
);
|
);
|
||||||
assert!(!payload["evidence"].as_array().expect("evidence").is_empty());
|
assert!(!payload["evidence"].as_array().expect("evidence").is_empty());
|
||||||
assert!(home["tags"]
|
assert!(
|
||||||
|
home["tags"]
|
||||||
.as_array()
|
.as_array()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.iter()
|
.iter()
|
||||||
.any(|tag| tag.as_str() == Some("alpha")));
|
.any(|tag| tag.as_str() == Some("alpha"))
|
||||||
assert!(home["backlinks"]
|
);
|
||||||
|
assert!(
|
||||||
|
home["backlinks"]
|
||||||
.as_array()
|
.as_array()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.iter()
|
.iter()
|
||||||
.any(|link| link.as_str() == Some("Daily")));
|
.any(|link| link.as_str() == Some("Daily"))
|
||||||
assert!(home["resourceRefs"]
|
);
|
||||||
|
assert!(
|
||||||
|
home["resourceRefs"]
|
||||||
.as_array()
|
.as_array()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.iter()
|
.iter()
|
||||||
.any(|reference| reference.as_str() == Some("assets/spec.pdf")));
|
.any(|reference| reference.as_str() == Some("assets/spec.pdf"))
|
||||||
assert!(root
|
);
|
||||||
.join(".mnote")
|
assert!(
|
||||||
|
root.join(".mnote")
|
||||||
.join("index")
|
.join("index")
|
||||||
.join("search-index.json")
|
.join("search-index.json")
|
||||||
.exists());
|
.exists()
|
||||||
|
);
|
||||||
let evidence_db = root.join(".mnote").join("index").join("evidence.sqlite");
|
let evidence_db = root.join(".mnote").join("index").join("evidence.sqlite");
|
||||||
assert!(evidence_db.exists(), "evidence sqlite should be built");
|
assert!(evidence_db.exists(), "evidence sqlite should be built");
|
||||||
let connection = rusqlite::Connection::open(&evidence_db).expect("open evidence sqlite");
|
let connection = rusqlite::Connection::open(&evidence_db).expect("open evidence sqlite");
|
||||||
@@ -1201,11 +1258,13 @@ mod tests {
|
|||||||
locator["schema"].as_str(),
|
locator["schema"].as_str(),
|
||||||
Some("mnote.evidence_locator.v1")
|
Some("mnote.evidence_locator.v1")
|
||||||
);
|
);
|
||||||
assert!(payload["recent"]
|
assert!(
|
||||||
|
payload["recent"]
|
||||||
.as_array()
|
.as_array()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.iter()
|
.iter()
|
||||||
.any(|item| item["documentId"].as_str() == Some("local-md:README.md")));
|
.any(|item| item["documentId"].as_str() == Some("local-md:README.md"))
|
||||||
|
);
|
||||||
|
|
||||||
let _ = fs::remove_dir_all(&root);
|
let _ = fs::remove_dir_all(&root);
|
||||||
}
|
}
|
||||||
@@ -1409,10 +1468,32 @@ mod tests {
|
|||||||
.await
|
.await
|
||||||
.expect("create settings response");
|
.expect("create settings response");
|
||||||
assert_eq!(create_response.status(), StatusCode::OK);
|
assert_eq!(create_response.status(), StatusCode::OK);
|
||||||
|
let create_refresh_response = app
|
||||||
|
.clone()
|
||||||
|
.oneshot(
|
||||||
|
Request::builder()
|
||||||
|
.method("POST")
|
||||||
|
.uri("/api/search/local-index/refresh")
|
||||||
|
.header("content-type", "application/json")
|
||||||
|
.header("x-mnote-actor-id", "user_test")
|
||||||
|
.header("x-mnote-actor-type", "user")
|
||||||
|
.body(Body::from(
|
||||||
|
json!({
|
||||||
|
"workspaceId": "local-ws-settings-delete",
|
||||||
|
"rootUri": root_uri
|
||||||
|
})
|
||||||
|
.to_string(),
|
||||||
|
))
|
||||||
|
.expect("create refresh request"),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("create refresh response");
|
||||||
|
assert_eq!(create_refresh_response.status(), StatusCode::OK);
|
||||||
assert!(root.join(".mnote/index/search-index.json").exists());
|
assert!(root.join(".mnote/index/search-index.json").exists());
|
||||||
assert!(root.join(".mnote/index/evidence.sqlite").exists());
|
assert!(root.join(".mnote/index/evidence.sqlite").exists());
|
||||||
|
|
||||||
let delete_response = app
|
let delete_response = app
|
||||||
|
.clone()
|
||||||
.oneshot(
|
.oneshot(
|
||||||
Request::builder()
|
Request::builder()
|
||||||
.method("PUT")
|
.method("PUT")
|
||||||
@@ -1440,18 +1521,36 @@ mod tests {
|
|||||||
.await
|
.await
|
||||||
.expect("delete body");
|
.expect("delete body");
|
||||||
let delete_payload: Value = serde_json::from_slice(&delete_body).expect("delete json");
|
let delete_payload: Value = serde_json::from_slice(&delete_body).expect("delete json");
|
||||||
assert_eq!(
|
|
||||||
delete_payload["index"]["indexedPaths"]
|
|
||||||
.as_array()
|
|
||||||
.map(Vec::len),
|
|
||||||
Some(0)
|
|
||||||
);
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
delete_payload["result"]["settings"]["includePaths"]
|
delete_payload["result"]["settings"]["includePaths"]
|
||||||
.as_array()
|
.as_array()
|
||||||
.map(Vec::len),
|
.map(Vec::len),
|
||||||
Some(0)
|
Some(0)
|
||||||
);
|
);
|
||||||
|
assert!(root.join(".mnote/index/search-index.json").exists());
|
||||||
|
assert!(root.join(".mnote/index/evidence.sqlite").exists());
|
||||||
|
|
||||||
|
let delete_refresh_response = app
|
||||||
|
.clone()
|
||||||
|
.oneshot(
|
||||||
|
Request::builder()
|
||||||
|
.method("POST")
|
||||||
|
.uri("/api/search/local-index/refresh")
|
||||||
|
.header("content-type", "application/json")
|
||||||
|
.header("x-mnote-actor-id", "user_test")
|
||||||
|
.header("x-mnote-actor-type", "user")
|
||||||
|
.body(Body::from(
|
||||||
|
json!({
|
||||||
|
"workspaceId": "local-ws-settings-delete",
|
||||||
|
"rootUri": root_uri
|
||||||
|
})
|
||||||
|
.to_string(),
|
||||||
|
))
|
||||||
|
.expect("delete refresh request"),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("delete refresh response");
|
||||||
|
assert_eq!(delete_refresh_response.status(), StatusCode::OK);
|
||||||
assert!(!root.join(".mnote/index/search-index.json").exists());
|
assert!(!root.join(".mnote/index/search-index.json").exists());
|
||||||
assert!(!root.join(".mnote/index/evidence.sqlite").exists());
|
assert!(!root.join(".mnote/index/evidence.sqlite").exists());
|
||||||
|
|
||||||
@@ -1508,11 +1607,13 @@ mod tests {
|
|||||||
backlinks_payload["meta"]["queryName"].as_str(),
|
backlinks_payload["meta"]["queryName"].as_str(),
|
||||||
Some("search.local_index.backlinks")
|
Some("search.local_index.backlinks")
|
||||||
);
|
);
|
||||||
assert!(backlinks_payload["result"]["backlinks"]
|
assert!(
|
||||||
|
backlinks_payload["result"]["backlinks"]
|
||||||
.as_array()
|
.as_array()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.iter()
|
.iter()
|
||||||
.any(|item| item["documentId"].as_str() == Some("local-md:README.md")));
|
.any(|item| item["documentId"].as_str() == Some("local-md:README.md"))
|
||||||
|
);
|
||||||
|
|
||||||
let tags_response = app()
|
let tags_response = app()
|
||||||
.oneshot(
|
.oneshot(
|
||||||
|
|||||||
Reference in New Issue
Block a user