lix/mnote
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve evidence search ranking and diagnostics

Browse Source
This commit is contained in:
lix-2026
2026-06-06 00:33:02 +08:00
parent 4dbd9a978b
commit 5157a5960c
6 changed files with 891 additions and 235 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rust/crates/mnote-web/src/routes/local_search_index.rs
+536 -158
View File
@@ -1,6 +1,6 @@
use crate::error::WebError;
use crate::evidence_parse::{
liteparse_runtime_available, parse_liteparse_input_blocking, ParseInput, ParseProviderMode,
ParseInput, ParseProviderMode, liteparse_runtime_available, parse_liteparse_input_blocking,
};
use crate::routes::local_folder_source::encode_local_id_segment;
use crate::routes::local_markdown_parser::{
@@ -9,12 +9,12 @@ use crate::routes::local_markdown_parser::{
use crate::routes::local_ocr;
use control_plane::{ControlPlaneStore, UpsertUserInput, UpsertUserUiPreferenceInput};
use core_protocol::{
EvidenceLocator, EvidenceSearchResult, ParsedResourceArtifact, ResourceSourceMap,
SourceMapBlock, PARSED_RESOURCE_ARTIFACT_SCHEMA,
EvidenceLocator, EvidenceSearchMatchInfo, EvidenceSearchResult,
PARSED_RESOURCE_ARTIFACT_SCHEMA, ParsedResourceArtifact, ResourceSourceMap, SourceMapBlock,
};
use rusqlite::{params, Connection, OptionalExtension};
use rusqlite::{Connection, OptionalExtension, params};
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
use serde_json::{Value, json};
use std::collections::BTreeSet;
use std::fs;
use std::path::{Component, Path, PathBuf};
@@ -671,6 +671,7 @@ pub(crate) fn refresh_local_search_index_if_scheduled_due_with_store(
.map(Some)
}
#[cfg(test)]
pub(crate) fn query_evidence_sqlite_results(
root_path: &Path,
query: &str,
@@ -812,6 +813,7 @@ pub(crate) fn read_evidence_sqlite_context(
0.8
},
source: source.clone(),
match_info: None,
citation_url: None,
citation_label: None,
citation_markdown: None,
@@ -901,6 +903,7 @@ fn graph_edge_row(row: &rusqlite::Row<'_>) -> rusqlite::Result<EvidenceSearchRes
block_id: source.block_id.or(Some(source_block_id)),
..source
},
match_info: None,
citation_url: None,
citation_label: None,
citation_markdown: None,
@@ -1059,43 +1062,76 @@ fn query_evidence_sqlite_fuzzy(
let scope_value = owner_document_id.or(resource_path);
let rows = match (scope_value, first_char_like.as_deref()) {
(Some(scope_value), Some(first_char_like)) => statement
.query_map(params![scope_value, first_char_like, scan_limit], |row| {
evidence_result_from_sqlite_row(row, query)
})
.query_map(
params![scope_value, first_char_like, scan_limit],
evidence_sqlite_row_parts,
)
.map_err(sqlite_error)?
.collect::<Result<Vec<_>, _>>(),
(Some(scope_value), None) => statement
.query_map(params![scope_value, scan_limit], |row| {
evidence_result_from_sqlite_row(row, query)
})
.query_map(params![scope_value, scan_limit], evidence_sqlite_row_parts)
.map_err(sqlite_error)?
.collect::<Result<Vec<_>, _>>(),
(None, Some(first_char_like)) => statement
.query_map(params![first_char_like, scan_limit], |row| {
evidence_result_from_sqlite_row(row, query)
})
.query_map(
params![first_char_like, scan_limit],
evidence_sqlite_row_parts,
)
.map_err(sqlite_error)?
.collect::<Result<Vec<_>, _>>(),
(None, None) => statement
.query_map(params![scan_limit], |row| {
evidence_result_from_sqlite_row(row, query)
})
.query_map(params![scan_limit], evidence_sqlite_row_parts)
.map_err(sqlite_error)?
.collect::<Result<Vec<_>, _>>(),
}
.map_err(sqlite_error)?;
Ok(rows
let mut scored = rows
.into_iter()
.filter(|result| {
fuzzy_search_match(
&normalize_search_text(&result.quote),
&normalize_search_text(query),
)
.filter_map(|(block_id, text, source)| {
let text_match = score_evidence_text_match(&text, query)?;
Some(EvidenceSearchResult {
evidence_id: block_id,
quote: ocr_search_snippet_for_terms(&text, query, &text_match.matched_terms),
score: text_match.score,
source,
match_info: Some(text_match.into_match_info(None)),
citation_url: None,
citation_label: None,
citation_markdown: None,
})
})
.collect::<Vec<_>>();
scored.sort_by(|left, right| {
right
.score
.partial_cmp(&left.score)
.unwrap_or(std::cmp::Ordering::Equal)
});
Ok(scored
.into_iter()
.enumerate()
.map(|(index, mut result)| {
if let Some(match_info) = result.match_info.as_mut() {
match_info.rank = Some((index + 1) as u32);
}
result
})
.take(limit.max(1) as usize)
.collect())
}
fn evidence_sqlite_row_parts(
row: &rusqlite::Row<'_>,
) -> rusqlite::Result<(String, String, EvidenceLocator)> {
let block_id: String = row.get(0)?;
let text: String = row.get(1)?;
let locator_json: String = row.get(2)?;
let source = serde_json::from_str::<EvidenceLocator>(&locator_json).map_err(|error| {
rusqlite::Error::FromSqlConversionFailure(2, rusqlite::types::Type::Text, Box::new(error))
})?;
Ok((block_id, text, source))
}
fn evidence_result_from_sqlite_row(
row: &rusqlite::Row<'_>,
query: &str,
@@ -1107,15 +1143,20 @@ fn evidence_result_from_sqlite_row(
let source = serde_json::from_str::<EvidenceLocator>(&locator_json).map_err(|error| {
rusqlite::Error::FromSqlConversionFailure(2, rusqlite::types::Type::Text, Box::new(error))
})?;
let text_match = score_evidence_text_match(&text, query);
Ok(EvidenceSearchResult {
evidence_id: block_id,
quote: ocr_search_snippet(&text, query),
quote: text_match
.as_ref()
.map(|text_match| ocr_search_snippet_for_terms(&text, query, &text_match.matched_terms))
.unwrap_or_else(|| ocr_search_snippet(&text, query)),
score: if rank == 0.0 {
1.0
} else {
1.0 / (1.0 + rank.abs())
},
source,
match_info: text_match.map(|text_match| text_match.into_match_info(None)),
citation_url: None,
citation_label: None,
citation_markdown: None,
@@ -1952,6 +1993,13 @@ fn index_relative_path_is_included(relative_path: &str, include_paths: &[String]
})
}
pub(crate) fn local_index_relative_path_is_included(
relative_path: &str,
settings: &LocalIndexSettings,
) -> bool {
index_relative_path_is_included(relative_path, &settings.include_paths)
}
fn collect_markdown_documents(
root_path: &Path,
current: &Path,
@@ -3605,7 +3653,7 @@ fn local_search_document_matches(
if exact {
haystack.contains(query)
} else {
fuzzy_search_match(&haystack, query)
token_search_match(&haystack, query)
}
}
@@ -3629,7 +3677,7 @@ fn local_search_resource_matches(
if exact {
haystack.contains(query)
} else {
fuzzy_search_match(&haystack, query)
token_search_match(&haystack, query)
}
}
@@ -3657,7 +3705,7 @@ fn local_search_ocr_matches(
if exact {
haystack.contains(query)
} else {
fuzzy_search_match(&haystack, query)
token_search_match(&haystack, query)
}
}
@@ -3929,6 +3977,203 @@ fn search_document_hit(document: &LocalSearchDocument, query: &str) -> SearchDoc
}
}
#[derive(Debug, Clone)]
struct EvidenceQueryTerm {
display: String,
normalized: String,
alternatives: Vec<String>,
}
#[derive(Debug, Clone)]
struct EvidenceTextMatch {
score: f64,
matched_terms: Vec<String>,
missing_terms: Vec<String>,
match_mode: String,
strong: bool,
}
impl EvidenceTextMatch {
fn into_match_info(self, rank: Option<u32>) -> EvidenceSearchMatchInfo {
EvidenceSearchMatchInfo {
rank,
matched_terms: self.matched_terms,
missing_terms: self.missing_terms,
match_mode: self.match_mode,
match_scope: "block".into(),
strong: self.strong,
}
}
}
fn score_evidence_text_match(text: &str, query: &str) -> Option<EvidenceTextMatch> {
let normalized_text = normalize_search_text(text);
let normalized_query = normalize_search_text(query);
if normalized_query.is_empty() {
return None;
}
let terms = evidence_query_terms(&normalized_query);
if terms.is_empty() {
return None;
}
let phrase_exact = normalized_text.contains(&normalized_query);
let mut matched_terms = Vec::new();
let mut missing_terms = Vec::new();
let mut exact_count = 0usize;
let mut partial_count = 0usize;
let mut positions = Vec::new();
for term in &terms {
if let Some(index) = normalized_text.find(&term.normalized) {
exact_count += 1;
positions.push(index);
push_unique(&mut matched_terms, term.display.clone());
continue;
}
let partial_matches = term
.alternatives
.iter()
.filter_map(|alternative| {
normalized_text
.find(alternative)
.map(|index| (alternative.clone(), index))
})
.collect::<Vec<_>>();
let partial_allowed = if terms.len() > 1 {
!partial_matches.is_empty()
} else {
partial_matches.len() >= 2 || term.normalized.chars().count() <= 2
};
if partial_allowed {
partial_count += 1;
if let Some((_, index)) = partial_matches.first() {
positions.push(*index);
}
for (alternative, _) in partial_matches {
push_unique(&mut matched_terms, alternative);
}
push_unique(&mut missing_terms, term.display.clone());
} else {
return None;
}
}
let term_count = terms.len().max(1) as f64;
let coverage = (exact_count as f64 + partial_count as f64 * 0.45) / term_count;
let proximity = if positions.len() > 1 {
let min = positions.iter().min().copied().unwrap_or(0);
let max = positions.iter().max().copied().unwrap_or(min);
let span = max.saturating_sub(min);
if span <= 80 {
0.1
} else if span <= 240 {
0.05
} else {
0.0
}
} else {
0.0
};
let score = if phrase_exact {
1.0
} else {
(coverage * 0.85 + proximity).min(0.95)
};
let match_mode = if phrase_exact {
"exact_phrase"
} else if partial_count == 0 {
"all_terms"
} else {
"cjk_partial"
};
Some(EvidenceTextMatch {
score,
matched_terms,
missing_terms,
match_mode: match_mode.into(),
strong: phrase_exact || partial_count == 0 && exact_count == terms.len(),
})
}
fn evidence_query_terms(query: &str) -> Vec<EvidenceQueryTerm> {
split_search_query_tokens(query)
.into_iter()
.filter(|token| !token.is_empty())
.map(|token| {
let alternatives = cjk_token_bigrams(&token)
.into_iter()
.filter(|alternative| alternative != &token)
.collect::<Vec<_>>();
EvidenceQueryTerm {
display: token.clone(),
normalized: token,
alternatives,
}
})
.collect()
}
fn split_search_query_tokens(query: &str) -> Vec<String> {
let mut tokens = Vec::new();
let mut current = String::new();
for ch in query.chars() {
if ch.is_whitespace() || is_search_separator(ch) {
if !current.is_empty() {
tokens.push(std::mem::take(&mut current));
}
} else {
current.push(ch);
}
}
if !current.is_empty() {
tokens.push(current);
}
tokens
}
fn is_search_separator(ch: char) -> bool {
matches!(
ch,
',' | ';' | ':' | '，' | '；' | '：' | '。' | '、' | '(' | ')' | '[' | ']' | '{' | '}'
)
}
fn cjk_token_bigrams(token: &str) -> Vec<String> {
let chars = token.chars().collect::<Vec<_>>();
if chars.len() < 3 || !chars.iter().any(|ch| is_cjk_char(*ch)) {
return Vec::new();
}
chars
.windows(2)
.filter_map(|window| {
if window.iter().any(|ch| is_cjk_stop_char(*ch)) {
return None;
}
Some(window.iter().collect::<String>())
})
.collect()
}
fn is_cjk_char(ch: char) -> bool {
matches!(
ch as u32,
0x3400..=0x4DBF | 0x4E00..=0x9FFF | 0xF900..=0xFAFF
)
}
fn is_cjk_stop_char(ch: char) -> bool {
matches!(
ch,
'的' | '了' | '和' | '与' | '及' | '在' | '是' | '为' | '对' | '中'
)
}
fn push_unique(values: &mut Vec<String>, value: String) {
if !value.is_empty() && !values.iter().any(|existing| existing == &value) {
values.push(value);
}
}
fn ocr_search_snippet(body: &str, query: &str) -> String {
let normalized_body = body.replace('\n', " ");
let normalized_query = query.trim();
@@ -3958,26 +4203,37 @@ fn ocr_search_snippet(body: &str, query: &str) -> String {
}
}
fn fuzzy_search_match(haystack: &str, query: &str) -> bool {
if query.is_empty() {
return false;
}
if haystack.contains(query) {
return true;
}
let mut query_chars = query.chars().filter(|ch| !ch.is_whitespace());
let Some(mut wanted) = query_chars.next() else {
return false;
};
for ch in haystack.chars().filter(|ch| !ch.is_whitespace()) {
if ch == wanted {
match query_chars.next() {
Some(next) => wanted = next,
None => return true,
}
fn ocr_search_snippet_for_terms(body: &str, query: &str, terms: &[String]) -> String {
let normalized_body = body.replace('\n', " ");
let normalized_query = query.trim();
if !normalized_query.is_empty() {
let lower = normalized_body.to_lowercase();
let lower_query = normalized_query.to_lowercase();
if let Some(byte_index) = lower.find(&lower_query) {
return snippet_from_byte_index(&normalized_body, byte_index, 180);
}
}
false
let lower = normalized_body.to_lowercase();
for term in terms {
let term = term.trim().to_lowercase();
if term.is_empty() {
continue;
}
if let Some(byte_index) = lower.find(&term) {
return snippet_from_byte_index(&normalized_body, byte_index, 180);
}
}
ocr_search_snippet(body, query)
}
fn snippet_from_byte_index(body: &str, byte_index: usize, len: usize) -> String {
let start = body[..byte_index]
.char_indices()
.rev()
.nth(40)
.map(|(idx, _)| idx)
.unwrap_or(0);
body[start..].chars().take(len).collect()
}
fn fuzzy_search_start_byte(haystack: &str, query: &str) -> Option<usize> {
@@ -4012,6 +4268,10 @@ fn fuzzy_search_start_byte(haystack: &str, query: &str) -> Option<usize> {
None
}
fn token_search_match(haystack: &str, query: &str) -> bool {
score_evidence_text_match(haystack, query).is_some()
}
fn is_markdown_path(path: &Path) -> bool {
path.extension()
.and_then(|value| value.to_str())
@@ -4034,6 +4294,7 @@ fn resource_type_from_path(path: &Path) -> Option<&'static str> {
.unwrap_or_default()
.to_lowercase();
match extension.as_str() {
"png" | "jpg" | "jpeg" | "webp" | "bmp" | "gif" | "tif" | "tiff" | "svg" => Some("image"),
"pdf" => Some("pdf"),
"doc" | "docx" | "odt" | "ppt" | "pptx" | "odp" | "xls" | "xlsx" | "ods" => Some("office"),
_ => None,
@@ -4092,7 +4353,10 @@ mod tests {
.expect("write mindmap");
fs::create_dir_all(root.join("assets")).expect("create assets");
fs::write(root.join("assets").join("spec.pdf"), b"%PDF-1.4\n").expect("write pdf");
fs::write(root.join("assets").join("diagram.png"), b"png").expect("write image");
fs::write(root.join("office").join("report.xlsx"), b"office bytes").expect("write office");
write_local_index_settings(&root, &[String::from(".")], None, None, None, None)
.expect("settings");
let projection = query_local_search_index(
&root,
@@ -4111,41 +4375,55 @@ mod tests {
.iter()
.find(|item| item["documentId"].as_str() == Some("local-md:README.md"))
.expect("home result");
assert!(home["tags"]
.as_array()
.unwrap()
.iter()
.any(|tag| tag.as_str() == Some("alpha")));
assert!(home["backlinks"]
.as_array()
.unwrap()
.iter()
.any(|link| link.as_str() == Some("Daily")));
assert!(home["backlinks"]
.as_array()
.unwrap()
.iter()
.any(|link| link.as_str() == Some("docs/child.md")));
assert!(home["resourceRefs"]
.as_array()
.unwrap()
.iter()
.any(|reference| reference.as_str() == Some("assets/spec.pdf")));
assert!(home["resourceRefs"]
.as_array()
.unwrap()
.iter()
.any(|reference| reference.as_str() == Some("maps/idea.mindmap.json")));
assert!(home["resourceRefs"]
.as_array()
.unwrap()
.iter()
.any(|reference| reference.as_str() == Some("office/report.xlsx")));
assert!(home["publicPath"]
.as_str()
.is_some_and(|path| path.starts_with(
"/documents/local-md:README.md?sourceKind=local_folder&rootUri=file%3A%2F%2F"
)));
assert!(
home["tags"]
.as_array()
.unwrap()
.iter()
.any(|tag| tag.as_str() == Some("alpha"))
);
assert!(
home["backlinks"]
.as_array()
.unwrap()
.iter()
.any(|link| link.as_str() == Some("Daily"))
);
assert!(
home["backlinks"]
.as_array()
.unwrap()
.iter()
.any(|link| link.as_str() == Some("docs/child.md"))
);
assert!(
home["resourceRefs"]
.as_array()
.unwrap()
.iter()
.any(|reference| reference.as_str() == Some("assets/spec.pdf"))
);
assert!(
home["resourceRefs"]
.as_array()
.unwrap()
.iter()
.any(|reference| reference.as_str() == Some("maps/idea.mindmap.json"))
);
assert!(
home["resourceRefs"]
.as_array()
.unwrap()
.iter()
.any(|reference| reference.as_str() == Some("office/report.xlsx"))
);
assert!(
home["publicPath"]
.as_str()
.is_some_and(|path| path.starts_with(
"/documents/local-md:README.md?sourceKind=local_folder&rootUri=file%3A%2F%2F"
))
);
let evidence_db = root.join(".mnote").join("index").join("evidence.sqlite");
let connection = Connection::open(&evidence_db).expect("open evidence sqlite");
let markdown_edge_count: i64 = connection
@@ -4201,11 +4479,12 @@ mod tests {
Some("local-mdid:child-page")
);
assert!(root
.join(".mnote")
.join("index")
.join("search-index.json")
.exists());
assert!(
root.join(".mnote")
.join("index")
.join("search-index.json")
.exists()
);
let mindmap_projection = query_local_search_index(
&root,
&format!("file://{}", root.display()),
@@ -4218,19 +4497,19 @@ mod tests {
false,
)
.expect("mindmap projection");
assert!(mindmap_projection["results"]
.as_array()
.unwrap()
.iter()
.any(|item| item["resourceType"].as_str() == Some("mindmap")
&& item["path"].as_str() == Some("maps/idea.mindmap.json")
&& item["publicPath"]
.as_str()
.is_some_and(|path| path
assert!(
mindmap_projection["results"]
.as_array()
.unwrap()
.iter()
.any(|item| item["resourceType"].as_str() == Some("mindmap")
&& item["path"].as_str() == Some("maps/idea.mindmap.json")
&& item["publicPath"].as_str().is_some_and(|path| path
.starts_with("/?treeView=filetree&sourceKind=local_folder&rootUri="))
&& item["publicPath"]
.as_str()
.is_some_and(|path| !path.starts_with("/tree?"))));
&& item["publicPath"]
.as_str()
.is_some_and(|path| !path.starts_with("/tree?")))
);
let office_projection = query_local_search_index(
&root,
&format!("file://{}", root.display()),
@@ -4243,12 +4522,14 @@ mod tests {
false,
)
.expect("office projection");
assert!(office_projection["results"]
.as_array()
.unwrap()
.iter()
.any(|item| item["resourceType"].as_str() == Some("office")
&& item["path"].as_str() == Some("office/report.xlsx")));
assert!(
office_projection["results"]
.as_array()
.unwrap()
.iter()
.any(|item| item["resourceType"].as_str() == Some("office")
&& item["path"].as_str() == Some("office/report.xlsx"))
);
let pdf_projection = query_local_search_index(
&root,
&format!("file://{}", root.display()),
@@ -4261,12 +4542,34 @@ mod tests {
false,
)
.expect("pdf projection");
assert!(pdf_projection["results"]
.as_array()
.unwrap()
.iter()
.any(|item| item["resourceType"].as_str() == Some("pdf")
&& item["path"].as_str() == Some("assets/spec.pdf")));
assert!(
pdf_projection["results"]
.as_array()
.unwrap()
.iter()
.any(|item| item["resourceType"].as_str() == Some("pdf")
&& item["path"].as_str() == Some("assets/spec.pdf"))
);
let image_projection = query_local_search_index(
&root,
&format!("file://{}", root.display()),
"local-ws-test",
"diagram.png",
None,
10,
false,
false,
false,
)
.expect("image projection");
assert!(
image_projection["results"]
.as_array()
.unwrap()
.iter()
.any(|item| item["resourceType"].as_str() == Some("image")
&& item["path"].as_str() == Some("assets/diagram.png"))
);
let _ = fs::remove_dir_all(&root);
}
@@ -4891,10 +5194,12 @@ mod tests {
let index = read_local_search_index(&root)
.expect("read index")
.expect("index exists");
assert!(index
.documents
.iter()
.any(|document| document.path == "README.md"));
assert!(
index
.documents
.iter()
.any(|document| document.path == "README.md")
);
let child = index
.documents
.iter()
@@ -4922,14 +5227,18 @@ mod tests {
let index = read_local_search_index(&root)
.expect("read index")
.expect("index exists");
assert!(!index
.documents
.iter()
.any(|document| document.path == "docs/child.md"));
assert!(index
.documents
.iter()
.any(|document| document.path == "README.md"));
assert!(
!index
.documents
.iter()
.any(|document| document.path == "docs/child.md")
);
assert!(
index
.documents
.iter()
.any(|document| document.path == "README.md")
);
let removed_evidence = query_evidence_sqlite_results(&root, "ChangedToken", None, 10)
.expect("removed evidence")
.expect("sqlite exists");
@@ -4990,6 +5299,8 @@ mod tests {
.expect("serialize index"),
)
.expect("ocr index");
write_local_index_settings(&root, &[String::from(".")], None, None, None, None)
.expect("settings");
let without_ocr = query_local_search_index(
&root,
@@ -5053,14 +5364,18 @@ mod tests {
let index = read_local_search_index(&root)
.expect("read search index")
.expect("search index");
assert!(!index
.documents
.iter()
.any(|document| document.path == "docs/Page.ocr/photo.png.ocr.md"));
assert!(!index
.documents
.iter()
.any(|document| document.path == "docs/Page.ocr/photo.png-704905.ocr.md"));
assert!(
!index
.documents
.iter()
.any(|document| document.path == "docs/Page.ocr/photo.png.ocr.md")
);
assert!(
!index
.documents
.iter()
.any(|document| document.path == "docs/Page.ocr/photo.png-704905.ocr.md")
);
refresh_local_search_index_for_path(
&root,
&root_uri,
@@ -5071,10 +5386,12 @@ mod tests {
let refreshed_index = read_local_search_index(&root)
.expect("read refreshed search index")
.expect("refreshed search index");
assert!(!refreshed_index
.documents
.iter()
.any(|document| document.path == "docs/Page.ocr/photo.png-704905.ocr.md"));
assert!(
!refreshed_index
.documents
.iter()
.any(|document| document.path == "docs/Page.ocr/photo.png-704905.ocr.md")
);
let _ = fs::remove_dir_all(&root);
}
@@ -5175,15 +5492,21 @@ mod tests {
.expect("sqlite query")
.expect("sqlite exists");
assert!(results.len() >= 2);
assert!(results
.iter()
.any(|result| result.source.owner_document_path == "README.md"));
assert!(results
.iter()
.any(|result| result.source.owner_document_path == "docs/child.md"));
assert!(results
.iter()
.all(|result| result.source.schema == "mnote.evidence_locator.v1"));
assert!(
results
.iter()
.any(|result| result.source.owner_document_path == "README.md")
);
assert!(
results
.iter()
.any(|result| result.source.owner_document_path == "docs/child.md")
);
assert!(
results
.iter()
.all(|result| result.source.schema == "mnote.evidence_locator.v1")
);
let home_result = results
.iter()
.find(|result| result.source.owner_document_path == "README.md")
@@ -5265,6 +5588,55 @@ mod tests {
let _ = fs::remove_dir_all(&root);
}
#[test]
fn evidence_sqlite_fuzzy_uses_cjk_terms_without_overrequiring_suffix() {
let root = temp_root("mnote-evidence-sqlite-cjk-fuzzy");
let root_uri = format!("file://{}", root.display());
let workspace_id = "local-ws-evidence-cjk";
fs::write(
root.join("README.md"),
"# 有机合成\n羧酸需要被保护有许多原因，常见做法是形成稳定酯类后再脱保护。\n",
)
.expect("write home");
write_local_index_settings(&root, &[String::from(".")], None, None, None, None)
.expect("settings");
refresh_local_search_index(&root, &root_uri, workspace_id).expect("refresh");
let exact = query_evidence_sqlite_results_with_mode(&root, "羧酸 保护基", None, 10, true)
.expect("exact sqlite query")
.expect("sqlite exists");
assert!(exact.is_empty(), "精确短语不应伪造不存在的同串命中");
let fuzzy = query_evidence_sqlite_results_with_mode(&root, "羧酸 保护基", None, 10, false)
.expect("fuzzy sqlite query")
.expect("sqlite exists");
assert_eq!(fuzzy.len(), 1);
let info = fuzzy[0].match_info.as_ref().expect("match info");
assert_eq!(info.match_mode, "cjk_partial");
assert!(!info.strong);
assert!(info.matched_terms.iter().any(|term| term == "羧酸"));
assert!(info.matched_terms.iter().any(|term| term == "保护"));
assert!(info.missing_terms.iter().any(|term| term == "保护基"));
assert!(fuzzy[0].quote.contains("羧酸"));
let projection = query_local_search_index(
&root,
&root_uri,
workspace_id,
"羧酸 保护基",
None,
10,
false,
false,
false,
)
.expect("local search projection");
assert_eq!(projection["results"].as_array().map(Vec::len), Some(1));
let _ = fs::remove_dir_all(&root);
}
#[test]
fn evidence_sqlite_read_context_returns_anchor_block() {
let root = temp_root("mnote-evidence-sqlite-read-context");
@@ -5292,12 +5664,16 @@ mod tests {
.expect("sqlite exists");
assert_eq!(context.len(), 2);
assert!(context
.iter()
.all(|item| item.source.owner_document_path == "README.md"));
assert!(context
.iter()
.any(|item| item.quote.contains("ReadContextToken")));
assert!(
context
.iter()
.all(|item| item.source.owner_document_path == "README.md")
);
assert!(
context
.iter()
.any(|item| item.quote.contains("ReadContextToken"))
);
let _ = fs::remove_dir_all(&root);
}
@@ -5395,16 +5771,18 @@ JSON
Some("docs/Page.assets/spec.pdf"),
"页面内搜索打开资源时应只限制到当前资源，而不是它的 owner Markdown"
);
assert!(root
.join("docs")
.join("Page.ocr")
.join("spec.pdf.parse.md")
.exists());
assert!(root
.join("docs")
.join("Page.ocr")
.join("spec.pdf.source-map.json")
.exists());
assert!(
root.join("docs")
.join("Page.ocr")
.join("spec.pdf.parse.md")
.exists()
);
assert!(
root.join("docs")
.join("Page.ocr")
.join("spec.pdf.source-map.json")
.exists()
);
let _ = fs::remove_dir_all(&root);
}