lix/mnote
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve evidence search ranking and diagnostics

Browse Source
This commit is contained in:
lix-2026
2026-06-06 00:33:02 +08:00
parent 4dbd9a978b
commit 5157a5960c
6 changed files with 891 additions and 235 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rust/crates/mnote-web/src/routes/search.rs
+157 -56
View File
@@ -9,14 +9,14 @@ use crate::routes::query_support::{
use crate::routes::web_shell::load_sidebar_tree_html;
use crate::routes::{evidence, local_folder_source, local_search_index};
use crate::ssr::pages::search::SearchPage;
use axum::Json;
use axum::extract::{Extension, Query, State};
use axum::http::{HeaderMap, HeaderName, HeaderValue, StatusCode};
use axum::response::{Html, IntoResponse, Response};
use axum::Json;
use bridge_runtime::RuntimeQueryEnvelopeWire;
use core_protocol::{EvidenceSearchMode, EvidenceSearchResult};
use serde::Deserialize;
use serde_json::{json, Value};
use serde_json::{Value, json};
const HEADER_MNOTE_WEB_OWNER: &str = "x-mnote-web-owner";
const HEADER_QUERY_NAME: &str = "x-query-name";
@@ -239,6 +239,16 @@ pub async fn documents(
filters.exact.unwrap_or(false),
)?
.unwrap_or_default()
.into_iter()
.filter(|evidence| {
let path = evidence
.source
.resource_path
.as_deref()
.unwrap_or(evidence.source.owner_document_path.as_str());
local_search_index::local_index_relative_path_is_included(path, &user_settings)
})
.collect::<Vec<_>>()
} else {
Vec::new()
};
@@ -438,12 +448,6 @@ pub async fn update_local_index_settings(
&effective_workspace_id,
&root_path,
)?;
let refreshed = local_search_index::refresh_local_search_index_with_settings(
&root_path,
root_uri,
&effective_workspace_id,
&effective_settings,
)?;
let status = local_search_index::local_index_status_with_settings(
&root_path,
root_uri,
@@ -459,7 +463,6 @@ pub async fn update_local_index_settings(
Json(json!({
"ok": true,
"settings": settings,
"index": refreshed,
"result": status,
"meta": {
"owner": "mnote-web",
@@ -481,26 +484,72 @@ fn merge_local_search_with_evidence_results(
if direct_evidence_results.is_empty() {
return (result, evidence_results);
}
let mut result_items = result
let original_items = result
.get("results")
.and_then(Value::as_array)
.cloned()
.unwrap_or_default();
let mut seen_evidence_ids = evidence_results
.iter()
.map(|item| item.evidence_id.clone())
.collect::<std::collections::HashSet<_>>();
let original_evidence_results = std::mem::take(&mut evidence_results);
let mut result_items = Vec::new();
let mut merged_evidence_results = Vec::new();
let mut seen_result_ids = std::collections::HashSet::new();
let mut seen_evidence_ids = std::collections::HashSet::new();
let mut seen_paths = std::collections::HashSet::new();
for evidence in direct_evidence_results {
if result_items.len() >= limit || !seen_evidence_ids.insert(evidence.evidence_id.clone()) {
continue;
}
result_items.push(search_result_from_evidence(&evidence));
evidence_results.push(evidence);
let item = search_result_from_evidence(&evidence);
if let Some(id) = item.get("id").and_then(Value::as_str) {
seen_result_ids.insert(id.to_string());
}
if let Some(path) = search_result_dedupe_path(&item) {
seen_paths.insert(path);
}
result_items.push(item);
merged_evidence_results.push(evidence);
}
for (index, item) in original_items.into_iter().enumerate() {
if result_items.len() >= limit {
break;
}
let item_id = item
.get("id")
.and_then(Value::as_str)
.map(ToOwned::to_owned)
.unwrap_or_default();
if !item_id.is_empty() && !seen_result_ids.insert(item_id) {
continue;
}
if let Some(path) = search_result_dedupe_path(&item) {
if !seen_paths.insert(path) {
continue;
}
}
if let Some(evidence) = original_evidence_results.get(index).cloned() {
merged_evidence_results.push(evidence);
}
result_items.push(item);
}
if let Some(map) = result.as_object_mut() {
map.insert("results".into(), Value::Array(result_items));
}
(result, evidence_results)
(result, merged_evidence_results)
}
fn search_result_dedupe_path(item: &Value) -> Option<String> {
let source_kind = item
.get("sourceKind")
.and_then(Value::as_str)
.unwrap_or_default();
if source_kind != "local_folder" {
return None;
}
item.get("path")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
}
fn search_result_from_evidence(evidence: &EvidenceSearchResult) -> Value {
@@ -533,6 +582,7 @@ fn search_result_from_evidence(evidence: &EvidenceSearchResult) -> Value {
"rootUri": source.root_uri,
"snippet": evidence.quote,
"score": evidence.score,
"matchInfo": evidence.match_info,
"publicPath": source.open_action.url,
"evidence": evidence_value,
"source": {
@@ -890,12 +940,12 @@ fn stamp_search_headers(headers: &mut HeaderMap) {
#[cfg(test)]
mod tests {
use crate::app::{build_app, AppConfig, AppState};
use crate::app::{AppConfig, AppState, build_app};
use crate::routes::local_search_index;
use axum::body::{to_bytes, Body};
use axum::body::{Body, to_bytes};
use axum::http::{Request, StatusCode};
use control_plane::{DirectoryGrantInput, UpsertUserInput};
use serde_json::{json, Value};
use serde_json::{Value, json};
use std::fs;
use tower::util::ServiceExt;
@@ -1152,26 +1202,33 @@ mod tests {
Some("README.md")
);
assert!(!payload["evidence"].as_array().expect("evidence").is_empty());
assert!(home["tags"]
.as_array()
.unwrap()
.iter()
.any(|tag| tag.as_str() == Some("alpha")));
assert!(home["backlinks"]
.as_array()
.unwrap()
.iter()
.any(|link| link.as_str() == Some("Daily")));
assert!(home["resourceRefs"]
.as_array()
.unwrap()
.iter()
.any(|reference| reference.as_str() == Some("assets/spec.pdf")));
assert!(root
.join(".mnote")
.join("index")
.join("search-index.json")
.exists());
assert!(
home["tags"]
.as_array()
.unwrap()
.iter()
.any(|tag| tag.as_str() == Some("alpha"))
);
assert!(
home["backlinks"]
.as_array()
.unwrap()
.iter()
.any(|link| link.as_str() == Some("Daily"))
);
assert!(
home["resourceRefs"]
.as_array()
.unwrap()
.iter()
.any(|reference| reference.as_str() == Some("assets/spec.pdf"))
);
assert!(
root.join(".mnote")
.join("index")
.join("search-index.json")
.exists()
);
let evidence_db = root.join(".mnote").join("index").join("evidence.sqlite");
assert!(evidence_db.exists(), "evidence sqlite should be built");
let connection = rusqlite::Connection::open(&evidence_db).expect("open evidence sqlite");
@@ -1201,11 +1258,13 @@ mod tests {
locator["schema"].as_str(),
Some("mnote.evidence_locator.v1")
);
assert!(payload["recent"]
.as_array()
.unwrap()
.iter()
.any(|item| item["documentId"].as_str() == Some("local-md:README.md")));
assert!(
payload["recent"]
.as_array()
.unwrap()
.iter()
.any(|item| item["documentId"].as_str() == Some("local-md:README.md"))
);
let _ = fs::remove_dir_all(&root);
}
@@ -1409,10 +1468,32 @@ mod tests {
.await
.expect("create settings response");
assert_eq!(create_response.status(), StatusCode::OK);
let create_refresh_response = app
.clone()
.oneshot(
Request::builder()
.method("POST")
.uri("/api/search/local-index/refresh")
.header("content-type", "application/json")
.header("x-mnote-actor-id", "user_test")
.header("x-mnote-actor-type", "user")
.body(Body::from(
json!({
"workspaceId": "local-ws-settings-delete",
"rootUri": root_uri
})
.to_string(),
))
.expect("create refresh request"),
)
.await
.expect("create refresh response");
assert_eq!(create_refresh_response.status(), StatusCode::OK);
assert!(root.join(".mnote/index/search-index.json").exists());
assert!(root.join(".mnote/index/evidence.sqlite").exists());
let delete_response = app
.clone()
.oneshot(
Request::builder()
.method("PUT")
@@ -1440,18 +1521,36 @@ mod tests {
.await
.expect("delete body");
let delete_payload: Value = serde_json::from_slice(&delete_body).expect("delete json");
assert_eq!(
delete_payload["index"]["indexedPaths"]
.as_array()
.map(Vec::len),
Some(0)
);
assert_eq!(
delete_payload["result"]["settings"]["includePaths"]
.as_array()
.map(Vec::len),
Some(0)
);
assert!(root.join(".mnote/index/search-index.json").exists());
assert!(root.join(".mnote/index/evidence.sqlite").exists());
let delete_refresh_response = app
.clone()
.oneshot(
Request::builder()
.method("POST")
.uri("/api/search/local-index/refresh")
.header("content-type", "application/json")
.header("x-mnote-actor-id", "user_test")
.header("x-mnote-actor-type", "user")
.body(Body::from(
json!({
"workspaceId": "local-ws-settings-delete",
"rootUri": root_uri
})
.to_string(),
))
.expect("delete refresh request"),
)
.await
.expect("delete refresh response");
assert_eq!(delete_refresh_response.status(), StatusCode::OK);
assert!(!root.join(".mnote/index/search-index.json").exists());
assert!(!root.join(".mnote/index/evidence.sqlite").exists());
@@ -1508,11 +1607,13 @@ mod tests {
backlinks_payload["meta"]["queryName"].as_str(),
Some("search.local_index.backlinks")
);
assert!(backlinks_payload["result"]["backlinks"]
.as_array()
.unwrap()
.iter()
.any(|item| item["documentId"].as_str() == Some("local-md:README.md")));
assert!(
backlinks_payload["result"]["backlinks"]
.as_array()
.unwrap()
.iter()
.any(|item| item["documentId"].as_str() == Some("local-md:README.md"))
);
let tags_response = app()
.oneshot(