Improve local evidence search and AI capabilities

2026-06-05 23:00:53 +08:00
parent a1dcfc9f76
commit 4dbd9a978b
52 changed files with 6415 additions and 527 deletions
@@ -7,10 +7,10 @@ use axum::extract::{Extension, State};
 use axum::http::{HeaderMap, HeaderValue, StatusCode};
 use axum::Json;
 use core_protocol::{
-    EvidenceBBox, EvidenceLocator, EvidenceOpenAction, EvidenceOpenRequest, EvidenceReadRequest,
-    EvidenceResourceKind, EvidenceSearchMode, EvidenceSearchRequest, EvidenceSearchResponse,
-    EvidenceSearchResult, ResourceSourceMap, SourceMapBlock, SourceMapTextItem,
-    EVIDENCE_LOCATOR_SCHEMA,
+    EvidenceBBox, EvidenceLocator, EvidenceOpenAction, EvidenceOpenRequest, EvidenceRange,
+    EvidenceReadRequest, EvidenceResourceKind, EvidenceSearchMode, EvidenceSearchRequest,
+    EvidenceSearchResponse, EvidenceSearchResult, ResourceSourceMap, SourceMapBlock,
+    SourceMapTextItem, EVIDENCE_LOCATOR_SCHEMA,
 };
 use serde_json::Map;
 use serde_json::{json, Value};
@@ -47,18 +47,31 @@ pub(crate) async fn search_payload(
    )
    .map_err(|error| error.with_context(context))?;
    let page_id = body.scope.target_document_id.as_deref();
+    let evidence_owner_filter = if body.scope.include_resources || body.scope.include_ocr {
+        None
+    } else {
+        page_id
+    };
    let query = body.query.trim().to_string();
    if matches!(body.mode, EvidenceSearchMode::Graph) {
-        if let Some(results) = local_search_index::query_evidence_graph_results(
-            &root_path, &query, page_id, body.top_k,
+        if let Some(mut results) = local_search_index::query_evidence_graph_results(
+            &root_path,
+            &query,
+            evidence_owner_filter,
+            body.top_k,
        )? {
+            enrich_citation_links(&mut results);
            return Ok(json!(EvidenceSearchResponse { ok: true, results }));
        }
    }
-    if let Some(mut results) =
-        local_search_index::query_evidence_sqlite_results(&root_path, &query, page_id, body.top_k)?
-    {
+    if let Some(mut results) = local_search_index::query_evidence_sqlite_results(
+        &root_path,
+        &query,
+        evidence_owner_filter,
+        body.top_k,
+    )? {
        enrich_sqlite_evidence_results(&mut results, &root_path, &query);
+        enrich_citation_links(&mut results);
        if !results.is_empty() || !query.is_empty() {
            let response = EvidenceSearchResponse { ok: true, results };
            return Ok(json!(response));
@@ -77,9 +90,13 @@ pub(crate) async fn search_payload(
    )?;
    let response = EvidenceSearchResponse {
        ok: true,
-        results: evidence_results_from_local_search(
-            &search, &root_path, &root_uri, body.mode, &query,
-        ),
+        results: {
+            let mut results = evidence_results_from_local_search(
+                &search, &root_path, &root_uri, body.mode, &query,
+            );
+            enrich_citation_links(&mut results);
+            results
+        },
    };
    Ok(json!(response))
 }
@@ -127,6 +144,149 @@ fn enrich_sqlite_evidence_results(
    }
 }

+fn enrich_citation_links(results: &mut [EvidenceSearchResult]) {
+    for result in results {
+        let citation_url = citation_url_for_locator(&result.source);
+        result.source.open_action.url = citation_url.clone();
+        result.citation_label = Some(citation_label_for_locator(&result.source));
+        result.citation_markdown = Some(citation_markdown_for_locator(&result.source));
+        result.citation_url = Some(citation_url);
+    }
+}
+
+fn citation_markdown_for_locator(locator: &EvidenceLocator) -> String {
+    let label = citation_label_for_locator(locator);
+    let url = citation_url_for_locator(locator);
+    format!(
+        "[{}]({})",
+        markdown_link_label_escape(&label),
+        url.replace(')', "%29")
+    )
+}
+
+fn citation_label_for_locator(locator: &EvidenceLocator) -> String {
+    let source_path = locator
+        .resource_path
+        .as_deref()
+        .filter(|value| !value.trim().is_empty())
+        .unwrap_or(locator.owner_document_path.as_str());
+    let name = source_path
+        .rsplit('/')
+        .find(|part| !part.trim().is_empty())
+        .unwrap_or(source_path)
+        .trim();
+    let mut parts = vec![if name.is_empty() {
+        "证据".to_string()
+    } else {
+        name.to_string()
+    }];
+    if let Some(page) = locator.page {
+        parts.push(format!("p.{page}"));
+    }
+    if let Some(section) = locator
+        .section_path
+        .last()
+        .filter(|value| !value.trim().is_empty())
+    {
+        parts.push(section.to_string());
+    }
+    parts.join(" · ")
+}
+
+fn citation_url_for_locator(locator: &EvidenceLocator) -> String {
+    let owner_document_id = locator.owner_document_id.trim();
+    let mut url = if owner_document_id.is_empty() {
+        let existing = locator.open_action.url.trim();
+        if existing.is_empty() {
+            "/".to_string()
+        } else {
+            existing.to_string()
+        }
+    } else {
+        format!("/documents/{owner_document_id}")
+    };
+    append_query_param(&mut url, "sourceKind", "local_folder");
+    append_query_param(&mut url, "rootUri", locator.root_uri.trim());
+    if let Some(resource_path) = locator
+        .resource_path
+        .as_deref()
+        .filter(|value| !value.trim().is_empty())
+    {
+        append_query_param(
+            &mut url,
+            "resourceTab",
+            &format!(
+                "resource:file:{}:{}",
+                locator.root_uri.trim(),
+                resource_path
+            ),
+        );
+        append_query_param(&mut url, "resourcePath", resource_path);
+    }
+    if let Some(page) = locator.page {
+        append_query_param(&mut url, "page", &page.to_string());
+    }
+    if let Some(bbox) = &locator.bbox {
+        append_query_param(
+            &mut url,
+            "bbox",
+            &format!("{},{},{},{}", bbox.x0, bbox.y0, bbox.x1, bbox.y1),
+        );
+    }
+    if let Some(block_id) = locator.block_id.as_deref() {
+        append_query_param(&mut url, "blockId", block_id);
+    }
+    if let Some(source_map_path) = locator.source_map_path.as_deref() {
+        append_query_param(&mut url, "sourceMapPath", source_map_path);
+    }
+    if let Some(line_range) = &locator.line_range {
+        append_query_param(
+            &mut url,
+            "lineRange",
+            &format!("{}-{}", line_range.start, line_range.end),
+        );
+    }
+    if let Some(char_range) = &locator.char_range {
+        append_query_param(
+            &mut url,
+            "charRange",
+            &format!("{}-{}", char_range.start, char_range.end),
+        );
+    }
+    url
+}
+
+fn append_query_param(url: &mut String, key: &str, value: &str) {
+    let value = value.trim();
+    if value.is_empty() {
+        return;
+    }
+    let separator = if url.contains('?') { '&' } else { '?' };
+    url.push(separator);
+    url.push_str(&encode_query_component(key));
+    url.push('=');
+    url.push_str(&encode_query_component(value));
+}
+
+fn encode_query_component(value: &str) -> String {
+    let mut encoded = String::with_capacity(value.len());
+    for byte in value.as_bytes() {
+        if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~') {
+            encoded.push(*byte as char);
+        } else {
+            encoded.push_str(&format!("%{byte:02X}"));
+        }
+    }
+    encoded
+}
+
+fn markdown_link_label_escape(value: &str) -> String {
+    value
+        .replace('\\', "\\\\")
+        .replace('[', "\\[")
+        .replace(']', "\\]")
+}
+
 pub async fn read(
    State(state): State<AppState>,
    Extension(context): Extension<RequestContext>,
@@ -154,12 +314,13 @@ pub(crate) async fn read_payload(
        state, context, &root_uri,
    )
    .map_err(|error| error.with_context(context))?;
-    if let Some(results) = read_source_map_context(
+    if let Some(mut results) = read_source_map_context(
        &root_path,
        &body.locator,
        body.context.before_blocks,
        body.context.after_blocks,
    ) {
+        enrich_citation_links(&mut results);
        let quote = results
            .iter()
            .find(|item| locator_matches(&item.source, &body.locator))
@@ -176,17 +337,21 @@ pub(crate) async fn read_payload(
            "locator": body.locator,
            "quote": quote,
            "sectionPath": section_path,
+            "citationUrl": citation_url_for_locator(&body.locator),
+            "citationLabel": citation_label_for_locator(&body.locator),
+            "citationMarkdown": citation_markdown_for_locator(&body.locator),
            "contextBlocks": results,
        });
        return Ok(response);
    }
-    if let Some(results) = local_search_index::read_evidence_sqlite_context(
+    if let Some(mut results) = local_search_index::read_evidence_sqlite_context(
        &root_path,
        &body.locator,
        body.context.before_blocks,
        body.context.after_blocks,
    )? {
        if !results.is_empty() {
+            enrich_citation_links(&mut results);
            let quote = results
                .iter()
                .find(|item| locator_matches(&item.source, &body.locator))
@@ -197,6 +362,9 @@ pub(crate) async fn read_payload(
                "ok": true,
                "locator": body.locator,
                "quote": quote,
+                "citationUrl": citation_url_for_locator(&body.locator),
+                "citationLabel": citation_label_for_locator(&body.locator),
+                "citationMarkdown": citation_markdown_for_locator(&body.locator),
                "contextBlocks": results,
            });
            return Ok(response);
@@ -222,13 +390,14 @@ pub(crate) async fn read_payload(
        false,
        true,
    )?;
-    let results = evidence_results_from_local_search(
+    let mut results = evidence_results_from_local_search(
        &search,
        &root_path,
        &root_uri,
        EvidenceSearchMode::Tree,
        &query,
    );
+    enrich_citation_links(&mut results);
    let quote = results
        .first()
        .map(|item| item.quote.clone())
@@ -237,6 +406,9 @@ pub(crate) async fn read_payload(
        "ok": true,
        "locator": body.locator,
        "quote": quote,
+        "citationUrl": citation_url_for_locator(&body.locator),
+        "citationLabel": citation_label_for_locator(&body.locator),
+        "citationMarkdown": citation_markdown_for_locator(&body.locator),
        "contextBlocks": results,
    });
    Ok(response)
@@ -365,6 +537,9 @@ fn source_map_block_result(
        quote: block.text.clone(),
        score,
        source,
+        citation_url: None,
+        citation_label: None,
+        citation_markdown: None,
    }
 }

@@ -394,7 +569,9 @@ pub(crate) async fn open_payload(
    context: &RequestContext,
    body: EvidenceOpenRequest,
 ) -> Result<Value, WebError> {
-    let locator = body.locator;
+    let mut locator = body.locator;
+    let citation_url = citation_url_for_locator(&locator);
+    locator.open_action.url = citation_url.clone();
    let root_uri = locator.root_uri.trim().to_string();
    if root_uri.is_empty() {
        return Err(
@@ -410,6 +587,9 @@ pub(crate) async fn open_payload(
        "ok": true,
        "locator": locator.clone(),
        "openAction": locator.open_action,
+        "citationUrl": citation_url,
+        "citationLabel": citation_label_for_locator(&locator),
+        "citationMarkdown": citation_markdown_for_locator(&locator),
    });
    Ok(response)
 }
@@ -499,19 +679,29 @@ pub(crate) fn evidence_results_from_local_search(
                })
                .or_else(|| source_map_hit.as_ref().and_then(|hit| hit.bbox.clone()));
            let block_id = result
-                .get("ocrEvidence")
-                .and_then(|_| source_map_hit.as_ref().and_then(|hit| hit.block_id.clone()))
-                .or_else(|| result.get("id").and_then(Value::as_str).map(str::to_string))
+                .get("blockId")
+                .and_then(Value::as_str)
+                .map(str::to_string)
+                .or_else(|| {
+                    result
+                        .get("ocrEvidence")
+                        .and_then(|_| source_map_hit.as_ref().and_then(|hit| hit.block_id.clone()))
+                })
                .or_else(|| source_map_hit.as_ref().and_then(|hit| hit.block_id.clone()));
+            let line_range = result.get("lineRange").and_then(evidence_range_from_value);
            let char_range = source_map_hit
                .as_ref()
-                .and_then(|hit| hit.char_range.clone());
+                .and_then(|hit| hit.char_range.clone())
+                .or_else(|| result.get("charRange").and_then(evidence_range_from_value));
            let open_action_params = evidence_open_action_params(
                &result,
                query,
                &mode,
                page,
                bbox.clone(),
+                block_id.clone(),
+                line_range.clone(),
+                char_range.clone(),
                source_map_path.clone(),
            );
            let locator = EvidenceLocator {
@@ -534,7 +724,7 @@ pub(crate) fn evidence_results_from_local_search(
                            .collect::<Vec<_>>()
                    })
                    .unwrap_or_default(),
-                line_range: None,
+                line_range,
                char_range,
                block_id,
                source_map_path: source_map_path.clone(),
@@ -557,6 +747,9 @@ pub(crate) fn evidence_results_from_local_search(
                    .to_string(),
                score: result.get("score").and_then(Value::as_f64).unwrap_or(0.0),
                source: locator,
+                citation_url: None,
+                citation_label: None,
+                citation_markdown: None,
            }
        })
        .collect()
@@ -568,6 +761,9 @@ fn evidence_open_action_params(
    mode: &EvidenceSearchMode,
    page: Option<u32>,
    bbox: Option<EvidenceBBox>,
+    block_id: Option<String>,
+    line_range: Option<EvidenceRange>,
+    char_range: Option<EvidenceRange>,
    source_map_path: Option<String>,
 ) -> Map<String, Value> {
    let mut params = Map::new();
@@ -586,12 +782,35 @@ fn evidence_open_action_params(
    if let Some(bbox) = bbox {
        params.insert("bbox".into(), json!(bbox));
    }
+    if let Some(block_id) = block_id {
+        params.insert("blockId".into(), json!(block_id));
+    }
+    if let Some(line_range) = line_range {
+        params.insert("lineRange".into(), json!(line_range));
+    }
+    if let Some(char_range) = char_range {
+        params.insert("charRange".into(), json!(char_range));
+    }
    if let Some(source_map_path) = source_map_path {
        params.insert("sourceMapPath".into(), json!(source_map_path));
    }
    params
 }

+fn evidence_range_from_value(value: &Value) -> Option<EvidenceRange> {
+    if let Some(map) = value.as_object() {
+        let start = map.get("start")?.as_u64()?;
+        let end = map.get("end")?.as_u64()?;
+        return Some(EvidenceRange { start, end });
+    }
+    let text = value.as_str()?.trim();
+    let (start, end) = text.split_once('-')?;
+    Some(EvidenceRange {
+        start: start.trim().parse().ok()?,
+        end: end.trim().parse().ok()?,
+    })
+}
+
 fn infer_resource_kind(resource_type: &str, resource_path: Option<&str>) -> EvidenceResourceKind {
    let extension = resource_path
        .and_then(|path| Path::new(path).extension())
@@ -788,6 +1007,15 @@ mod tests {
        )
        .expect("readme");
        let root_uri = format!("file://{}", root.display());
+        local_search_index::write_local_index_settings(
+            &root,
+            &[String::from(".")],
+            None,
+            None,
+            None,
+            None,
+        )
+        .expect("settings");
        local_search_index::refresh_local_search_index(&root, &root_uri, "local-ws-evidence-route")
            .expect("refresh");
        fs::write(
@@ -838,6 +1066,7 @@ mod tests {
                            "scope": {
                                "workspaceId": "local-ws-evidence-route",
                                "rootUri": root_uri,
+                                "targetDocumentId": "local-md:missing.md",
                                "includeResources": true,
                                "includeOcr": true
                            },
@@ -925,13 +1154,14 @@ mod tests {
            }]
        });

-        let results = evidence_results_from_local_search(
+        let mut results = evidence_results_from_local_search(
            &payload,
            &root,
            "file:///workspace",
            EvidenceSearchMode::Hybrid,
            "OCR-only-token",
        );
+        enrich_citation_links(&mut results);

        fs::remove_dir_all(&root).ok();

@@ -954,6 +1184,16 @@ mod tests {
            locator.open_action.params["sourceMapPath"].as_str(),
            Some("docs/Page.ocr/photo.png.source-map.json")
        );
+        let citation_url = results[0].citation_url.as_deref().unwrap_or_default();
+        assert!(citation_url.starts_with("/documents/local-md:docs~2FPage.md?"));
+        assert!(citation_url.contains("resourceTab=resource%3Afile%3Afile%3A%2F%2F%2Fworkspace%3Adocs%2FPage.assets%2Fphoto.png"));
+        assert!(citation_url.contains("page=1"));
+        assert!(citation_url.contains("blockId=p1_b1"));
+        assert!(results[0]
+            .citation_markdown
+            .as_deref()
+            .unwrap_or_default()
+            .contains("photo.png"));
    }

    #[test]
@@ -1011,9 +1251,13 @@ mod tests {
            quote: "NeedleToken 原文定位".into(),
            score: 1.0,
            source: locator,
+            citation_url: None,
+            citation_label: None,
+            citation_markdown: None,
        }];

        enrich_sqlite_evidence_results(&mut results, &root, "NeedleToken");
+        enrich_citation_links(&mut results);
        fs::remove_dir_all(&root).ok();

        let locator = &results[0].source;
@@ -1029,6 +1273,16 @@ mod tests {
            locator.open_action.params["blockId"].as_str(),
            Some("p2_b7")
        );
+        assert!(results[0]
+            .citation_url
+            .as_deref()
+            .unwrap_or_default()
+            .contains("sourceMapPath=docs%2FPage.ocr%2Fspec.pdf.source-map.json"));
+        assert!(results[0]
+            .citation_markdown
+            .as_deref()
+            .unwrap_or_default()
+            .contains("spec.pdf"));
    }

    #[test]