mnote/rust/crates/mnote-web/src/evidence_parse.rs

use core_protocol::{
    ParsedResourceArtifact, ResourceSourceMap, SourceMapBlock, SourceMapBlockKind, SourceMapPage,
    SourceMapSection, PARSED_RESOURCE_ARTIFACT_SCHEMA, RESOURCE_SOURCE_MAP_SCHEMA,
};
use std::fs;
use std::future::Future;
use std::path::{Path, PathBuf};
use std::pin::Pin;
use std::time::UNIX_EPOCH;

#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ParseCapability {
    Unsupported,
    Supported,
    Preferred,
}

#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ParseProviderMode {
    Auto,
    NoOcr,
    Ocr,
}

#[derive(Debug, Clone)]
pub struct ParseInput {
    pub root_path: PathBuf,
    pub root_uri: String,
    pub owner_document_id: String,
    pub owner_document_path: String,
    pub source_root_relative_path: String,
    pub mode: ParseProviderMode,
}

#[derive(Debug, Clone)]
pub struct ParseProviderOutput {
    pub artifact: ParsedResourceArtifact,
    pub source_map: ResourceSourceMap,
    pub markdown: String,
}

#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ParseError {
    pub code: String,
    pub message: String,
}

impl ParseError {
    fn new(code: impl Into<String>, message: impl Into<String>) -> Self {
        Self {
            code: code.into(),
            message: message.into(),
        }
    }
}

pub trait ParseProvider {
    fn provider_id(&self) -> &'static str;
    fn can_parse(&self, input: &ParseInput) -> ParseCapability;
    fn parse<'a>(
        &'a self,
        input: ParseInput,
    ) -> Pin<Box<dyn Future<Output = Result<ParseProviderOutput, ParseError>> + Send + 'a>>;
}

#[derive(Debug, Default, Clone)]
pub struct MarkdownParserProvider;

impl ParseProvider for MarkdownParserProvider {
    fn provider_id(&self) -> &'static str {
        "markdown"
    }

    fn can_parse(&self, input: &ParseInput) -> ParseCapability {
        if is_markdown_path(&input.source_root_relative_path) {
            ParseCapability::Preferred
        } else {
            ParseCapability::Unsupported
        }
    }

    fn parse<'a>(
        &'a self,
        input: ParseInput,
    ) -> Pin<Box<dyn Future<Output = Result<ParseProviderOutput, ParseError>> + Send + 'a>> {
        Box::pin(async move { parse_markdown_input(input) })
    }
}

#[derive(Debug, Default, Clone)]
pub struct LiteParseProvider;

impl ParseProvider for LiteParseProvider {
    fn provider_id(&self) -> &'static str {
        "liteparse"
    }

    fn can_parse(&self, input: &ParseInput) -> ParseCapability {
        let _ = input;
        ParseCapability::Unsupported
    }

    fn parse<'a>(
        &'a self,
        input: ParseInput,
    ) -> Pin<Box<dyn Future<Output = Result<ParseProviderOutput, ParseError>> + Send + 'a>> {
        Box::pin(async move { parse_liteparse_input(input).await })
    }
}

pub fn default_parse_provider_id(input: &ParseInput) -> &'static str {
    if is_markdown_path(&input.source_root_relative_path) {
        "markdown"
    } else {
        "lightrag"
    }
}

pub fn select_parse_provider_id(
    input: &ParseInput,
    native_text_confidence: Option<f64>,
) -> &'static str {
    if is_markdown_path(&input.source_root_relative_path) {
        return "markdown";
    }
    let _ = native_text_confidence;
    "lightrag"
}

async fn parse_liteparse_input(input: ParseInput) -> Result<ParseProviderOutput, ParseError> {
    let _ = input;
    Err(ParseError::new(
        "liteparse_retired",
        "LiteParse 已退役；PDF、Office、图片 OCR 与资料索引统一由 LightRAG provider 处理",
    ))
}

fn parse_markdown_input(input: ParseInput) -> Result<ParseProviderOutput, ParseError> {
    let source_path = input.root_path.join(&input.source_root_relative_path);
    let markdown = fs::read_to_string(&source_path).map_err(|error| {
        ParseError::new(
            "markdown_parse_read_failed",
            format!(
                "无法读取 Markdown 证据源 {}: {error}",
                source_path.display()
            ),
        )
    })?;
    let metadata = fs::metadata(&source_path).map_err(|error| {
        ParseError::new(
            "markdown_parse_stat_failed",
            format!(
                "无法读取 Markdown 证据源状态 {}: {error}",
                source_path.display()
            ),
        )
    })?;
    let updated_at_ms = metadata
        .modified()
        .ok()
        .and_then(|value| value.duration_since(UNIX_EPOCH).ok())
        .map(|value| value.as_millis() as u64)
        .unwrap_or_default();
    let source_hash = source_hash(&markdown, metadata.len(), updated_at_ms);
    let source_map = markdown_source_map(&input, &markdown, &source_hash);
    let artifact = ParsedResourceArtifact {
        schema: PARSED_RESOURCE_ARTIFACT_SCHEMA.into(),
        provider: "markdown".into(),
        model_version: None,
        owner_document_id: input.owner_document_id,
        owner_document_path: input.owner_document_path,
        source_root_relative_path: input.source_root_relative_path.clone(),
        source_hash,
        artifact_root_relative_path: input.source_root_relative_path.clone(),
        source_map_root_relative_path: format!(
            "{}.source-map.json",
            input.source_root_relative_path
        ),
        updated_at_ms,
    };
    Ok(ParseProviderOutput {
        artifact,
        source_map,
        markdown,
    })
}

fn markdown_source_map(input: &ParseInput, markdown: &str, source_hash: &str) -> ResourceSourceMap {
    let mut section_path: Vec<String> = Vec::new();
    let mut sections = Vec::new();
    let mut blocks = Vec::new();
    let mut char_start = 0_u64;
    for (index, line) in markdown.lines().enumerate() {
        let line_number = (index + 1) as u64;
        let trimmed = line.trim();
        if trimmed.is_empty() {
            char_start += line.len() as u64 + 1;
            continue;
        }
        let block_type = if let Some((level, title)) = markdown_heading(trimmed) {
            section_path.truncate(level.saturating_sub(1));
            section_path.push(title.clone());
            let id = format!(
                "sec_{}",
                stable_segment(&format!(
                    "{}:{}",
                    input.source_root_relative_path,
                    section_path.join("/")
                ))
            );
            sections.push(SourceMapSection {
                id,
                title,
                path: section_path.clone(),
                page_start: Some(line_number as u32),
                page_end: Some(line_number as u32),
                block_ids: vec![format!("line{line_number}")],
            });
            SourceMapBlockKind::Heading
        } else {
            SourceMapBlockKind::Paragraph
        };
        blocks.push(SourceMapBlock {
            id: format!("line{line_number}"),
            block_type,
            text: trimmed.to_string(),
            bbox: None,
            char_range: Some(core_protocol::EvidenceRange {
                start: char_start,
                end: char_start + line.len() as u64,
            }),
        });
        char_start += line.len() as u64 + 1;
    }
    ResourceSourceMap {
        schema: RESOURCE_SOURCE_MAP_SCHEMA.into(),
        provider: "markdown".into(),
        model_version: None,
        owner_document_path: input.owner_document_path.clone(),
        source_root_relative_path: input.source_root_relative_path.clone(),
        source_hash: source_hash.to_string(),
        page_count: Some(1),
        pages: vec![SourceMapPage {
            page: 1,
            width: None,
            height: None,
            text_items: Vec::new(),
            blocks,
        }],
        sections,
    }
}

fn markdown_heading(line: &str) -> Option<(usize, String)> {
    let marker_count = line.chars().take_while(|value| *value == '#').count();
    if marker_count == 0 || marker_count > 6 {
        return None;
    }
    let rest = line.get(marker_count..)?.trim();
    if rest.is_empty() {
        return None;
    }
    Some((marker_count, rest.trim_matches('#').trim().to_string()))
}

fn is_markdown_path(path: &str) -> bool {
    Path::new(path)
        .extension()
        .and_then(|value| value.to_str())
        .is_some_and(|ext| ext.eq_ignore_ascii_case("md") || ext.eq_ignore_ascii_case("markdown"))
}

fn source_hash(content: &str, size: u64, updated_at_ms: u64) -> String {
    format!(
        "fnv1a64:{:016x}:size:{size}:mtime:{updated_at_ms}",
        fnv1a64(content.as_bytes())
    )
}

fn fnv1a64(bytes: &[u8]) -> u64 {
    let mut hash = 0xcbf29ce484222325_u64;
    for byte in bytes {
        hash ^= u64::from(*byte);
        hash = hash.wrapping_mul(0x100000001b3);
    }
    hash
}

fn stable_segment(value: &str) -> String {
    format!("{:016x}", fnv1a64(value.as_bytes()))
}

#[cfg(test)]
mod tests {
    use super::*;

    fn temp_root(name: &str) -> PathBuf {
        let root = std::env::temp_dir().join(format!("{name}-{}", std::process::id()));
        let _ = fs::remove_dir_all(&root);
        fs::create_dir_all(root.join("docs")).expect("create root");
        root
    }

    #[tokio::test]
    async fn markdown_parser_provider_returns_artifact_and_source_map() {
        let root = temp_root("mnote-markdown-parse-provider");
        fs::write(
            root.join("docs").join("Page.md"),
            "# 合同\n正文\n## 解除条件\n提前三十日通知\n",
        )
        .expect("write markdown");
        let input = ParseInput {
            root_path: root.clone(),
            root_uri: format!("file://{}", root.display()),
            owner_document_id: "local-md:docs~2FPage.md".into(),
            owner_document_path: "docs/Page.md".into(),
            source_root_relative_path: "docs/Page.md".into(),
            mode: ParseProviderMode::Auto,
        };
        let provider = MarkdownParserProvider;
        assert_eq!(provider.can_parse(&input), ParseCapability::Preferred);
        let output = provider.parse(input).await.expect("parse markdown");

        assert_eq!(output.artifact.schema, PARSED_RESOURCE_ARTIFACT_SCHEMA);
        assert_eq!(output.artifact.provider, "markdown");
        assert!(output.artifact.source_hash.starts_with("fnv1a64:"));
        assert_eq!(output.source_map.schema, RESOURCE_SOURCE_MAP_SCHEMA);
        assert_eq!(output.source_map.sections.len(), 2);
        assert!(output
            .source_map
            .sections
            .iter()
            .any(|section| section.path == vec!["合同".to_string(), "解除条件".to_string()]));
        assert!(output.source_map.pages[0]
            .blocks
            .iter()
            .any(|block| block.id == "line4" && block.text == "提前三十日通知"));

        let _ = fs::remove_dir_all(&root);
    }

    #[test]
    fn default_provider_routes_pdf_to_lightrag_after_liteparse_retirement() {
        let input = ParseInput {
            root_path: PathBuf::from("/workspace"),
            root_uri: "file:///workspace".into(),
            owner_document_id: "local-md:Page.md".into(),
            owner_document_path: "Page.md".into(),
            source_root_relative_path: "Page.assets/spec.pdf".into(),
            mode: ParseProviderMode::NoOcr,
        };
        let liteparse = LiteParseProvider;
        assert_eq!(default_parse_provider_id(&input), "lightrag");
        assert_eq!(liteparse.can_parse(&input), ParseCapability::Unsupported);
    }

    #[tokio::test]
    #[cfg(unix)]
    async fn liteparse_provider_returns_retired_error() {
        let root = temp_root("mnote-liteparse-provider");
        fs::create_dir_all(root.join("docs").join("Page.assets")).expect("assets");
        fs::write(
            root.join("docs").join("Page.assets").join("spec.pdf"),
            b"%PDF-1.4",
        )
        .expect("pdf");

        let input = ParseInput {
            root_path: root.clone(),
            root_uri: format!("file://{}", root.display()),
            owner_document_id: "local-md:docs~2FPage.md".into(),
            owner_document_path: "docs/Page.md".into(),
            source_root_relative_path: "docs/Page.assets/spec.pdf".into(),
            mode: ParseProviderMode::NoOcr,
        };
        let output = LiteParseProvider
            .parse(input)
            .await
            .expect_err("liteparse retired");

        assert_eq!(output.code, "liteparse_retired");

        let _ = fs::remove_dir_all(&root);
    }

    #[test]
    fn default_provider_routes_ocr_policy_to_lightrag() {
        let input = ParseInput {
            root_path: PathBuf::from("/workspace"),
            root_uri: "file:///workspace".into(),
            owner_document_id: "local-md:Page.md".into(),
            owner_document_path: "Page.md".into(),
            source_root_relative_path: "Page.assets/spec.pdf".into(),
            mode: ParseProviderMode::Ocr,
        };
        assert_eq!(default_parse_provider_id(&input), "lightrag");
    }

    #[test]
    fn provider_selection_routes_scanned_pdf_or_image_to_lightrag() {
        let scanned_pdf = ParseInput {
            root_path: PathBuf::from("/workspace"),
            root_uri: "file:///workspace".into(),
            owner_document_id: "local-md:Page.md".into(),
            owner_document_path: "Page.md".into(),
            source_root_relative_path: "Page.assets/scan.pdf".into(),
            mode: ParseProviderMode::Auto,
        };
        assert_eq!(
            select_parse_provider_id(&scanned_pdf, Some(0.05)),
            "lightrag"
        );

        let image = ParseInput {
            source_root_relative_path: "Page.assets/photo.png".into(),
            ..scanned_pdf
        };
        assert_eq!(select_parse_provider_id(&image, None), "lightrag");
    }
}