use core_protocol::{ ParsedResourceArtifact, ResourceSourceMap, SourceMapBlock, SourceMapBlockKind, SourceMapPage, SourceMapSection, PARSED_RESOURCE_ARTIFACT_SCHEMA, RESOURCE_SOURCE_MAP_SCHEMA, }; use std::fs; use std::future::Future; use std::path::{Path, PathBuf}; use std::pin::Pin; use std::time::UNIX_EPOCH; #[derive(Debug, Clone, PartialEq, Eq)] pub enum ParseCapability { Unsupported, Supported, Preferred, } #[derive(Debug, Clone, PartialEq, Eq)] pub enum ParseProviderMode { Auto, NoOcr, Ocr, } #[derive(Debug, Clone)] pub struct ParseInput { pub root_path: PathBuf, pub root_uri: String, pub owner_document_id: String, pub owner_document_path: String, pub source_root_relative_path: String, pub mode: ParseProviderMode, } #[derive(Debug, Clone)] pub struct ParseProviderOutput { pub artifact: ParsedResourceArtifact, pub source_map: ResourceSourceMap, pub markdown: String, } #[derive(Debug, Clone, PartialEq, Eq)] pub struct ParseError { pub code: String, pub message: String, } impl ParseError { fn new(code: impl Into, message: impl Into) -> Self { Self { code: code.into(), message: message.into(), } } } pub trait ParseProvider { fn provider_id(&self) -> &'static str; fn can_parse(&self, input: &ParseInput) -> ParseCapability; fn parse<'a>( &'a self, input: ParseInput, ) -> Pin> + Send + 'a>>; } #[derive(Debug, Default, Clone)] pub struct MarkdownParserProvider; impl ParseProvider for MarkdownParserProvider { fn provider_id(&self) -> &'static str { "markdown" } fn can_parse(&self, input: &ParseInput) -> ParseCapability { if is_markdown_path(&input.source_root_relative_path) { ParseCapability::Preferred } else { ParseCapability::Unsupported } } fn parse<'a>( &'a self, input: ParseInput, ) -> Pin> + Send + 'a>> { Box::pin(async move { parse_markdown_input(input) }) } } #[derive(Debug, Default, Clone)] pub struct LiteParseProvider; impl ParseProvider for LiteParseProvider { fn provider_id(&self) -> &'static str { "liteparse" } fn can_parse(&self, input: &ParseInput) -> ParseCapability { let _ = input; ParseCapability::Unsupported } fn parse<'a>( &'a self, input: ParseInput, ) -> Pin> + Send + 'a>> { Box::pin(async move { parse_liteparse_input(input).await }) } } pub fn default_parse_provider_id(input: &ParseInput) -> &'static str { if is_markdown_path(&input.source_root_relative_path) { "markdown" } else { "lightrag" } } pub fn select_parse_provider_id( input: &ParseInput, native_text_confidence: Option, ) -> &'static str { if is_markdown_path(&input.source_root_relative_path) { return "markdown"; } let _ = native_text_confidence; "lightrag" } async fn parse_liteparse_input(input: ParseInput) -> Result { let _ = input; Err(ParseError::new( "liteparse_retired", "LiteParse 已退役;PDF、Office、图片 OCR 与资料索引统一由 LightRAG provider 处理", )) } fn parse_markdown_input(input: ParseInput) -> Result { let source_path = input.root_path.join(&input.source_root_relative_path); let markdown = fs::read_to_string(&source_path).map_err(|error| { ParseError::new( "markdown_parse_read_failed", format!( "无法读取 Markdown 证据源 {}: {error}", source_path.display() ), ) })?; let metadata = fs::metadata(&source_path).map_err(|error| { ParseError::new( "markdown_parse_stat_failed", format!( "无法读取 Markdown 证据源状态 {}: {error}", source_path.display() ), ) })?; let updated_at_ms = metadata .modified() .ok() .and_then(|value| value.duration_since(UNIX_EPOCH).ok()) .map(|value| value.as_millis() as u64) .unwrap_or_default(); let source_hash = source_hash(&markdown, metadata.len(), updated_at_ms); let source_map = markdown_source_map(&input, &markdown, &source_hash); let artifact = ParsedResourceArtifact { schema: PARSED_RESOURCE_ARTIFACT_SCHEMA.into(), provider: "markdown".into(), model_version: None, owner_document_id: input.owner_document_id, owner_document_path: input.owner_document_path, source_root_relative_path: input.source_root_relative_path.clone(), source_hash, artifact_root_relative_path: input.source_root_relative_path.clone(), source_map_root_relative_path: format!( "{}.source-map.json", input.source_root_relative_path ), updated_at_ms, }; Ok(ParseProviderOutput { artifact, source_map, markdown, }) } fn markdown_source_map(input: &ParseInput, markdown: &str, source_hash: &str) -> ResourceSourceMap { let mut section_path: Vec = Vec::new(); let mut sections = Vec::new(); let mut blocks = Vec::new(); let mut char_start = 0_u64; for (index, line) in markdown.lines().enumerate() { let line_number = (index + 1) as u64; let trimmed = line.trim(); if trimmed.is_empty() { char_start += line.len() as u64 + 1; continue; } let block_type = if let Some((level, title)) = markdown_heading(trimmed) { section_path.truncate(level.saturating_sub(1)); section_path.push(title.clone()); let id = format!( "sec_{}", stable_segment(&format!( "{}:{}", input.source_root_relative_path, section_path.join("/") )) ); sections.push(SourceMapSection { id, title, path: section_path.clone(), page_start: Some(line_number as u32), page_end: Some(line_number as u32), block_ids: vec![format!("line{line_number}")], }); SourceMapBlockKind::Heading } else { SourceMapBlockKind::Paragraph }; blocks.push(SourceMapBlock { id: format!("line{line_number}"), block_type, text: trimmed.to_string(), bbox: None, char_range: Some(core_protocol::EvidenceRange { start: char_start, end: char_start + line.len() as u64, }), }); char_start += line.len() as u64 + 1; } ResourceSourceMap { schema: RESOURCE_SOURCE_MAP_SCHEMA.into(), provider: "markdown".into(), model_version: None, owner_document_path: input.owner_document_path.clone(), source_root_relative_path: input.source_root_relative_path.clone(), source_hash: source_hash.to_string(), page_count: Some(1), pages: vec![SourceMapPage { page: 1, width: None, height: None, text_items: Vec::new(), blocks, }], sections, } } fn markdown_heading(line: &str) -> Option<(usize, String)> { let marker_count = line.chars().take_while(|value| *value == '#').count(); if marker_count == 0 || marker_count > 6 { return None; } let rest = line.get(marker_count..)?.trim(); if rest.is_empty() { return None; } Some((marker_count, rest.trim_matches('#').trim().to_string())) } fn is_markdown_path(path: &str) -> bool { Path::new(path) .extension() .and_then(|value| value.to_str()) .is_some_and(|ext| ext.eq_ignore_ascii_case("md") || ext.eq_ignore_ascii_case("markdown")) } fn source_hash(content: &str, size: u64, updated_at_ms: u64) -> String { format!( "fnv1a64:{:016x}:size:{size}:mtime:{updated_at_ms}", fnv1a64(content.as_bytes()) ) } fn fnv1a64(bytes: &[u8]) -> u64 { let mut hash = 0xcbf29ce484222325_u64; for byte in bytes { hash ^= u64::from(*byte); hash = hash.wrapping_mul(0x100000001b3); } hash } fn stable_segment(value: &str) -> String { format!("{:016x}", fnv1a64(value.as_bytes())) } #[cfg(test)] mod tests { use super::*; fn temp_root(name: &str) -> PathBuf { let root = std::env::temp_dir().join(format!("{name}-{}", std::process::id())); let _ = fs::remove_dir_all(&root); fs::create_dir_all(root.join("docs")).expect("create root"); root } #[tokio::test] async fn markdown_parser_provider_returns_artifact_and_source_map() { let root = temp_root("mnote-markdown-parse-provider"); fs::write( root.join("docs").join("Page.md"), "# 合同\n正文\n## 解除条件\n提前三十日通知\n", ) .expect("write markdown"); let input = ParseInput { root_path: root.clone(), root_uri: format!("file://{}", root.display()), owner_document_id: "local-md:docs~2FPage.md".into(), owner_document_path: "docs/Page.md".into(), source_root_relative_path: "docs/Page.md".into(), mode: ParseProviderMode::Auto, }; let provider = MarkdownParserProvider; assert_eq!(provider.can_parse(&input), ParseCapability::Preferred); let output = provider.parse(input).await.expect("parse markdown"); assert_eq!(output.artifact.schema, PARSED_RESOURCE_ARTIFACT_SCHEMA); assert_eq!(output.artifact.provider, "markdown"); assert!(output.artifact.source_hash.starts_with("fnv1a64:")); assert_eq!(output.source_map.schema, RESOURCE_SOURCE_MAP_SCHEMA); assert_eq!(output.source_map.sections.len(), 2); assert!(output .source_map .sections .iter() .any(|section| section.path == vec!["合同".to_string(), "解除条件".to_string()])); assert!(output.source_map.pages[0] .blocks .iter() .any(|block| block.id == "line4" && block.text == "提前三十日通知")); let _ = fs::remove_dir_all(&root); } #[test] fn default_provider_routes_pdf_to_lightrag_after_liteparse_retirement() { let input = ParseInput { root_path: PathBuf::from("/workspace"), root_uri: "file:///workspace".into(), owner_document_id: "local-md:Page.md".into(), owner_document_path: "Page.md".into(), source_root_relative_path: "Page.assets/spec.pdf".into(), mode: ParseProviderMode::NoOcr, }; let liteparse = LiteParseProvider; assert_eq!(default_parse_provider_id(&input), "lightrag"); assert_eq!(liteparse.can_parse(&input), ParseCapability::Unsupported); } #[tokio::test] #[cfg(unix)] async fn liteparse_provider_returns_retired_error() { let root = temp_root("mnote-liteparse-provider"); fs::create_dir_all(root.join("docs").join("Page.assets")).expect("assets"); fs::write( root.join("docs").join("Page.assets").join("spec.pdf"), b"%PDF-1.4", ) .expect("pdf"); let input = ParseInput { root_path: root.clone(), root_uri: format!("file://{}", root.display()), owner_document_id: "local-md:docs~2FPage.md".into(), owner_document_path: "docs/Page.md".into(), source_root_relative_path: "docs/Page.assets/spec.pdf".into(), mode: ParseProviderMode::NoOcr, }; let output = LiteParseProvider .parse(input) .await .expect_err("liteparse retired"); assert_eq!(output.code, "liteparse_retired"); let _ = fs::remove_dir_all(&root); } #[test] fn default_provider_routes_ocr_policy_to_lightrag() { let input = ParseInput { root_path: PathBuf::from("/workspace"), root_uri: "file:///workspace".into(), owner_document_id: "local-md:Page.md".into(), owner_document_path: "Page.md".into(), source_root_relative_path: "Page.assets/spec.pdf".into(), mode: ParseProviderMode::Ocr, }; assert_eq!(default_parse_provider_id(&input), "lightrag"); } #[test] fn provider_selection_routes_scanned_pdf_or_image_to_lightrag() { let scanned_pdf = ParseInput { root_path: PathBuf::from("/workspace"), root_uri: "file:///workspace".into(), owner_document_id: "local-md:Page.md".into(), owner_document_path: "Page.md".into(), source_root_relative_path: "Page.assets/scan.pdf".into(), mode: ParseProviderMode::Auto, }; assert_eq!( select_parse_provider_id(&scanned_pdf, Some(0.05)), "lightrag" ); let image = ParseInput { source_root_relative_path: "Page.assets/photo.png".into(), ..scanned_pdf }; assert_eq!(select_parse_provider_id(&image, None), "lightrag"); } }