913 lines
30 KiB
Rust
913 lines
30 KiB
Rust
|
use core_protocol::{
|
||
|
|
EvidenceBBox, EvidenceRange, ParsedResourceArtifact, ResourceSourceMap, SourceMapBlock,
|
||
|
|
SourceMapBlockKind, SourceMapPage, SourceMapSection, SourceMapTextItem,
|
||
|
|
PARSED_RESOURCE_ARTIFACT_SCHEMA, RESOURCE_SOURCE_MAP_SCHEMA,
|
||
|
|
};
|
||
|
|
use serde_json::{json, Value};
|
||
|
|
use std::env;
|
||
|
|
use std::fs;
|
||
|
|
use std::future::Future;
|
||
|
|
use std::path::{Path, PathBuf};
|
||
|
|
use std::pin::Pin;
|
||
|
|
use std::time::UNIX_EPOCH;
|
||
|
|
use std::{env::split_paths, process::Command as StdCommand};
|
||
|
|
use tokio::process::Command;
|
||
|
|
|
||
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||
|
|
pub enum ParseCapability {
|
||
|
|
Unsupported,
|
||
|
|
Supported,
|
||
|
|
Preferred,
|
||
|
|
}
|
||
|
|
|
||
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||
|
|
pub enum ParseProviderMode {
|
||
|
|
Auto,
|
||
|
|
NoOcr,
|
||
|
|
Ocr,
|
||
|
|
}
|
||
|
|
|
||
|
|
#[derive(Debug, Clone)]
|
||
|
|
pub struct ParseInput {
|
||
|
|
pub root_path: PathBuf,
|
||
|
|
pub root_uri: String,
|
||
|
|
pub owner_document_id: String,
|
||
|
|
pub owner_document_path: String,
|
||
|
|
pub source_root_relative_path: String,
|
||
|
|
pub mode: ParseProviderMode,
|
||
|
|
}
|
||
|
|
|
||
|
|
#[derive(Debug, Clone)]
|
||
|
|
pub struct ParseProviderOutput {
|
||
|
|
pub artifact: ParsedResourceArtifact,
|
||
|
|
pub source_map: ResourceSourceMap,
|
||
|
|
pub markdown: String,
|
||
|
|
}
|
||
|
|
|
||
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||
|
|
pub struct ParseError {
|
||
|
|
pub code: String,
|
||
|
|
pub message: String,
|
||
|
|
}
|
||
|
|
|
||
|
|
impl ParseError {
|
||
|
|
fn new(code: impl Into<String>, message: impl Into<String>) -> Self {
|
||
|
|
Self {
|
||
|
|
code: code.into(),
|
||
|
|
message: message.into(),
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
pub trait ParseProvider {
|
||
|
|
fn provider_id(&self) -> &'static str;
|
||
|
|
fn can_parse(&self, input: &ParseInput) -> ParseCapability;
|
||
|
|
fn parse<'a>(
|
||
|
|
&'a self,
|
||
|
|
input: ParseInput,
|
||
|
|
) -> Pin<Box<dyn Future<Output = Result<ParseProviderOutput, ParseError>> + Send + 'a>>;
|
||
|
|
}
|
||
|
|
|
||
|
|
#[derive(Debug, Default, Clone)]
|
||
|
|
pub struct MarkdownParserProvider;
|
||
|
|
|
||
|
|
impl ParseProvider for MarkdownParserProvider {
|
||
|
|
fn provider_id(&self) -> &'static str {
|
||
|
|
"markdown"
|
||
|
|
}
|
||
|
|
|
||
|
|
fn can_parse(&self, input: &ParseInput) -> ParseCapability {
|
||
|
|
if is_markdown_path(&input.source_root_relative_path) {
|
||
|
|
ParseCapability::Preferred
|
||
|
|
} else {
|
||
|
|
ParseCapability::Unsupported
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
fn parse<'a>(
|
||
|
|
&'a self,
|
||
|
|
input: ParseInput,
|
||
|
|
) -> Pin<Box<dyn Future<Output = Result<ParseProviderOutput, ParseError>> + Send + 'a>> {
|
||
|
|
Box::pin(async move { parse_markdown_input(input) })
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
#[derive(Debug, Default, Clone)]
|
||
|
|
pub struct LiteParseProvider;
|
||
|
|
|
||
|
|
impl ParseProvider for LiteParseProvider {
|
||
|
|
fn provider_id(&self) -> &'static str {
|
||
|
|
"liteparse"
|
||
|
|
}
|
||
|
|
|
||
|
|
fn can_parse(&self, input: &ParseInput) -> ParseCapability {
|
||
|
|
if is_liteparse_path(&input.source_root_relative_path) {
|
||
|
|
match input.mode {
|
||
|
|
ParseProviderMode::Ocr => ParseCapability::Unsupported,
|
||
|
|
ParseProviderMode::Auto | ParseProviderMode::NoOcr => ParseCapability::Preferred,
|
||
|
|
}
|
||
|
|
} else {
|
||
|
|
ParseCapability::Unsupported
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
fn parse<'a>(
|
||
|
|
&'a self,
|
||
|
|
input: ParseInput,
|
||
|
|
) -> Pin<Box<dyn Future<Output = Result<ParseProviderOutput, ParseError>> + Send + 'a>> {
|
||
|
|
Box::pin(async move { parse_liteparse_input(input).await })
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
pub fn default_parse_provider_id(input: &ParseInput) -> &'static str {
|
||
|
|
if is_markdown_path(&input.source_root_relative_path) {
|
||
|
|
"markdown"
|
||
|
|
} else if is_liteparse_path(&input.source_root_relative_path)
|
||
|
|
&& !matches!(input.mode, ParseProviderMode::Ocr)
|
||
|
|
{
|
||
|
|
"liteparse"
|
||
|
|
} else {
|
||
|
|
"mineru"
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
pub fn select_parse_provider_id(
|
||
|
|
input: &ParseInput,
|
||
|
|
native_text_confidence: Option<f64>,
|
||
|
|
) -> &'static str {
|
||
|
|
if is_markdown_path(&input.source_root_relative_path) {
|
||
|
|
return "markdown";
|
||
|
|
}
|
||
|
|
if is_liteparse_path(&input.source_root_relative_path) {
|
||
|
|
if matches!(input.mode, ParseProviderMode::Ocr) {
|
||
|
|
return "mineru";
|
||
|
|
}
|
||
|
|
if native_text_confidence.is_some_and(|confidence| confidence < 0.2) {
|
||
|
|
return "mineru";
|
||
|
|
}
|
||
|
|
return "liteparse";
|
||
|
|
}
|
||
|
|
"mineru"
|
||
|
|
}
|
||
|
|
|
||
|
|
async fn parse_liteparse_input(input: ParseInput) -> Result<ParseProviderOutput, ParseError> {
|
||
|
|
let source_path = input.root_path.join(&input.source_root_relative_path);
|
||
|
|
let metadata = read_liteparse_source_metadata(&source_path)?;
|
||
|
|
let bytes = read_liteparse_source_bytes(&source_path)?;
|
||
|
|
let output = Command::new(liteparse_bin())
|
||
|
|
.arg("parse")
|
||
|
|
.arg("--format")
|
||
|
|
.arg("json")
|
||
|
|
.arg("--no-ocr")
|
||
|
|
.arg("-q")
|
||
|
|
.arg(&source_path)
|
||
|
|
.output()
|
||
|
|
.await
|
||
|
|
.map_err(|error| {
|
||
|
|
ParseError::new(
|
||
|
|
"liteparse_command_failed",
|
||
|
|
format!("LiteParse 命令执行失败: {error}"),
|
||
|
|
)
|
||
|
|
})?;
|
||
|
|
parse_liteparse_command_output(input, metadata, bytes, output)
|
||
|
|
}
|
||
|
|
|
||
|
|
pub(crate) fn parse_liteparse_input_blocking(
|
||
|
|
input: ParseInput,
|
||
|
|
) -> Result<ParseProviderOutput, ParseError> {
|
||
|
|
let source_path = input.root_path.join(&input.source_root_relative_path);
|
||
|
|
let metadata = read_liteparse_source_metadata(&source_path)?;
|
||
|
|
let bytes = read_liteparse_source_bytes(&source_path)?;
|
||
|
|
let output = StdCommand::new(liteparse_bin())
|
||
|
|
.arg("parse")
|
||
|
|
.arg("--format")
|
||
|
|
.arg("json")
|
||
|
|
.arg("--no-ocr")
|
||
|
|
.arg("-q")
|
||
|
|
.arg(&source_path)
|
||
|
|
.output()
|
||
|
|
.map_err(|error| {
|
||
|
|
ParseError::new(
|
||
|
|
"liteparse_command_failed",
|
||
|
|
format!("LiteParse 命令执行失败: {error}"),
|
||
|
|
)
|
||
|
|
})?;
|
||
|
|
parse_liteparse_command_output(input, metadata, bytes, output)
|
||
|
|
}
|
||
|
|
|
||
|
|
pub(crate) fn liteparse_runtime_available() -> bool {
|
||
|
|
if let Ok(value) = env::var("MNOTE_LITEPARSE_BIN") {
|
||
|
|
let trimmed = value.trim();
|
||
|
|
if trimmed.is_empty() {
|
||
|
|
return false;
|
||
|
|
}
|
||
|
|
let path = Path::new(trimmed);
|
||
|
|
return path.components().count() > 1 && path.exists() || command_exists(trimmed);
|
||
|
|
}
|
||
|
|
command_exists("lit") || command_exists("liteparse")
|
||
|
|
}
|
||
|
|
|
||
|
|
fn read_liteparse_source_metadata(source_path: &Path) -> Result<fs::Metadata, ParseError> {
|
||
|
|
fs::metadata(source_path).map_err(|error| {
|
||
|
|
ParseError::new(
|
||
|
|
"liteparse_parse_stat_failed",
|
||
|
|
format!(
|
||
|
|
"无法读取 LiteParse 证据源状态 {}: {error}",
|
||
|
|
source_path.display()
|
||
|
|
),
|
||
|
|
)
|
||
|
|
})
|
||
|
|
}
|
||
|
|
|
||
|
|
fn read_liteparse_source_bytes(source_path: &Path) -> Result<Vec<u8>, ParseError> {
|
||
|
|
fs::read(source_path).map_err(|error| {
|
||
|
|
ParseError::new(
|
||
|
|
"liteparse_parse_read_failed",
|
||
|
|
format!(
|
||
|
|
"无法读取 LiteParse 证据源 {}: {error}",
|
||
|
|
source_path.display()
|
||
|
|
),
|
||
|
|
)
|
||
|
|
})
|
||
|
|
}
|
||
|
|
|
||
|
|
fn parse_liteparse_command_output(
|
||
|
|
input: ParseInput,
|
||
|
|
metadata: fs::Metadata,
|
||
|
|
bytes: Vec<u8>,
|
||
|
|
output: std::process::Output,
|
||
|
|
) -> Result<ParseProviderOutput, ParseError> {
|
||
|
|
let updated_at_ms = metadata
|
||
|
|
.modified()
|
||
|
|
.ok()
|
||
|
|
.and_then(|value| value.duration_since(UNIX_EPOCH).ok())
|
||
|
|
.map(|value| value.as_millis() as u64)
|
||
|
|
.unwrap_or_default();
|
||
|
|
if !output.status.success() {
|
||
|
|
let stderr = String::from_utf8_lossy(&output.stderr);
|
||
|
|
return Err(ParseError::new(
|
||
|
|
"liteparse_command_failed",
|
||
|
|
format!("LiteParse 解析失败: {}", stderr.trim()),
|
||
|
|
));
|
||
|
|
}
|
||
|
|
let stdout = String::from_utf8(output.stdout).map_err(|error| {
|
||
|
|
ParseError::new(
|
||
|
|
"liteparse_output_utf8_invalid",
|
||
|
|
format!("LiteParse 输出不是 UTF-8: {error}"),
|
||
|
|
)
|
||
|
|
})?;
|
||
|
|
let value = serde_json::from_str::<Value>(&stdout).map_err(|error| {
|
||
|
|
ParseError::new(
|
||
|
|
"liteparse_output_json_invalid",
|
||
|
|
format!("LiteParse JSON 输出无效: {error}"),
|
||
|
|
)
|
||
|
|
})?;
|
||
|
|
let source_hash = binary_source_hash(&bytes, metadata.len(), updated_at_ms);
|
||
|
|
let markdown = liteparse_markdown(&value);
|
||
|
|
let source_map = liteparse_source_map(&input, &value, &source_hash);
|
||
|
|
let artifact = ParsedResourceArtifact {
|
||
|
|
schema: PARSED_RESOURCE_ARTIFACT_SCHEMA.into(),
|
||
|
|
provider: "liteparse".into(),
|
||
|
|
model_version: liteparse_version(&value),
|
||
|
|
owner_document_id: input.owner_document_id,
|
||
|
|
owner_document_path: input.owner_document_path,
|
||
|
|
source_root_relative_path: input.source_root_relative_path.clone(),
|
||
|
|
source_hash,
|
||
|
|
artifact_root_relative_path: format!("{}.parse.md", input.source_root_relative_path),
|
||
|
|
source_map_root_relative_path: format!(
|
||
|
|
"{}.source-map.json",
|
||
|
|
input.source_root_relative_path
|
||
|
|
),
|
||
|
|
updated_at_ms,
|
||
|
|
};
|
||
|
|
Ok(ParseProviderOutput {
|
||
|
|
artifact,
|
||
|
|
source_map,
|
||
|
|
markdown,
|
||
|
|
})
|
||
|
|
}
|
||
|
|
|
||
|
|
fn parse_markdown_input(input: ParseInput) -> Result<ParseProviderOutput, ParseError> {
|
||
|
|
let source_path = input.root_path.join(&input.source_root_relative_path);
|
||
|
|
let markdown = fs::read_to_string(&source_path).map_err(|error| {
|
||
|
|
ParseError::new(
|
||
|
|
"markdown_parse_read_failed",
|
||
|
|
format!(
|
||
|
|
"无法读取 Markdown 证据源 {}: {error}",
|
||
|
|
source_path.display()
|
||
|
|
),
|
||
|
|
)
|
||
|
|
})?;
|
||
|
|
let metadata = fs::metadata(&source_path).map_err(|error| {
|
||
|
|
ParseError::new(
|
||
|
|
"markdown_parse_stat_failed",
|
||
|
|
format!(
|
||
|
|
"无法读取 Markdown 证据源状态 {}: {error}",
|
||
|
|
source_path.display()
|
||
|
|
),
|
||
|
|
)
|
||
|
|
})?;
|
||
|
|
let updated_at_ms = metadata
|
||
|
|
.modified()
|
||
|
|
.ok()
|
||
|
|
.and_then(|value| value.duration_since(UNIX_EPOCH).ok())
|
||
|
|
.map(|value| value.as_millis() as u64)
|
||
|
|
.unwrap_or_default();
|
||
|
|
let source_hash = source_hash(&markdown, metadata.len(), updated_at_ms);
|
||
|
|
let source_map = markdown_source_map(&input, &markdown, &source_hash);
|
||
|
|
let artifact = ParsedResourceArtifact {
|
||
|
|
schema: PARSED_RESOURCE_ARTIFACT_SCHEMA.into(),
|
||
|
|
provider: "markdown".into(),
|
||
|
|
model_version: None,
|
||
|
|
owner_document_id: input.owner_document_id,
|
||
|
|
owner_document_path: input.owner_document_path,
|
||
|
|
source_root_relative_path: input.source_root_relative_path.clone(),
|
||
|
|
source_hash,
|
||
|
|
artifact_root_relative_path: input.source_root_relative_path.clone(),
|
||
|
|
source_map_root_relative_path: format!(
|
||
|
|
"{}.source-map.json",
|
||
|
|
input.source_root_relative_path
|
||
|
|
),
|
||
|
|
updated_at_ms,
|
||
|
|
};
|
||
|
|
Ok(ParseProviderOutput {
|
||
|
|
artifact,
|
||
|
|
source_map,
|
||
|
|
markdown,
|
||
|
|
})
|
||
|
|
}
|
||
|
|
|
||
|
|
fn liteparse_bin() -> String {
|
||
|
|
env::var("MNOTE_LITEPARSE_BIN")
|
||
|
|
.ok()
|
||
|
|
.filter(|value| !value.trim().is_empty())
|
||
|
|
.unwrap_or_else(|| {
|
||
|
|
if command_exists("lit") {
|
||
|
|
"lit".into()
|
||
|
|
} else {
|
||
|
|
"liteparse".into()
|
||
|
|
}
|
||
|
|
})
|
||
|
|
}
|
||
|
|
|
||
|
|
fn command_exists(command: &str) -> bool {
|
||
|
|
let path = Path::new(command);
|
||
|
|
if path.components().count() > 1 {
|
||
|
|
return path.exists();
|
||
|
|
}
|
||
|
|
let Some(paths) = env::var_os("PATH") else {
|
||
|
|
return false;
|
||
|
|
};
|
||
|
|
split_paths(&paths).any(|base| base.join(command).exists())
|
||
|
|
}
|
||
|
|
|
||
|
|
fn liteparse_version(value: &Value) -> Option<String> {
|
||
|
|
value
|
||
|
|
.get("version")
|
||
|
|
.or_else(|| value.get("modelVersion"))
|
||
|
|
.or_else(|| value.get("model_version"))
|
||
|
|
.and_then(Value::as_str)
|
||
|
|
.map(str::trim)
|
||
|
|
.filter(|value| !value.is_empty())
|
||
|
|
.map(ToOwned::to_owned)
|
||
|
|
.or_else(|| Some("liteparse-v2".into()))
|
||
|
|
}
|
||
|
|
|
||
|
|
fn liteparse_markdown(value: &Value) -> String {
|
||
|
|
for key in ["markdown", "text", "content"] {
|
||
|
|
if let Some(text) = value.get(key).and_then(Value::as_str) {
|
||
|
|
let text = text.trim();
|
||
|
|
if !text.is_empty() {
|
||
|
|
return text.to_string();
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
let lines = liteparse_pages(value)
|
||
|
|
.into_iter()
|
||
|
|
.flat_map(|(_, page)| liteparse_blocks(&page))
|
||
|
|
.filter_map(|block| liteparse_item_text(&block))
|
||
|
|
.collect::<Vec<_>>();
|
||
|
|
lines.join("\n\n")
|
||
|
|
}
|
||
|
|
|
||
|
|
fn liteparse_source_map(input: &ParseInput, value: &Value, source_hash: &str) -> ResourceSourceMap {
|
||
|
|
let mut pages = Vec::new();
|
||
|
|
let mut sections = Vec::new();
|
||
|
|
let mut section_path = Vec::<String>::new();
|
||
|
|
let mut global_char = 0_u64;
|
||
|
|
for (page_index, page_value) in liteparse_pages(value) {
|
||
|
|
let page_number = liteparse_page_number(&page_value).unwrap_or(page_index);
|
||
|
|
let blocks = liteparse_blocks(&page_value);
|
||
|
|
let mut source_blocks = Vec::new();
|
||
|
|
let mut text_items = Vec::new();
|
||
|
|
for (block_index, block) in blocks.iter().enumerate() {
|
||
|
|
let Some(text) = liteparse_item_text(block) else {
|
||
|
|
continue;
|
||
|
|
};
|
||
|
|
let block_id = liteparse_item_id(block)
|
||
|
|
.unwrap_or_else(|| format!("p{page_number}_b{}", block_index + 1));
|
||
|
|
let text_item_id = format!("p{page_number}_t{}", block_index + 1);
|
||
|
|
let block_type = liteparse_block_kind(block);
|
||
|
|
let char_range = EvidenceRange {
|
||
|
|
start: global_char,
|
||
|
|
end: global_char + text.chars().count() as u64,
|
||
|
|
};
|
||
|
|
global_char = char_range.end + 1;
|
||
|
|
let bbox = liteparse_bbox(block);
|
||
|
|
if matches!(block_type, SourceMapBlockKind::Heading) {
|
||
|
|
let level = liteparse_heading_level(block).unwrap_or(1).max(1);
|
||
|
|
section_path.truncate(level.saturating_sub(1));
|
||
|
|
section_path.push(text.clone());
|
||
|
|
sections.push(SourceMapSection {
|
||
|
|
id: format!(
|
||
|
|
"sec_{}",
|
||
|
|
stable_segment(&format!(
|
||
|
|
"{}:{}:{}",
|
||
|
|
input.source_root_relative_path,
|
||
|
|
page_number,
|
||
|
|
section_path.join("/")
|
||
|
|
))
|
||
|
|
),
|
||
|
|
title: text.clone(),
|
||
|
|
path: section_path.clone(),
|
||
|
|
page_start: Some(page_number),
|
||
|
|
page_end: Some(page_number),
|
||
|
|
block_ids: vec![block_id.clone()],
|
||
|
|
});
|
||
|
|
} else if let Some(section) = sections.last_mut() {
|
||
|
|
section.page_end = Some(page_number);
|
||
|
|
if !section.block_ids.iter().any(|id| id == &block_id) {
|
||
|
|
section.block_ids.push(block_id.clone());
|
||
|
|
}
|
||
|
|
}
|
||
|
|
text_items.push(SourceMapTextItem {
|
||
|
|
id: text_item_id,
|
||
|
|
text: text.clone(),
|
||
|
|
bbox: bbox.clone(),
|
||
|
|
char_range: Some(char_range.clone()),
|
||
|
|
});
|
||
|
|
source_blocks.push(SourceMapBlock {
|
||
|
|
id: block_id,
|
||
|
|
block_type,
|
||
|
|
text,
|
||
|
|
bbox,
|
||
|
|
char_range: Some(char_range),
|
||
|
|
});
|
||
|
|
}
|
||
|
|
pages.push(SourceMapPage {
|
||
|
|
page: page_number,
|
||
|
|
width: liteparse_number(&page_value, &["width", "pageWidth"]),
|
||
|
|
height: liteparse_number(&page_value, &["height", "pageHeight"]),
|
||
|
|
text_items,
|
||
|
|
blocks: source_blocks,
|
||
|
|
});
|
||
|
|
}
|
||
|
|
ResourceSourceMap {
|
||
|
|
schema: RESOURCE_SOURCE_MAP_SCHEMA.into(),
|
||
|
|
provider: "liteparse".into(),
|
||
|
|
model_version: liteparse_version(value),
|
||
|
|
owner_document_path: input.owner_document_path.clone(),
|
||
|
|
source_root_relative_path: input.source_root_relative_path.clone(),
|
||
|
|
source_hash: source_hash.to_string(),
|
||
|
|
page_count: pages.iter().map(|page| page.page).max(),
|
||
|
|
pages,
|
||
|
|
sections,
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
fn liteparse_pages(value: &Value) -> Vec<(u32, Value)> {
|
||
|
|
if let Some(pages) = value.get("pages").and_then(Value::as_array) {
|
||
|
|
return pages
|
||
|
|
.iter()
|
||
|
|
.enumerate()
|
||
|
|
.map(|(index, page)| ((index + 1) as u32, page.clone()))
|
||
|
|
.collect();
|
||
|
|
}
|
||
|
|
vec![(1, json!({ "blocks": liteparse_blocks(value) }))]
|
||
|
|
}
|
||
|
|
|
||
|
|
fn liteparse_blocks(page: &Value) -> Vec<Value> {
|
||
|
|
for key in ["blocks", "items", "textItems", "text_items", "elements"] {
|
||
|
|
if let Some(items) = page.get(key).and_then(Value::as_array) {
|
||
|
|
return items.clone();
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if let Some(text) = liteparse_item_text(page) {
|
||
|
|
return vec![json!({ "text": text })];
|
||
|
|
}
|
||
|
|
Vec::new()
|
||
|
|
}
|
||
|
|
|
||
|
|
fn liteparse_item_text(value: &Value) -> Option<String> {
|
||
|
|
for key in ["text", "content", "markdown", "value"] {
|
||
|
|
if let Some(text) = value.get(key).and_then(Value::as_str) {
|
||
|
|
let trimmed = text.trim();
|
||
|
|
if !trimmed.is_empty() {
|
||
|
|
return Some(trimmed.to_string());
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
None
|
||
|
|
}
|
||
|
|
|
||
|
|
fn liteparse_item_id(value: &Value) -> Option<String> {
|
||
|
|
value
|
||
|
|
.get("id")
|
||
|
|
.or_else(|| value.get("blockId"))
|
||
|
|
.or_else(|| value.get("block_id"))
|
||
|
|
.and_then(Value::as_str)
|
||
|
|
.map(str::trim)
|
||
|
|
.filter(|value| !value.is_empty())
|
||
|
|
.map(ToOwned::to_owned)
|
||
|
|
}
|
||
|
|
|
||
|
|
fn liteparse_page_number(value: &Value) -> Option<u32> {
|
||
|
|
for key in ["page", "pageNumber", "page_number", "page_no", "pageNo"] {
|
||
|
|
if let Some(page) = value.get(key).and_then(Value::as_u64) {
|
||
|
|
return u32::try_from(page.max(1)).ok();
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if let Some(page_idx) = value.get("page_idx").and_then(Value::as_u64) {
|
||
|
|
return u32::try_from(page_idx + 1).ok();
|
||
|
|
}
|
||
|
|
None
|
||
|
|
}
|
||
|
|
|
||
|
|
fn liteparse_block_kind(value: &Value) -> SourceMapBlockKind {
|
||
|
|
match value
|
||
|
|
.get("type")
|
||
|
|
.or_else(|| value.get("blockType"))
|
||
|
|
.or_else(|| value.get("block_type"))
|
||
|
|
.and_then(Value::as_str)
|
||
|
|
.unwrap_or_default()
|
||
|
|
.to_ascii_lowercase()
|
||
|
|
.as_str()
|
||
|
|
{
|
||
|
|
"title" | "heading" | "header" => SourceMapBlockKind::Heading,
|
||
|
|
"table" => SourceMapBlockKind::Table,
|
||
|
|
"figure" => SourceMapBlockKind::Figure,
|
||
|
|
"image" => SourceMapBlockKind::Image,
|
||
|
|
"list" => SourceMapBlockKind::List,
|
||
|
|
"text" | "paragraph" => SourceMapBlockKind::Paragraph,
|
||
|
|
_ => SourceMapBlockKind::Text,
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
fn liteparse_heading_level(value: &Value) -> Option<usize> {
|
||
|
|
value
|
||
|
|
.get("level")
|
||
|
|
.or_else(|| value.pointer("/props/level"))
|
||
|
|
.and_then(Value::as_u64)
|
||
|
|
.map(|value| value as usize)
|
||
|
|
}
|
||
|
|
|
||
|
|
fn liteparse_bbox(value: &Value) -> Option<EvidenceBBox> {
|
||
|
|
if let Some(bbox) = value
|
||
|
|
.get("bbox")
|
||
|
|
.or_else(|| value.get("boundingBox"))
|
||
|
|
.or_else(|| value.get("bounding_box"))
|
||
|
|
.and_then(Value::as_array)
|
||
|
|
{
|
||
|
|
if bbox.len() != 4 {
|
||
|
|
return None;
|
||
|
|
}
|
||
|
|
return Some(EvidenceBBox {
|
||
|
|
x0: bbox[0].as_f64()?,
|
||
|
|
y0: bbox[1].as_f64()?,
|
||
|
|
x1: bbox[2].as_f64()?,
|
||
|
|
y1: bbox[3].as_f64()?,
|
||
|
|
});
|
||
|
|
}
|
||
|
|
let x = value.get("x").and_then(Value::as_f64)?;
|
||
|
|
let y = value.get("y").and_then(Value::as_f64)?;
|
||
|
|
let width = value
|
||
|
|
.get("width")
|
||
|
|
.or_else(|| value.get("w"))
|
||
|
|
.and_then(Value::as_f64)?;
|
||
|
|
let height = value
|
||
|
|
.get("height")
|
||
|
|
.or_else(|| value.get("h"))
|
||
|
|
.and_then(Value::as_f64)?;
|
||
|
|
Some(EvidenceBBox {
|
||
|
|
x0: x,
|
||
|
|
y0: y,
|
||
|
|
x1: x + width,
|
||
|
|
y1: y + height,
|
||
|
|
})
|
||
|
|
}
|
||
|
|
|
||
|
|
fn liteparse_number(value: &Value, keys: &[&str]) -> Option<f64> {
|
||
|
|
keys.iter()
|
||
|
|
.find_map(|key| value.get(*key).and_then(Value::as_f64))
|
||
|
|
}
|
||
|
|
|
||
|
|
fn markdown_source_map(input: &ParseInput, markdown: &str, source_hash: &str) -> ResourceSourceMap {
|
||
|
|
let mut section_path: Vec<String> = Vec::new();
|
||
|
|
let mut sections = Vec::new();
|
||
|
|
let mut blocks = Vec::new();
|
||
|
|
let mut char_start = 0_u64;
|
||
|
|
for (index, line) in markdown.lines().enumerate() {
|
||
|
|
let line_number = (index + 1) as u64;
|
||
|
|
let trimmed = line.trim();
|
||
|
|
if trimmed.is_empty() {
|
||
|
|
char_start += line.len() as u64 + 1;
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
let block_type = if let Some((level, title)) = markdown_heading(trimmed) {
|
||
|
|
section_path.truncate(level.saturating_sub(1));
|
||
|
|
section_path.push(title.clone());
|
||
|
|
let id = format!(
|
||
|
|
"sec_{}",
|
||
|
|
stable_segment(&format!(
|
||
|
|
"{}:{}",
|
||
|
|
input.source_root_relative_path,
|
||
|
|
section_path.join("/")
|
||
|
|
))
|
||
|
|
);
|
||
|
|
sections.push(SourceMapSection {
|
||
|
|
id,
|
||
|
|
title,
|
||
|
|
path: section_path.clone(),
|
||
|
|
page_start: Some(line_number as u32),
|
||
|
|
page_end: Some(line_number as u32),
|
||
|
|
block_ids: vec![format!("line{line_number}")],
|
||
|
|
});
|
||
|
|
SourceMapBlockKind::Heading
|
||
|
|
} else {
|
||
|
|
SourceMapBlockKind::Paragraph
|
||
|
|
};
|
||
|
|
blocks.push(SourceMapBlock {
|
||
|
|
id: format!("line{line_number}"),
|
||
|
|
block_type,
|
||
|
|
text: trimmed.to_string(),
|
||
|
|
bbox: None,
|
||
|
|
char_range: Some(core_protocol::EvidenceRange {
|
||
|
|
start: char_start,
|
||
|
|
end: char_start + line.len() as u64,
|
||
|
|
}),
|
||
|
|
});
|
||
|
|
char_start += line.len() as u64 + 1;
|
||
|
|
}
|
||
|
|
ResourceSourceMap {
|
||
|
|
schema: RESOURCE_SOURCE_MAP_SCHEMA.into(),
|
||
|
|
provider: "markdown".into(),
|
||
|
|
model_version: None,
|
||
|
|
owner_document_path: input.owner_document_path.clone(),
|
||
|
|
source_root_relative_path: input.source_root_relative_path.clone(),
|
||
|
|
source_hash: source_hash.to_string(),
|
||
|
|
page_count: Some(1),
|
||
|
|
pages: vec![SourceMapPage {
|
||
|
|
page: 1,
|
||
|
|
width: None,
|
||
|
|
height: None,
|
||
|
|
text_items: Vec::new(),
|
||
|
|
blocks,
|
||
|
|
}],
|
||
|
|
sections,
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
fn markdown_heading(line: &str) -> Option<(usize, String)> {
|
||
|
|
let marker_count = line.chars().take_while(|value| *value == '#').count();
|
||
|
|
if marker_count == 0 || marker_count > 6 {
|
||
|
|
return None;
|
||
|
|
}
|
||
|
|
let rest = line.get(marker_count..)?.trim();
|
||
|
|
if rest.is_empty() {
|
||
|
|
return None;
|
||
|
|
}
|
||
|
|
Some((marker_count, rest.trim_matches('#').trim().to_string()))
|
||
|
|
}
|
||
|
|
|
||
|
|
fn is_markdown_path(path: &str) -> bool {
|
||
|
|
Path::new(path)
|
||
|
|
.extension()
|
||
|
|
.and_then(|value| value.to_str())
|
||
|
|
.is_some_and(|ext| ext.eq_ignore_ascii_case("md") || ext.eq_ignore_ascii_case("markdown"))
|
||
|
|
}
|
||
|
|
|
||
|
|
fn is_liteparse_path(path: &str) -> bool {
|
||
|
|
Path::new(path)
|
||
|
|
.extension()
|
||
|
|
.and_then(|value| value.to_str())
|
||
|
|
.is_some_and(|ext| {
|
||
|
|
matches!(
|
||
|
|
ext.to_ascii_lowercase().as_str(),
|
||
|
|
"pdf" | "doc" | "docx" | "odt" | "ppt" | "pptx" | "odp" | "xls" | "xlsx" | "ods"
|
||
|
|
)
|
||
|
|
})
|
||
|
|
}
|
||
|
|
|
||
|
|
fn source_hash(content: &str, size: u64, updated_at_ms: u64) -> String {
|
||
|
|
format!(
|
||
|
|
"fnv1a64:{:016x}:size:{size}:mtime:{updated_at_ms}",
|
||
|
|
fnv1a64(content.as_bytes())
|
||
|
|
)
|
||
|
|
}
|
||
|
|
|
||
|
|
fn binary_source_hash(content: &[u8], size: u64, updated_at_ms: u64) -> String {
|
||
|
|
format!(
|
||
|
|
"fnv1a64:{:016x}:size:{size}:mtime:{updated_at_ms}",
|
||
|
|
fnv1a64(content)
|
||
|
|
)
|
||
|
|
}
|
||
|
|
|
||
|
|
fn fnv1a64(bytes: &[u8]) -> u64 {
|
||
|
|
let mut hash = 0xcbf29ce484222325_u64;
|
||
|
|
for byte in bytes {
|
||
|
|
hash ^= u64::from(*byte);
|
||
|
|
hash = hash.wrapping_mul(0x100000001b3);
|
||
|
|
}
|
||
|
|
hash
|
||
|
|
}
|
||
|
|
|
||
|
|
fn stable_segment(value: &str) -> String {
|
||
|
|
format!("{:016x}", fnv1a64(value.as_bytes()))
|
||
|
|
}
|
||
|
|
|
||
|
|
#[cfg(test)]
|
||
|
|
mod tests {
|
||
|
|
use super::*;
|
||
|
|
use std::sync::{Mutex, OnceLock};
|
||
|
|
|
||
|
|
fn env_lock() -> &'static Mutex<()> {
|
||
|
|
static LOCK: OnceLock<Mutex<()>> = OnceLock::new();
|
||
|
|
LOCK.get_or_init(|| Mutex::new(()))
|
||
|
|
}
|
||
|
|
|
||
|
|
fn temp_root(name: &str) -> PathBuf {
|
||
|
|
let root = std::env::temp_dir().join(format!("{name}-{}", std::process::id()));
|
||
|
|
let _ = fs::remove_dir_all(&root);
|
||
|
|
fs::create_dir_all(root.join("docs")).expect("create root");
|
||
|
|
root
|
||
|
|
}
|
||
|
|
|
||
|
|
#[tokio::test]
|
||
|
|
async fn markdown_parser_provider_returns_artifact_and_source_map() {
|
||
|
|
let root = temp_root("mnote-markdown-parse-provider");
|
||
|
|
fs::write(
|
||
|
|
root.join("docs").join("Page.md"),
|
||
|
|
"# 合同\n正文\n## 解除条件\n提前三十日通知\n",
|
||
|
|
)
|
||
|
|
.expect("write markdown");
|
||
|
|
let input = ParseInput {
|
||
|
|
root_path: root.clone(),
|
||
|
|
root_uri: format!("file://{}", root.display()),
|
||
|
|
owner_document_id: "local-md:docs~2FPage.md".into(),
|
||
|
|
owner_document_path: "docs/Page.md".into(),
|
||
|
|
source_root_relative_path: "docs/Page.md".into(),
|
||
|
|
mode: ParseProviderMode::Auto,
|
||
|
|
};
|
||
|
|
let provider = MarkdownParserProvider;
|
||
|
|
assert_eq!(provider.can_parse(&input), ParseCapability::Preferred);
|
||
|
|
let output = provider.parse(input).await.expect("parse markdown");
|
||
|
|
|
||
|
|
assert_eq!(output.artifact.schema, PARSED_RESOURCE_ARTIFACT_SCHEMA);
|
||
|
|
assert_eq!(output.artifact.provider, "markdown");
|
||
|
|
assert!(output.artifact.source_hash.starts_with("fnv1a64:"));
|
||
|
|
assert_eq!(output.source_map.schema, RESOURCE_SOURCE_MAP_SCHEMA);
|
||
|
|
assert_eq!(output.source_map.sections.len(), 2);
|
||
|
|
assert!(output
|
||
|
|
.source_map
|
||
|
|
.sections
|
||
|
|
.iter()
|
||
|
|
.any(|section| section.path == vec!["合同".to_string(), "解除条件".to_string()]));
|
||
|
|
assert!(output.source_map.pages[0]
|
||
|
|
.blocks
|
||
|
|
.iter()
|
||
|
|
.any(|block| block.id == "line4" && block.text == "提前三十日通知"));
|
||
|
|
|
||
|
|
let _ = fs::remove_dir_all(&root);
|
||
|
|
}
|
||
|
|
|
||
|
|
#[test]
|
||
|
|
fn default_provider_routes_text_pdf_to_liteparse_no_ocr() {
|
||
|
|
let input = ParseInput {
|
||
|
|
root_path: PathBuf::from("/workspace"),
|
||
|
|
root_uri: "file:///workspace".into(),
|
||
|
|
owner_document_id: "local-md:Page.md".into(),
|
||
|
|
owner_document_path: "Page.md".into(),
|
||
|
|
source_root_relative_path: "Page.assets/spec.pdf".into(),
|
||
|
|
mode: ParseProviderMode::NoOcr,
|
||
|
|
};
|
||
|
|
let liteparse = LiteParseProvider;
|
||
|
|
assert_eq!(default_parse_provider_id(&input), "liteparse");
|
||
|
|
assert_eq!(liteparse.can_parse(&input), ParseCapability::Preferred);
|
||
|
|
}
|
||
|
|
|
||
|
|
#[tokio::test]
|
||
|
|
#[cfg(unix)]
|
||
|
|
async fn liteparse_provider_maps_json_output_to_artifact_and_source_map() {
|
||
|
|
use std::os::unix::fs::PermissionsExt;
|
||
|
|
|
||
|
|
let _guard = env_lock().lock().expect("env lock");
|
||
|
|
let root = temp_root("mnote-liteparse-provider");
|
||
|
|
fs::create_dir_all(root.join("docs").join("Page.assets")).expect("assets");
|
||
|
|
fs::write(
|
||
|
|
root.join("docs").join("Page.assets").join("spec.pdf"),
|
||
|
|
b"%PDF-1.4",
|
||
|
|
)
|
||
|
|
.expect("pdf");
|
||
|
|
let lit = root.join("fake-lit");
|
||
|
|
fs::write(
|
||
|
|
&lit,
|
||
|
|
r#"#!/usr/bin/env bash
|
||
|
|
cat <<'JSON'
|
||
|
|
{
|
||
|
|
"version": "liteparse-test",
|
||
|
|
"pages": [{
|
||
|
|
"page": 3,
|
||
|
|
"width": 612,
|
||
|
|
"height": 792,
|
||
|
|
"blocks": [
|
||
|
|
{"id": "b1", "type": "heading", "level": 1, "text": "解除条件", "bbox": [72, 120, 220, 150]},
|
||
|
|
{"id": "b2", "type": "text", "text": "提前三十日通知", "x": 72, "y": 160, "width": 228, "height": 28}
|
||
|
|
]
|
||
|
|
}]
|
||
|
|
}
|
||
|
|
JSON
|
||
|
|
"#,
|
||
|
|
)
|
||
|
|
.expect("fake lit");
|
||
|
|
let mut permissions = fs::metadata(&lit).expect("fake lit metadata").permissions();
|
||
|
|
permissions.set_mode(0o755);
|
||
|
|
fs::set_permissions(&lit, permissions).expect("chmod fake lit");
|
||
|
|
let old_bin = env::var("MNOTE_LITEPARSE_BIN").ok();
|
||
|
|
env::set_var("MNOTE_LITEPARSE_BIN", &lit);
|
||
|
|
|
||
|
|
let input = ParseInput {
|
||
|
|
root_path: root.clone(),
|
||
|
|
root_uri: format!("file://{}", root.display()),
|
||
|
|
owner_document_id: "local-md:docs~2FPage.md".into(),
|
||
|
|
owner_document_path: "docs/Page.md".into(),
|
||
|
|
source_root_relative_path: "docs/Page.assets/spec.pdf".into(),
|
||
|
|
mode: ParseProviderMode::NoOcr,
|
||
|
|
};
|
||
|
|
let output = LiteParseProvider
|
||
|
|
.parse(input)
|
||
|
|
.await
|
||
|
|
.expect("parse liteparse");
|
||
|
|
|
||
|
|
if let Some(value) = old_bin {
|
||
|
|
env::set_var("MNOTE_LITEPARSE_BIN", value);
|
||
|
|
} else {
|
||
|
|
env::remove_var("MNOTE_LITEPARSE_BIN");
|
||
|
|
}
|
||
|
|
assert_eq!(output.artifact.provider, "liteparse");
|
||
|
|
assert_eq!(
|
||
|
|
output.artifact.model_version.as_deref(),
|
||
|
|
Some("liteparse-test")
|
||
|
|
);
|
||
|
|
assert!(output.artifact.source_hash.starts_with("fnv1a64:"));
|
||
|
|
assert_eq!(output.markdown, "解除条件\n\n提前三十日通知");
|
||
|
|
assert_eq!(output.source_map.provider, "liteparse");
|
||
|
|
assert_eq!(output.source_map.pages[0].page, 3);
|
||
|
|
assert_eq!(
|
||
|
|
output.source_map.pages[0].blocks[1]
|
||
|
|
.bbox
|
||
|
|
.as_ref()
|
||
|
|
.unwrap()
|
||
|
|
.x0,
|
||
|
|
72.0
|
||
|
|
);
|
||
|
|
assert_eq!(
|
||
|
|
output.source_map.sections[0].path,
|
||
|
|
vec!["解除条件".to_string()]
|
||
|
|
);
|
||
|
|
|
||
|
|
let _ = fs::remove_dir_all(&root);
|
||
|
|
}
|
||
|
|
|
||
|
|
#[test]
|
||
|
|
fn default_provider_routes_ocr_policy_to_mineru() {
|
||
|
|
let input = ParseInput {
|
||
|
|
root_path: PathBuf::from("/workspace"),
|
||
|
|
root_uri: "file:///workspace".into(),
|
||
|
|
owner_document_id: "local-md:Page.md".into(),
|
||
|
|
owner_document_path: "Page.md".into(),
|
||
|
|
source_root_relative_path: "Page.assets/spec.pdf".into(),
|
||
|
|
mode: ParseProviderMode::Ocr,
|
||
|
|
};
|
||
|
|
assert_eq!(default_parse_provider_id(&input), "mineru");
|
||
|
|
}
|
||
|
|
|
||
|
|
#[test]
|
||
|
|
fn provider_selection_routes_scanned_pdf_or_image_to_mineru() {
|
||
|
|
let scanned_pdf = ParseInput {
|
||
|
|
root_path: PathBuf::from("/workspace"),
|
||
|
|
root_uri: "file:///workspace".into(),
|
||
|
|
owner_document_id: "local-md:Page.md".into(),
|
||
|
|
owner_document_path: "Page.md".into(),
|
||
|
|
source_root_relative_path: "Page.assets/scan.pdf".into(),
|
||
|
|
mode: ParseProviderMode::Auto,
|
||
|
|
};
|
||
|
|
assert_eq!(select_parse_provider_id(&scanned_pdf, Some(0.05)), "mineru");
|
||
|
|
|
||
|
|
let image = ParseInput {
|
||
|
|
source_root_relative_path: "Page.assets/photo.png".into(),
|
||
|
|
..scanned_pdf
|
||
|
|
};
|
||
|
|
assert_eq!(select_parse_provider_id(&image, None), "mineru");
|
||
|
|
}
|
||
|
|
}
|