lix/mnote
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(rag): replace LiteParse flows with LightRAG provider

Browse Source
This commit is contained in:
lix-2026
2026-06-07 01:10:31 +08:00
parent f46c2fb5d0
commit 1f25364374
40 changed files with 7232 additions and 2350 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rust/crates/mnote-web/src/evidence_parse.rs
+26 -518
View File
@@ -1,17 +1,12 @@
use core_protocol::{
EvidenceBBox, EvidenceRange, ParsedResourceArtifact, ResourceSourceMap, SourceMapBlock,
SourceMapBlockKind, SourceMapPage, SourceMapSection, SourceMapTextItem,
PARSED_RESOURCE_ARTIFACT_SCHEMA, RESOURCE_SOURCE_MAP_SCHEMA,
ParsedResourceArtifact, ResourceSourceMap, SourceMapBlock, SourceMapBlockKind, SourceMapPage,
SourceMapSection, PARSED_RESOURCE_ARTIFACT_SCHEMA, RESOURCE_SOURCE_MAP_SCHEMA,
};
use serde_json::{json, Value};
use std::env;
use std::fs;
use std::future::Future;
use std::path::{Path, PathBuf};
use std::pin::Pin;
use std::time::UNIX_EPOCH;
use std::{env::split_paths, process::Command as StdCommand};
use tokio::process::Command;
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ParseCapability {
@@ -101,14 +96,8 @@ impl ParseProvider for LiteParseProvider {
}
fn can_parse(&self, input: &ParseInput) -> ParseCapability {
if is_liteparse_path(&input.source_root_relative_path) {
match input.mode {
ParseProviderMode::Ocr => ParseCapability::Unsupported,
ParseProviderMode::Auto | ParseProviderMode::NoOcr => ParseCapability::Preferred,
}
} else {
ParseCapability::Unsupported
}
let _ = input;
ParseCapability::Unsupported
}
fn parse<'a>(
@@ -122,12 +111,8 @@ impl ParseProvider for LiteParseProvider {
pub fn default_parse_provider_id(input: &ParseInput) -> &'static str {
if is_markdown_path(&input.source_root_relative_path) {
"markdown"
} else if is_liteparse_path(&input.source_root_relative_path)
&& !matches!(input.mode, ParseProviderMode::Ocr)
{
"liteparse"
} else {
"mineru"
"lightrag"
}
}
@@ -138,153 +123,16 @@ pub fn select_parse_provider_id(
if is_markdown_path(&input.source_root_relative_path) {
return "markdown";
}
if is_liteparse_path(&input.source_root_relative_path) {
if matches!(input.mode, ParseProviderMode::Ocr) {
return "mineru";
}
if native_text_confidence.is_some_and(|confidence| confidence < 0.2) {
return "mineru";
}
return "liteparse";
}
"mineru"
let _ = native_text_confidence;
"lightrag"
}
async fn parse_liteparse_input(input: ParseInput) -> Result<ParseProviderOutput, ParseError> {
let source_path = input.root_path.join(&input.source_root_relative_path);
let metadata = read_liteparse_source_metadata(&source_path)?;
let bytes = read_liteparse_source_bytes(&source_path)?;
let output = Command::new(liteparse_bin())
.arg("parse")
.arg("--format")
.arg("json")
.arg("--no-ocr")
.arg("-q")
.arg(&source_path)
.output()
.await
.map_err(|error| {
ParseError::new(
"liteparse_command_failed",
format!("LiteParse 命令执行失败: {error}"),
)
})?;
parse_liteparse_command_output(input, metadata, bytes, output)
}
pub(crate) fn parse_liteparse_input_blocking(
input: ParseInput,
) -> Result<ParseProviderOutput, ParseError> {
let source_path = input.root_path.join(&input.source_root_relative_path);
let metadata = read_liteparse_source_metadata(&source_path)?;
let bytes = read_liteparse_source_bytes(&source_path)?;
let output = StdCommand::new(liteparse_bin())
.arg("parse")
.arg("--format")
.arg("json")
.arg("--no-ocr")
.arg("-q")
.arg(&source_path)
.output()
.map_err(|error| {
ParseError::new(
"liteparse_command_failed",
format!("LiteParse 命令执行失败: {error}"),
)
})?;
parse_liteparse_command_output(input, metadata, bytes, output)
}
pub(crate) fn liteparse_runtime_available() -> bool {
if let Ok(value) = env::var("MNOTE_LITEPARSE_BIN") {
let trimmed = value.trim();
if trimmed.is_empty() {
return false;
}
let path = Path::new(trimmed);
return path.components().count() > 1 && path.exists() || command_exists(trimmed);
}
command_exists("lit") || command_exists("liteparse")
}
fn read_liteparse_source_metadata(source_path: &Path) -> Result<fs::Metadata, ParseError> {
fs::metadata(source_path).map_err(|error| {
ParseError::new(
"liteparse_parse_stat_failed",
format!(
"无法读取 LiteParse 证据源状态 {}: {error}",
source_path.display()
),
)
})
}
fn read_liteparse_source_bytes(source_path: &Path) -> Result<Vec<u8>, ParseError> {
fs::read(source_path).map_err(|error| {
ParseError::new(
"liteparse_parse_read_failed",
format!(
"无法读取 LiteParse 证据源 {}: {error}",
source_path.display()
),
)
})
}
fn parse_liteparse_command_output(
input: ParseInput,
metadata: fs::Metadata,
bytes: Vec<u8>,
output: std::process::Output,
) -> Result<ParseProviderOutput, ParseError> {
let updated_at_ms = metadata
.modified()
.ok()
.and_then(|value| value.duration_since(UNIX_EPOCH).ok())
.map(|value| value.as_millis() as u64)
.unwrap_or_default();
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
return Err(ParseError::new(
"liteparse_command_failed",
format!("LiteParse 解析失败: {}", stderr.trim()),
));
}
let stdout = String::from_utf8(output.stdout).map_err(|error| {
ParseError::new(
"liteparse_output_utf8_invalid",
format!("LiteParse 输出不是 UTF-8: {error}"),
)
})?;
let value = serde_json::from_str::<Value>(&stdout).map_err(|error| {
ParseError::new(
"liteparse_output_json_invalid",
format!("LiteParse JSON 输出无效: {error}"),
)
})?;
let source_hash = binary_source_hash(&bytes, metadata.len(), updated_at_ms);
let markdown = liteparse_markdown(&value);
let source_map = liteparse_source_map(&input, &value, &source_hash);
let artifact = ParsedResourceArtifact {
schema: PARSED_RESOURCE_ARTIFACT_SCHEMA.into(),
provider: "liteparse".into(),
model_version: liteparse_version(&value),
owner_document_id: input.owner_document_id,
owner_document_path: input.owner_document_path,
source_root_relative_path: input.source_root_relative_path.clone(),
source_hash,
artifact_root_relative_path: format!("{}.parse.md", input.source_root_relative_path),
source_map_root_relative_path: format!(
"{}.source-map.json",
input.source_root_relative_path
),
updated_at_ms,
};
Ok(ParseProviderOutput {
artifact,
source_map,
markdown,
})
let _ = input;
Err(ParseError::new(
"liteparse_retired",
"LiteParse 已退役；PDF、Office、图片 OCR 与资料索引统一由 LightRAG provider 处理",
))
}
fn parse_markdown_input(input: ParseInput) -> Result<ParseProviderOutput, ParseError> {
@@ -337,270 +185,6 @@ fn parse_markdown_input(input: ParseInput) -> Result<ParseProviderOutput, ParseE
})
}
fn liteparse_bin() -> String {
env::var("MNOTE_LITEPARSE_BIN")
.ok()
.filter(|value| !value.trim().is_empty())
.unwrap_or_else(|| {
if command_exists("lit") {
"lit".into()
} else {
"liteparse".into()
}
})
}
fn command_exists(command: &str) -> bool {
let path = Path::new(command);
if path.components().count() > 1 {
return path.exists();
}
let Some(paths) = env::var_os("PATH") else {
return false;
};
split_paths(&paths).any(|base| base.join(command).exists())
}
fn liteparse_version(value: &Value) -> Option<String> {
value
.get("version")
.or_else(|| value.get("modelVersion"))
.or_else(|| value.get("model_version"))
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
.or_else(|| Some("liteparse-v2".into()))
}
fn liteparse_markdown(value: &Value) -> String {
for key in ["markdown", "text", "content"] {
if let Some(text) = value.get(key).and_then(Value::as_str) {
let text = text.trim();
if !text.is_empty() {
return text.to_string();
}
}
}
let lines = liteparse_pages(value)
.into_iter()
.flat_map(|(_, page)| liteparse_blocks(&page))
.filter_map(|block| liteparse_item_text(&block))
.collect::<Vec<_>>();
lines.join("\n\n")
}
fn liteparse_source_map(input: &ParseInput, value: &Value, source_hash: &str) -> ResourceSourceMap {
let mut pages = Vec::new();
let mut sections = Vec::new();
let mut section_path = Vec::<String>::new();
let mut global_char = 0_u64;
for (page_index, page_value) in liteparse_pages(value) {
let page_number = liteparse_page_number(&page_value).unwrap_or(page_index);
let blocks = liteparse_blocks(&page_value);
let mut source_blocks = Vec::new();
let mut text_items = Vec::new();
for (block_index, block) in blocks.iter().enumerate() {
let Some(text) = liteparse_item_text(block) else {
continue;
};
let block_id = liteparse_item_id(block)
.unwrap_or_else(|| format!("p{page_number}_b{}", block_index + 1));
let text_item_id = format!("p{page_number}_t{}", block_index + 1);
let block_type = liteparse_block_kind(block);
let char_range = EvidenceRange {
start: global_char,
end: global_char + text.chars().count() as u64,
};
global_char = char_range.end + 1;
let bbox = liteparse_bbox(block);
if matches!(block_type, SourceMapBlockKind::Heading) {
let level = liteparse_heading_level(block).unwrap_or(1).max(1);
section_path.truncate(level.saturating_sub(1));
section_path.push(text.clone());
sections.push(SourceMapSection {
id: format!(
"sec_{}",
stable_segment(&format!(
"{}:{}:{}",
input.source_root_relative_path,
page_number,
section_path.join("/")
))
),
title: text.clone(),
path: section_path.clone(),
page_start: Some(page_number),
page_end: Some(page_number),
block_ids: vec![block_id.clone()],
});
} else if let Some(section) = sections.last_mut() {
section.page_end = Some(page_number);
if !section.block_ids.iter().any(|id| id == &block_id) {
section.block_ids.push(block_id.clone());
}
}
text_items.push(SourceMapTextItem {
id: text_item_id,
text: text.clone(),
bbox: bbox.clone(),
char_range: Some(char_range.clone()),
});
source_blocks.push(SourceMapBlock {
id: block_id,
block_type,
text,
bbox,
char_range: Some(char_range),
});
}
pages.push(SourceMapPage {
page: page_number,
width: liteparse_number(&page_value, &["width", "pageWidth"]),
height: liteparse_number(&page_value, &["height", "pageHeight"]),
text_items,
blocks: source_blocks,
});
}
ResourceSourceMap {
schema: RESOURCE_SOURCE_MAP_SCHEMA.into(),
provider: "liteparse".into(),
model_version: liteparse_version(value),
owner_document_path: input.owner_document_path.clone(),
source_root_relative_path: input.source_root_relative_path.clone(),
source_hash: source_hash.to_string(),
page_count: pages.iter().map(|page| page.page).max(),
pages,
sections,
}
}
fn liteparse_pages(value: &Value) -> Vec<(u32, Value)> {
if let Some(pages) = value.get("pages").and_then(Value::as_array) {
return pages
.iter()
.enumerate()
.map(|(index, page)| ((index + 1) as u32, page.clone()))
.collect();
}
vec![(1, json!({ "blocks": liteparse_blocks(value) }))]
}
fn liteparse_blocks(page: &Value) -> Vec<Value> {
for key in ["blocks", "items", "textItems", "text_items", "elements"] {
if let Some(items) = page.get(key).and_then(Value::as_array) {
return items.clone();
}
}
if let Some(text) = liteparse_item_text(page) {
return vec![json!({ "text": text })];
}
Vec::new()
}
fn liteparse_item_text(value: &Value) -> Option<String> {
for key in ["text", "content", "markdown", "value"] {
if let Some(text) = value.get(key).and_then(Value::as_str) {
let trimmed = text.trim();
if !trimmed.is_empty() {
return Some(trimmed.to_string());
}
}
}
None
}
fn liteparse_item_id(value: &Value) -> Option<String> {
value
.get("id")
.or_else(|| value.get("blockId"))
.or_else(|| value.get("block_id"))
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
}
fn liteparse_page_number(value: &Value) -> Option<u32> {
for key in ["page", "pageNumber", "page_number", "page_no", "pageNo"] {
if let Some(page) = value.get(key).and_then(Value::as_u64) {
return u32::try_from(page.max(1)).ok();
}
}
if let Some(page_idx) = value.get("page_idx").and_then(Value::as_u64) {
return u32::try_from(page_idx + 1).ok();
}
None
}
fn liteparse_block_kind(value: &Value) -> SourceMapBlockKind {
match value
.get("type")
.or_else(|| value.get("blockType"))
.or_else(|| value.get("block_type"))
.and_then(Value::as_str)
.unwrap_or_default()
.to_ascii_lowercase()
.as_str()
{
"title" | "heading" | "header" => SourceMapBlockKind::Heading,
"table" => SourceMapBlockKind::Table,
"figure" => SourceMapBlockKind::Figure,
"image" => SourceMapBlockKind::Image,
"list" => SourceMapBlockKind::List,
"text" | "paragraph" => SourceMapBlockKind::Paragraph,
_ => SourceMapBlockKind::Text,
}
}
fn liteparse_heading_level(value: &Value) -> Option<usize> {
value
.get("level")
.or_else(|| value.pointer("/props/level"))
.and_then(Value::as_u64)
.map(|value| value as usize)
}
fn liteparse_bbox(value: &Value) -> Option<EvidenceBBox> {
if let Some(bbox) = value
.get("bbox")
.or_else(|| value.get("boundingBox"))
.or_else(|| value.get("bounding_box"))
.and_then(Value::as_array)
{
if bbox.len() != 4 {
return None;
}
return Some(EvidenceBBox {
x0: bbox[0].as_f64()?,
y0: bbox[1].as_f64()?,
x1: bbox[2].as_f64()?,
y1: bbox[3].as_f64()?,
});
}
let x = value.get("x").and_then(Value::as_f64)?;
let y = value.get("y").and_then(Value::as_f64)?;
let width = value
.get("width")
.or_else(|| value.get("w"))
.and_then(Value::as_f64)?;
let height = value
.get("height")
.or_else(|| value.get("h"))
.and_then(Value::as_f64)?;
Some(EvidenceBBox {
x0: x,
y0: y,
x1: x + width,
y1: y + height,
})
}
fn liteparse_number(value: &Value, keys: &[&str]) -> Option<f64> {
keys.iter()
.find_map(|key| value.get(*key).and_then(Value::as_f64))
}
fn markdown_source_map(input: &ParseInput, markdown: &str, source_hash: &str) -> ResourceSourceMap {
let mut section_path: Vec<String> = Vec::new();
let mut sections = Vec::new();
@@ -686,18 +270,6 @@ fn is_markdown_path(path: &str) -> bool {
.is_some_and(|ext| ext.eq_ignore_ascii_case("md") || ext.eq_ignore_ascii_case("markdown"))
}
fn is_liteparse_path(path: &str) -> bool {
Path::new(path)
.extension()
.and_then(|value| value.to_str())
.is_some_and(|ext| {
matches!(
ext.to_ascii_lowercase().as_str(),
"pdf" | "doc" | "docx" | "odt" | "ppt" | "pptx" | "odp" | "xls" | "xlsx" | "ods"
)
})
}
fn source_hash(content: &str, size: u64, updated_at_ms: u64) -> String {
format!(
"fnv1a64:{:016x}:size:{size}:mtime:{updated_at_ms}",
@@ -705,13 +277,6 @@ fn source_hash(content: &str, size: u64, updated_at_ms: u64) -> String {
)
}
fn binary_source_hash(content: &[u8], size: u64, updated_at_ms: u64) -> String {
format!(
"fnv1a64:{:016x}:size:{size}:mtime:{updated_at_ms}",
fnv1a64(content)
)
}
fn fnv1a64(bytes: &[u8]) -> u64 {
let mut hash = 0xcbf29ce484222325_u64;
for byte in bytes {
@@ -728,12 +293,6 @@ fn stable_segment(value: &str) -> String {
#[cfg(test)]
mod tests {
use super::*;
use std::sync::{Mutex, OnceLock};
fn env_lock() -> &'static Mutex<()> {
static LOCK: OnceLock<Mutex<()>> = OnceLock::new();
LOCK.get_or_init(|| Mutex::new(()))
}
fn temp_root(name: &str) -> PathBuf {
let root = std::env::temp_dir().join(format!("{name}-{}", std::process::id()));
@@ -781,7 +340,7 @@ mod tests {
}
#[test]
fn default_provider_routes_text_pdf_to_liteparse_no_ocr() {
fn default_provider_routes_pdf_to_lightrag_after_liteparse_retirement() {
let input = ParseInput {
root_path: PathBuf::from("/workspace"),
root_uri: "file:///workspace".into(),
@@ -791,16 +350,13 @@ mod tests {
mode: ParseProviderMode::NoOcr,
};
let liteparse = LiteParseProvider;
assert_eq!(default_parse_provider_id(&input), "liteparse");
assert_eq!(liteparse.can_parse(&input), ParseCapability::Preferred);
assert_eq!(default_parse_provider_id(&input), "lightrag");
assert_eq!(liteparse.can_parse(&input), ParseCapability::Unsupported);
}
#[tokio::test]
#[cfg(unix)]
async fn liteparse_provider_maps_json_output_to_artifact_and_source_map() {
use std::os::unix::fs::PermissionsExt;
let _guard = env_lock().lock().expect("env lock");
async fn liteparse_provider_returns_retired_error() {
let root = temp_root("mnote-liteparse-provider");
fs::create_dir_all(root.join("docs").join("Page.assets")).expect("assets");
fs::write(
@@ -808,32 +364,6 @@ mod tests {
b"%PDF-1.4",
)
.expect("pdf");
let lit = root.join("fake-lit");
fs::write(
&lit,
r#"#!/usr/bin/env bash
cat <<'JSON'
{
"version": "liteparse-test",
"pages": [{
"page": 3,
"width": 612,
"height": 792,
"blocks": [
{"id": "b1", "type": "heading", "level": 1, "text": "解除条件", "bbox": [72, 120, 220, 150]},
{"id": "b2", "type": "text", "text": "提前三十日通知", "x": 72, "y": 160, "width": 228, "height": 28}
]
}]
}
JSON
"#,
)
.expect("fake lit");
let mut permissions = fs::metadata(&lit).expect("fake lit metadata").permissions();
permissions.set_mode(0o755);
fs::set_permissions(&lit, permissions).expect("chmod fake lit");
let old_bin = env::var("MNOTE_LITEPARSE_BIN").ok();
env::set_var("MNOTE_LITEPARSE_BIN", &lit);
let input = ParseInput {
root_path: root.clone(),
@@ -846,40 +376,15 @@ JSON
let output = LiteParseProvider
.parse(input)
.await
.expect("parse liteparse");
.expect_err("liteparse retired");
if let Some(value) = old_bin {
env::set_var("MNOTE_LITEPARSE_BIN", value);
} else {
env::remove_var("MNOTE_LITEPARSE_BIN");
}
assert_eq!(output.artifact.provider, "liteparse");
assert_eq!(
output.artifact.model_version.as_deref(),
Some("liteparse-test")
);
assert!(output.artifact.source_hash.starts_with("fnv1a64:"));
assert_eq!(output.markdown, "解除条件\n\n提前三十日通知");
assert_eq!(output.source_map.provider, "liteparse");
assert_eq!(output.source_map.pages[0].page, 3);
assert_eq!(
output.source_map.pages[0].blocks[1]
.bbox
.as_ref()
.unwrap()
.x0,
72.0
);
assert_eq!(
output.source_map.sections[0].path,
vec!["解除条件".to_string()]
);
assert_eq!(output.code, "liteparse_retired");
let _ = fs::remove_dir_all(&root);
}
#[test]
fn default_provider_routes_ocr_policy_to_mineru() {
fn default_provider_routes_ocr_policy_to_lightrag() {
let input = ParseInput {
root_path: PathBuf::from("/workspace"),
root_uri: "file:///workspace".into(),
@@ -888,11 +393,11 @@ JSON
source_root_relative_path: "Page.assets/spec.pdf".into(),
mode: ParseProviderMode::Ocr,
};
assert_eq!(default_parse_provider_id(&input), "mineru");
assert_eq!(default_parse_provider_id(&input), "lightrag");
}
#[test]
fn provider_selection_routes_scanned_pdf_or_image_to_mineru() {
fn provider_selection_routes_scanned_pdf_or_image_to_lightrag() {
let scanned_pdf = ParseInput {
root_path: PathBuf::from("/workspace"),
root_uri: "file:///workspace".into(),
@@ -901,12 +406,15 @@ JSON
source_root_relative_path: "Page.assets/scan.pdf".into(),
mode: ParseProviderMode::Auto,
};
assert_eq!(select_parse_provider_id(&scanned_pdf, Some(0.05)), "mineru");
assert_eq!(
select_parse_provider_id(&scanned_pdf, Some(0.05)),
"lightrag"
);
let image = ParseInput {
source_root_relative_path: "Page.assets/photo.png".into(),
..scanned_pdf
};
assert_eq!(select_parse_provider_id(&image, None), "mineru");
assert_eq!(select_parse_provider_id(&image, None), "lightrag");
}
}