2026-06-01 09:29:12 +08:00
|
|
|
|
use crate::app::AppState;
|
|
|
|
|
|
use crate::context::RequestContext;
|
|
|
|
|
|
use crate::error::WebError;
|
|
|
|
|
|
use crate::routes::local_folder_source::{
|
|
|
|
|
|
decode_local_id_segment, encode_local_id_segment,
|
|
|
|
|
|
ensure_local_workspace_read_access_with_state, ensure_local_workspace_write_access_with_state,
|
|
|
|
|
|
};
|
|
|
|
|
|
use axum::extract::{Extension, Query, State};
|
|
|
|
|
|
use axum::http::{HeaderMap, StatusCode};
|
|
|
|
|
|
use axum::Json;
|
2026-06-04 18:51:16 +08:00
|
|
|
|
use core_protocol::{
|
|
|
|
|
|
EvidenceBBox, EvidenceRange, ResourceSourceMap, SourceMapBlock, SourceMapBlockKind,
|
|
|
|
|
|
SourceMapPage, SourceMapSection, SourceMapTextItem, RESOURCE_SOURCE_MAP_SCHEMA,
|
|
|
|
|
|
};
|
2026-06-01 09:29:12 +08:00
|
|
|
|
use serde::{Deserialize, Serialize};
|
|
|
|
|
|
use serde_json::{json, Value};
|
|
|
|
|
|
use std::collections::{BTreeMap, HashMap};
|
|
|
|
|
|
use std::fs;
|
|
|
|
|
|
use std::hash::{Hash, Hasher};
|
|
|
|
|
|
use std::io::{Cursor, Read};
|
|
|
|
|
|
use std::path::{Component, Path, PathBuf};
|
|
|
|
|
|
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
|
|
|
|
|
|
|
|
|
|
|
const OCR_INDEX_VERSION: u32 = 1;
|
|
|
|
|
|
const DEFAULT_PROVIDER: &str = "mineru";
|
|
|
|
|
|
const DEFAULT_MODEL_VERSION: &str = "vlm";
|
|
|
|
|
|
const DEFAULT_MINERU_API_BASE_URL: &str = "https://mineru.net";
|
|
|
|
|
|
|
|
|
|
|
|
#[derive(Debug, Deserialize)]
|
|
|
|
|
|
#[serde(rename_all = "camelCase")]
|
|
|
|
|
|
pub(crate) struct OcrJobRequest {
|
|
|
|
|
|
root_uri: String,
|
|
|
|
|
|
document_id: String,
|
|
|
|
|
|
#[serde(default)]
|
|
|
|
|
|
source_path: Option<String>,
|
|
|
|
|
|
source_root_relative_path: String,
|
|
|
|
|
|
#[serde(default)]
|
|
|
|
|
|
provider: Option<String>,
|
|
|
|
|
|
#[serde(default)]
|
|
|
|
|
|
force: bool,
|
|
|
|
|
|
#[serde(default)]
|
|
|
|
|
|
mock_markdown: Option<String>,
|
|
|
|
|
|
#[serde(default)]
|
|
|
|
|
|
mock_error: Option<String>,
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[derive(Debug, Deserialize)]
|
|
|
|
|
|
#[serde(rename_all = "camelCase")]
|
|
|
|
|
|
pub(crate) struct OcrStatusQuery {
|
|
|
|
|
|
root_uri: String,
|
|
|
|
|
|
source_root_relative_path: Option<String>,
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[derive(Debug, Deserialize)]
|
|
|
|
|
|
#[serde(rename_all = "camelCase")]
|
|
|
|
|
|
pub(crate) struct OcrReadQuery {
|
|
|
|
|
|
root_uri: String,
|
|
|
|
|
|
ocr_root_relative_path: String,
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[derive(Debug, Deserialize)]
|
|
|
|
|
|
#[serde(rename_all = "camelCase")]
|
|
|
|
|
|
pub(crate) struct OcrInsertRequest {
|
|
|
|
|
|
root_uri: String,
|
|
|
|
|
|
document_id: String,
|
|
|
|
|
|
ocr_root_relative_path: String,
|
|
|
|
|
|
#[serde(default)]
|
|
|
|
|
|
mode: Option<String>,
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-06-02 17:17:49 +08:00
|
|
|
|
#[derive(Debug, Deserialize)]
|
|
|
|
|
|
#[serde(rename_all = "camelCase")]
|
|
|
|
|
|
pub(crate) struct OcrDeleteRequest {
|
|
|
|
|
|
root_uri: String,
|
|
|
|
|
|
source_root_relative_path: String,
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-06-01 09:29:12 +08:00
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
|
|
|
|
#[serde(rename_all = "camelCase")]
|
|
|
|
|
|
struct OcrIndex {
|
|
|
|
|
|
version: u32,
|
|
|
|
|
|
#[serde(default)]
|
|
|
|
|
|
entries: BTreeMap<String, OcrIndexEntry>,
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
|
|
|
|
#[serde(rename_all = "camelCase")]
|
|
|
|
|
|
pub(crate) struct OcrIndexEntry {
|
|
|
|
|
|
pub(crate) job_id: String,
|
|
|
|
|
|
pub(crate) owner_document_id: String,
|
|
|
|
|
|
pub(crate) owner_document_path: String,
|
|
|
|
|
|
pub(crate) source_root_relative_path: String,
|
|
|
|
|
|
pub(crate) ocr_root_relative_path: String,
|
|
|
|
|
|
pub(crate) provider: String,
|
|
|
|
|
|
pub(crate) model_version: String,
|
|
|
|
|
|
pub(crate) status: String,
|
|
|
|
|
|
pub(crate) source_size: u64,
|
|
|
|
|
|
pub(crate) source_mtime_ms: u128,
|
|
|
|
|
|
pub(crate) created_at_ms: u128,
|
|
|
|
|
|
pub(crate) updated_at_ms: u128,
|
|
|
|
|
|
#[serde(default)]
|
|
|
|
|
|
pub(crate) plain_text_preview: String,
|
|
|
|
|
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
|
|
|
|
|
pub(crate) error: Option<String>,
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[derive(Debug, Clone)]
|
|
|
|
|
|
struct SourceMetadata {
|
|
|
|
|
|
size: u64,
|
|
|
|
|
|
mtime_ms: u128,
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[derive(Debug, Clone)]
|
|
|
|
|
|
struct MineruClientConfig {
|
|
|
|
|
|
api_base_url: String,
|
|
|
|
|
|
token: String,
|
|
|
|
|
|
model_version: String,
|
|
|
|
|
|
poll_interval: Duration,
|
|
|
|
|
|
max_polls: usize,
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-06-02 17:17:49 +08:00
|
|
|
|
#[derive(Debug)]
|
|
|
|
|
|
struct MineruZipAsset {
|
|
|
|
|
|
relative_path: PathBuf,
|
|
|
|
|
|
bytes: Vec<u8>,
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[derive(Debug)]
|
|
|
|
|
|
struct MineruZipExtraction {
|
|
|
|
|
|
markdown: String,
|
|
|
|
|
|
assets: Vec<MineruZipAsset>,
|
2026-06-04 18:51:16 +08:00
|
|
|
|
source_map_input: Option<Value>,
|
2026-06-02 17:17:49 +08:00
|
|
|
|
}
|
|
|
|
|
|
|
2026-06-01 09:29:12 +08:00
|
|
|
|
#[derive(Debug, Clone)]
|
|
|
|
|
|
struct OcrSidecarPlan {
|
|
|
|
|
|
owner_document_path: String,
|
|
|
|
|
|
source_root_relative_path: String,
|
|
|
|
|
|
ocr_root_relative_path: String,
|
|
|
|
|
|
ocr_path: PathBuf,
|
|
|
|
|
|
provider: String,
|
|
|
|
|
|
model_version: String,
|
|
|
|
|
|
source_size: u64,
|
|
|
|
|
|
source_mtime_ms: u128,
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[derive(Debug, Clone)]
|
|
|
|
|
|
pub(crate) struct OcrFrontmatter {
|
|
|
|
|
|
pub provider: String,
|
|
|
|
|
|
pub owner_document: String,
|
|
|
|
|
|
pub source_path: String,
|
|
|
|
|
|
pub source_root_relative_path: String,
|
|
|
|
|
|
pub source_size: u64,
|
|
|
|
|
|
pub source_mtime_ms: u128,
|
|
|
|
|
|
pub status: String,
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
pub(crate) async fn create_job(
|
|
|
|
|
|
State(state): State<AppState>,
|
|
|
|
|
|
Extension(context): Extension<RequestContext>,
|
|
|
|
|
|
Json(body): Json<OcrJobRequest>,
|
|
|
|
|
|
) -> Result<(StatusCode, HeaderMap, Json<Value>), WebError> {
|
|
|
|
|
|
let root_uri = body.root_uri.trim();
|
|
|
|
|
|
let root = ensure_local_workspace_write_access_with_state(&state, &context, root_uri)
|
|
|
|
|
|
.map_err(|error| error.with_context(&context))?;
|
|
|
|
|
|
let provider = body
|
|
|
|
|
|
.provider
|
|
|
|
|
|
.as_deref()
|
|
|
|
|
|
.map(str::trim)
|
|
|
|
|
|
.filter(|value| !value.is_empty())
|
|
|
|
|
|
.unwrap_or(DEFAULT_PROVIDER);
|
|
|
|
|
|
let source_relative = body.source_root_relative_path.trim().replace('\\', "/");
|
|
|
|
|
|
let _source_path_hint = body
|
|
|
|
|
|
.source_path
|
|
|
|
|
|
.as_deref()
|
|
|
|
|
|
.map(str::trim)
|
|
|
|
|
|
.filter(|value| !value.is_empty())
|
|
|
|
|
|
.unwrap_or(&source_relative);
|
|
|
|
|
|
let plan = plan_ocr_sidecar_path(
|
|
|
|
|
|
&root,
|
|
|
|
|
|
&body.document_id,
|
|
|
|
|
|
&source_relative,
|
|
|
|
|
|
provider,
|
|
|
|
|
|
DEFAULT_MODEL_VERSION,
|
|
|
|
|
|
body.force,
|
|
|
|
|
|
)?;
|
2026-06-04 18:51:16 +08:00
|
|
|
|
if !body.force {
|
|
|
|
|
|
if let Some(entry) = reusable_existing_ocr_entry(&state, &root, root_uri, &plan)? {
|
|
|
|
|
|
return Ok(ok_json(
|
|
|
|
|
|
&context,
|
|
|
|
|
|
json!({
|
|
|
|
|
|
"ok": true,
|
|
|
|
|
|
"deduplicated": true,
|
|
|
|
|
|
"job": ocr_job_payload(&root, &entry),
|
|
|
|
|
|
}),
|
|
|
|
|
|
));
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
let token = if provider != "mock" {
|
|
|
|
|
|
Some(mineru_token().ok_or_else(|| {
|
|
|
|
|
|
WebError::new(
|
|
|
|
|
|
StatusCode::UNAUTHORIZED,
|
|
|
|
|
|
"mineru_token_missing",
|
|
|
|
|
|
"缺少 MinerU API token",
|
|
|
|
|
|
)
|
|
|
|
|
|
.with_context(&context)
|
|
|
|
|
|
})?)
|
|
|
|
|
|
} else {
|
|
|
|
|
|
None
|
|
|
|
|
|
};
|
2026-06-01 09:29:12 +08:00
|
|
|
|
let now = now_ms();
|
2026-06-01 10:30:42 +08:00
|
|
|
|
let mut entry = build_index_entry(&plan, "queued", now, "", None);
|
|
|
|
|
|
upsert_and_broadcast_ocr_index_entry(&state, &root, root_uri, entry.clone())?;
|
2026-06-01 09:29:12 +08:00
|
|
|
|
if let Some(error) = body.mock_error.as_deref() {
|
2026-06-01 10:30:42 +08:00
|
|
|
|
entry = advance_ocr_entry(&entry, "failed", now_ms(), "", Some(redact_error(error)));
|
|
|
|
|
|
upsert_and_broadcast_ocr_index_entry(&state, &root, root_uri, entry.clone())?;
|
2026-06-01 09:29:12 +08:00
|
|
|
|
return Ok(ok_json(
|
|
|
|
|
|
&context,
|
|
|
|
|
|
json!({
|
|
|
|
|
|
"ok": true,
|
|
|
|
|
|
"job": ocr_job_payload(&root, &entry),
|
|
|
|
|
|
}),
|
|
|
|
|
|
));
|
|
|
|
|
|
}
|
|
|
|
|
|
let markdown = if provider == "mock" {
|
2026-06-01 10:30:42 +08:00
|
|
|
|
entry = advance_ocr_entry(&entry, "writing_sidecar", now_ms(), "", None);
|
|
|
|
|
|
upsert_and_broadcast_ocr_index_entry(&state, &root, root_uri, entry.clone())?;
|
2026-06-01 09:29:12 +08:00
|
|
|
|
body.mock_markdown
|
|
|
|
|
|
.unwrap_or_else(|| format!("OCR mock result for {}", plan.source_root_relative_path))
|
|
|
|
|
|
} else {
|
2026-06-01 10:30:42 +08:00
|
|
|
|
entry = advance_ocr_entry(&entry, "uploading", now_ms(), "", None);
|
|
|
|
|
|
upsert_and_broadcast_ocr_index_entry(&state, &root, root_uri, entry.clone())?;
|
|
|
|
|
|
match run_mineru_ocr(
|
|
|
|
|
|
&state,
|
|
|
|
|
|
&root,
|
|
|
|
|
|
root_uri,
|
|
|
|
|
|
&plan,
|
|
|
|
|
|
token.unwrap_or_default(),
|
|
|
|
|
|
entry.clone(),
|
|
|
|
|
|
)
|
|
|
|
|
|
.await
|
|
|
|
|
|
{
|
2026-06-01 09:29:12 +08:00
|
|
|
|
Ok(markdown) => markdown,
|
|
|
|
|
|
Err(error) => {
|
2026-06-01 10:30:42 +08:00
|
|
|
|
let failed_entry = advance_ocr_entry(
|
|
|
|
|
|
&entry,
|
2026-06-01 10:07:42 +08:00
|
|
|
|
"failed",
|
2026-06-01 10:30:42 +08:00
|
|
|
|
now_ms(),
|
2026-06-01 10:07:42 +08:00
|
|
|
|
"",
|
|
|
|
|
|
Some(redact_error(error.message())),
|
|
|
|
|
|
);
|
2026-06-01 10:30:42 +08:00
|
|
|
|
upsert_and_broadcast_ocr_index_entry(&state, &root, root_uri, failed_entry)?;
|
2026-06-01 09:29:12 +08:00
|
|
|
|
return Err(error.with_context(&context));
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
};
|
2026-06-01 10:30:42 +08:00
|
|
|
|
if entry.status != "writing_sidecar" {
|
|
|
|
|
|
entry = advance_ocr_entry(&entry, "writing_sidecar", now_ms(), "", None);
|
|
|
|
|
|
upsert_and_broadcast_ocr_index_entry(&state, &root, root_uri, entry.clone())?;
|
|
|
|
|
|
}
|
2026-06-01 09:29:12 +08:00
|
|
|
|
write_ocr_sidecar(&plan, &markdown, now)?;
|
2026-06-01 10:30:42 +08:00
|
|
|
|
entry = advance_ocr_entry(&entry, "done", now_ms(), &markdown, None);
|
|
|
|
|
|
upsert_and_broadcast_ocr_index_entry(&state, &root, root_uri, entry.clone())?;
|
2026-06-01 09:29:12 +08:00
|
|
|
|
Ok(ok_json(
|
|
|
|
|
|
&context,
|
|
|
|
|
|
json!({
|
|
|
|
|
|
"ok": true,
|
|
|
|
|
|
"job": ocr_job_payload(&root, &entry),
|
|
|
|
|
|
}),
|
|
|
|
|
|
))
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
pub(crate) async fn list_jobs(
|
|
|
|
|
|
State(state): State<AppState>,
|
|
|
|
|
|
Extension(context): Extension<RequestContext>,
|
|
|
|
|
|
Query(query): Query<OcrStatusQuery>,
|
|
|
|
|
|
) -> Result<(StatusCode, HeaderMap, Json<Value>), WebError> {
|
|
|
|
|
|
let root =
|
|
|
|
|
|
ensure_local_workspace_read_access_with_state(&state, &context, query.root_uri.trim())
|
|
|
|
|
|
.map_err(|error| error.with_context(&context))?;
|
|
|
|
|
|
let index = read_ocr_index(&root)?;
|
|
|
|
|
|
let jobs = index
|
|
|
|
|
|
.entries
|
|
|
|
|
|
.values()
|
|
|
|
|
|
.map(|entry| ocr_job_payload(&root, entry))
|
|
|
|
|
|
.collect::<Vec<_>>();
|
|
|
|
|
|
Ok(ok_json(&context, json!({ "ok": true, "jobs": jobs })))
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
pub(crate) async fn status(
|
|
|
|
|
|
State(state): State<AppState>,
|
|
|
|
|
|
Extension(context): Extension<RequestContext>,
|
|
|
|
|
|
Query(query): Query<OcrStatusQuery>,
|
|
|
|
|
|
) -> Result<(StatusCode, HeaderMap, Json<Value>), WebError> {
|
|
|
|
|
|
let root =
|
|
|
|
|
|
ensure_local_workspace_read_access_with_state(&state, &context, query.root_uri.trim())
|
|
|
|
|
|
.map_err(|error| error.with_context(&context))?;
|
|
|
|
|
|
let source = query
|
|
|
|
|
|
.source_root_relative_path
|
|
|
|
|
|
.as_deref()
|
|
|
|
|
|
.map(str::trim)
|
|
|
|
|
|
.filter(|value| !value.is_empty())
|
|
|
|
|
|
.ok_or_else(|| {
|
|
|
|
|
|
WebError::bad_request_code("local_ocr_source_required", "OCR 状态查询缺少来源路径")
|
|
|
|
|
|
.with_context(&context)
|
|
|
|
|
|
})?;
|
|
|
|
|
|
let index = read_ocr_index(&root)?;
|
|
|
|
|
|
let job = index
|
|
|
|
|
|
.entries
|
|
|
|
|
|
.get(source)
|
|
|
|
|
|
.map(|entry| ocr_job_payload(&root, entry));
|
|
|
|
|
|
Ok(ok_json(&context, json!({ "ok": true, "job": job })))
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-06-02 17:17:49 +08:00
|
|
|
|
pub(crate) async fn delete_job(
|
|
|
|
|
|
State(state): State<AppState>,
|
|
|
|
|
|
Extension(context): Extension<RequestContext>,
|
|
|
|
|
|
Json(request): Json<OcrDeleteRequest>,
|
|
|
|
|
|
) -> Result<(StatusCode, HeaderMap, Json<Value>), WebError> {
|
|
|
|
|
|
let root =
|
|
|
|
|
|
ensure_local_workspace_write_access_with_state(&state, &context, request.root_uri.trim())
|
|
|
|
|
|
.map_err(|error| error.with_context(&context))?;
|
|
|
|
|
|
let source = normalize_relative_path(&request.source_root_relative_path)?;
|
|
|
|
|
|
let mut index = read_ocr_index(&root)?;
|
|
|
|
|
|
let removed = index.entries.remove(&source);
|
|
|
|
|
|
if let Some(entry) = &removed {
|
|
|
|
|
|
let sidecar = root.join(&entry.ocr_root_relative_path);
|
2026-06-04 18:51:16 +08:00
|
|
|
|
let parse_sidecar = root.join(parse_sidecar_relative_path(&entry.ocr_root_relative_path));
|
|
|
|
|
|
let source_map_sidecar = root.join(source_map_sidecar_relative_path(
|
|
|
|
|
|
&entry.ocr_root_relative_path,
|
|
|
|
|
|
));
|
2026-06-02 17:17:49 +08:00
|
|
|
|
ensure_target_under_root(&root, &sidecar, "local_ocr_delete_root_escape")?;
|
|
|
|
|
|
if sidecar.exists() {
|
|
|
|
|
|
fs::remove_file(&sidecar).map_err(|error| {
|
|
|
|
|
|
WebError::bad_request_code(
|
|
|
|
|
|
"local_ocr_delete_failed",
|
|
|
|
|
|
format!("无法删除 OCR Markdown {}: {error}", sidecar.display()),
|
|
|
|
|
|
)
|
|
|
|
|
|
.with_context(&context)
|
|
|
|
|
|
})?;
|
|
|
|
|
|
}
|
2026-06-04 18:51:16 +08:00
|
|
|
|
if parse_sidecar.exists() {
|
|
|
|
|
|
fs::remove_file(&parse_sidecar).map_err(|error| {
|
|
|
|
|
|
WebError::bad_request_code(
|
|
|
|
|
|
"local_ocr_delete_failed",
|
|
|
|
|
|
format!(
|
|
|
|
|
|
"无法删除 OCR Parse Markdown {}: {error}",
|
|
|
|
|
|
parse_sidecar.display()
|
|
|
|
|
|
),
|
|
|
|
|
|
)
|
|
|
|
|
|
.with_context(&context)
|
|
|
|
|
|
})?;
|
|
|
|
|
|
}
|
|
|
|
|
|
if source_map_sidecar.exists() {
|
|
|
|
|
|
fs::remove_file(&source_map_sidecar).map_err(|error| {
|
|
|
|
|
|
WebError::bad_request_code(
|
|
|
|
|
|
"local_ocr_delete_failed",
|
|
|
|
|
|
format!(
|
|
|
|
|
|
"无法删除 OCR source-map {}: {error}",
|
|
|
|
|
|
source_map_sidecar.display()
|
|
|
|
|
|
),
|
|
|
|
|
|
)
|
|
|
|
|
|
.with_context(&context)
|
|
|
|
|
|
})?;
|
|
|
|
|
|
}
|
2026-06-02 17:17:49 +08:00
|
|
|
|
cleanup_empty_ocr_sidecar_dir(&root, &sidecar)?;
|
|
|
|
|
|
}
|
|
|
|
|
|
write_ocr_index(&root, &index)?;
|
|
|
|
|
|
let key = format!("{}:{source}", request.root_uri.trim());
|
|
|
|
|
|
if let Ok(mut jobs) = state.local_ocr_active_jobs.write() {
|
|
|
|
|
|
jobs.remove(&key);
|
|
|
|
|
|
}
|
|
|
|
|
|
broadcast_ocr_job_deleted(&state, request.root_uri.trim(), &source, removed.as_ref());
|
|
|
|
|
|
Ok(ok_json(
|
|
|
|
|
|
&context,
|
|
|
|
|
|
json!({
|
|
|
|
|
|
"ok": true,
|
|
|
|
|
|
"deleted": removed.is_some(),
|
|
|
|
|
|
"sourceRootRelativePath": source,
|
|
|
|
|
|
}),
|
|
|
|
|
|
))
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-06-01 09:29:12 +08:00
|
|
|
|
pub(crate) async fn read(
|
|
|
|
|
|
State(state): State<AppState>,
|
|
|
|
|
|
Extension(context): Extension<RequestContext>,
|
|
|
|
|
|
Query(query): Query<OcrReadQuery>,
|
|
|
|
|
|
) -> Result<(StatusCode, HeaderMap, Json<Value>), WebError> {
|
|
|
|
|
|
let root =
|
|
|
|
|
|
ensure_local_workspace_read_access_with_state(&state, &context, query.root_uri.trim())
|
|
|
|
|
|
.map_err(|error| error.with_context(&context))?;
|
|
|
|
|
|
let relative = normalize_relative_path(&query.ocr_root_relative_path)?;
|
|
|
|
|
|
if !is_local_ocr_sidecar_relative_path(&relative) {
|
|
|
|
|
|
return Err(WebError::bad_request_code(
|
|
|
|
|
|
"local_ocr_sidecar_invalid",
|
|
|
|
|
|
"OCR 读取目标不是 OCR sidecar",
|
|
|
|
|
|
)
|
|
|
|
|
|
.with_context(&context));
|
|
|
|
|
|
}
|
|
|
|
|
|
let path = root.join(&relative);
|
|
|
|
|
|
ensure_target_under_root(&root, &path, "local_ocr_read_root_escape")?;
|
|
|
|
|
|
let markdown = fs::read_to_string(&path).map_err(|error| {
|
|
|
|
|
|
WebError::bad_request_code(
|
|
|
|
|
|
"local_ocr_read_failed",
|
|
|
|
|
|
format!("无法读取 OCR Markdown {}: {error}", path.display()),
|
|
|
|
|
|
)
|
|
|
|
|
|
.with_context(&context)
|
|
|
|
|
|
})?;
|
|
|
|
|
|
let frontmatter = parse_ocr_frontmatter(&markdown);
|
|
|
|
|
|
Ok(ok_json(
|
|
|
|
|
|
&context,
|
|
|
|
|
|
json!({
|
|
|
|
|
|
"ok": true,
|
|
|
|
|
|
"markdown": markdown,
|
|
|
|
|
|
"frontmatter": frontmatter.map(ocr_frontmatter_payload),
|
|
|
|
|
|
"ocrRootRelativePath": relative,
|
|
|
|
|
|
}),
|
|
|
|
|
|
))
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
pub(crate) async fn insert(
|
|
|
|
|
|
State(state): State<AppState>,
|
|
|
|
|
|
Extension(context): Extension<RequestContext>,
|
|
|
|
|
|
Json(body): Json<OcrInsertRequest>,
|
|
|
|
|
|
) -> Result<(StatusCode, HeaderMap, Json<Value>), WebError> {
|
|
|
|
|
|
let root =
|
|
|
|
|
|
ensure_local_workspace_write_access_with_state(&state, &context, body.root_uri.trim())
|
|
|
|
|
|
.map_err(|error| error.with_context(&context))?;
|
|
|
|
|
|
let mode = body
|
|
|
|
|
|
.mode
|
|
|
|
|
|
.as_deref()
|
|
|
|
|
|
.map(str::trim)
|
|
|
|
|
|
.filter(|value| !value.is_empty())
|
|
|
|
|
|
.unwrap_or("link");
|
|
|
|
|
|
if mode != "link" {
|
|
|
|
|
|
return Err(WebError::bad_request_code(
|
|
|
|
|
|
"local_ocr_insert_mode_unsupported",
|
|
|
|
|
|
"当前仅支持插入 OCR 链接",
|
|
|
|
|
|
)
|
|
|
|
|
|
.with_context(&context));
|
|
|
|
|
|
}
|
|
|
|
|
|
let owner_document_path = owner_document_path_from_id(&body.document_id)?;
|
|
|
|
|
|
let owner_path = root.join(&owner_document_path);
|
|
|
|
|
|
ensure_target_under_root(&root, &owner_path, "local_ocr_owner_root_escape")
|
|
|
|
|
|
.map_err(|error| error.with_context(&context))?;
|
|
|
|
|
|
if !owner_path.is_file() {
|
|
|
|
|
|
return Err(WebError::bad_request_code(
|
|
|
|
|
|
"local_ocr_owner_missing",
|
|
|
|
|
|
"OCR owner Markdown 不存在",
|
|
|
|
|
|
)
|
|
|
|
|
|
.with_context(&context));
|
|
|
|
|
|
}
|
|
|
|
|
|
let ocr_root_relative_path = normalize_relative_path(&body.ocr_root_relative_path)?;
|
|
|
|
|
|
if !is_local_ocr_sidecar_relative_path(&ocr_root_relative_path) {
|
|
|
|
|
|
return Err(WebError::bad_request_code(
|
|
|
|
|
|
"local_ocr_sidecar_invalid",
|
|
|
|
|
|
"OCR 插入目标不是 OCR sidecar",
|
|
|
|
|
|
)
|
|
|
|
|
|
.with_context(&context));
|
|
|
|
|
|
}
|
|
|
|
|
|
let ocr_path = root.join(&ocr_root_relative_path);
|
|
|
|
|
|
ensure_target_under_root(&root, &ocr_path, "local_ocr_read_root_escape")
|
|
|
|
|
|
.map_err(|error| error.with_context(&context))?;
|
|
|
|
|
|
if !ocr_path.is_file() {
|
|
|
|
|
|
return Err(
|
|
|
|
|
|
WebError::bad_request_code("local_ocr_read_failed", "OCR Markdown 不存在")
|
|
|
|
|
|
.with_context(&context),
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
let current = fs::read_to_string(&owner_path).map_err(|error| {
|
|
|
|
|
|
WebError::bad_request_code(
|
|
|
|
|
|
"local_ocr_owner_read_failed",
|
|
|
|
|
|
format!(
|
|
|
|
|
|
"无法读取 OCR owner Markdown {}: {error}",
|
|
|
|
|
|
owner_path.display()
|
|
|
|
|
|
),
|
|
|
|
|
|
)
|
|
|
|
|
|
.with_context(&context)
|
|
|
|
|
|
})?;
|
|
|
|
|
|
let link_target = relative_from_owner_dir(&owner_document_path, &ocr_root_relative_path);
|
|
|
|
|
|
let label = ocr_link_label(&ocr_root_relative_path);
|
|
|
|
|
|
let inserted_markdown = format!("[OCR:{}]({})", label, link_target);
|
|
|
|
|
|
let mut next = current.trim_end_matches('\n').to_string();
|
|
|
|
|
|
if next.is_empty() {
|
|
|
|
|
|
next.push_str(&inserted_markdown);
|
|
|
|
|
|
next.push('\n');
|
|
|
|
|
|
} else {
|
|
|
|
|
|
next.push_str("\n\n");
|
|
|
|
|
|
next.push_str(&inserted_markdown);
|
|
|
|
|
|
next.push('\n');
|
|
|
|
|
|
}
|
|
|
|
|
|
fs::write(&owner_path, next).map_err(|error| {
|
|
|
|
|
|
WebError::bad_request_code(
|
|
|
|
|
|
"local_ocr_owner_write_failed",
|
|
|
|
|
|
format!(
|
|
|
|
|
|
"无法写入 OCR owner Markdown {}: {error}",
|
|
|
|
|
|
owner_path.display()
|
|
|
|
|
|
),
|
|
|
|
|
|
)
|
|
|
|
|
|
.with_context(&context)
|
|
|
|
|
|
})?;
|
|
|
|
|
|
Ok(ok_json(
|
|
|
|
|
|
&context,
|
|
|
|
|
|
json!({
|
|
|
|
|
|
"ok": true,
|
|
|
|
|
|
"documentId": body.document_id,
|
|
|
|
|
|
"ownerDocumentPath": owner_document_path,
|
|
|
|
|
|
"ocrRootRelativePath": ocr_root_relative_path,
|
|
|
|
|
|
"insertedMarkdown": inserted_markdown,
|
|
|
|
|
|
}),
|
|
|
|
|
|
))
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
pub(crate) fn is_local_ocr_sidecar_relative_path(relative_path: &str) -> bool {
|
|
|
|
|
|
let normalized = relative_path.trim().replace('\\', "/");
|
|
|
|
|
|
let path = Path::new(&normalized);
|
|
|
|
|
|
let Some(file_name) = path.file_name().and_then(|value| value.to_str()) else {
|
|
|
|
|
|
return false;
|
|
|
|
|
|
};
|
|
|
|
|
|
let Some(parent_name) = path
|
|
|
|
|
|
.parent()
|
|
|
|
|
|
.and_then(|value| value.file_name())
|
|
|
|
|
|
.and_then(|value| value.to_str())
|
|
|
|
|
|
else {
|
|
|
|
|
|
return false;
|
|
|
|
|
|
};
|
|
|
|
|
|
parent_name.ends_with(".ocr") && file_name.ends_with(".ocr.md")
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
pub(crate) fn parse_ocr_frontmatter(markdown: &str) -> Option<OcrFrontmatter> {
|
|
|
|
|
|
let trimmed = markdown.strip_prefix("---\n")?;
|
|
|
|
|
|
let end = trimmed.find("\n---")?;
|
|
|
|
|
|
let frontmatter = &trimmed[..end];
|
|
|
|
|
|
let mut fields = HashMap::<String, String>::new();
|
|
|
|
|
|
for line in frontmatter.lines() {
|
|
|
|
|
|
let Some((key, value)) = line.split_once(':') else {
|
|
|
|
|
|
continue;
|
|
|
|
|
|
};
|
|
|
|
|
|
fields.insert(
|
|
|
|
|
|
key.trim().to_string(),
|
|
|
|
|
|
value.trim().trim_matches('"').to_string(),
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
if fields.get("mnote_ocr_version").map(String::as_str) != Some("1") {
|
|
|
|
|
|
return None;
|
|
|
|
|
|
}
|
|
|
|
|
|
Some(OcrFrontmatter {
|
|
|
|
|
|
provider: fields.get("provider")?.to_string(),
|
|
|
|
|
|
owner_document: fields.get("owner_document")?.to_string(),
|
|
|
|
|
|
source_path: fields.get("source_path")?.to_string(),
|
|
|
|
|
|
source_root_relative_path: fields.get("source_root_relative_path")?.to_string(),
|
|
|
|
|
|
source_size: fields.get("source_size")?.parse().ok()?,
|
|
|
|
|
|
source_mtime_ms: fields.get("source_mtime_ms")?.parse().ok()?,
|
|
|
|
|
|
status: fields.get("status")?.to_string(),
|
|
|
|
|
|
})
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[cfg(test)]
|
|
|
|
|
|
fn read_ocr_index_entry(
|
|
|
|
|
|
root: &Path,
|
|
|
|
|
|
source_root_relative_path: &str,
|
|
|
|
|
|
) -> Result<Option<OcrIndexEntry>, WebError> {
|
|
|
|
|
|
let source = normalize_relative_path(source_root_relative_path)?;
|
|
|
|
|
|
Ok(read_ocr_index(root)?.entries.remove(&source))
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
pub(crate) fn ocr_index_entries(root: &Path) -> Result<Vec<OcrIndexEntry>, WebError> {
|
|
|
|
|
|
Ok(read_ocr_index(root)?.entries.into_values().collect())
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
pub(crate) fn strip_ocr_frontmatter(markdown: &str) -> &str {
|
|
|
|
|
|
let Some(trimmed) = markdown.strip_prefix("---\n") else {
|
|
|
|
|
|
return markdown;
|
|
|
|
|
|
};
|
|
|
|
|
|
let Some(end) = trimmed.find("\n---") else {
|
|
|
|
|
|
return markdown;
|
|
|
|
|
|
};
|
|
|
|
|
|
trimmed[end + 4..].trim_start_matches('\n')
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-06-01 10:07:42 +08:00
|
|
|
|
async fn run_mineru_ocr(
|
2026-06-01 10:30:42 +08:00
|
|
|
|
state: &AppState,
|
2026-06-01 10:07:42 +08:00
|
|
|
|
root: &Path,
|
2026-06-01 10:30:42 +08:00
|
|
|
|
root_uri: &str,
|
2026-06-01 10:07:42 +08:00
|
|
|
|
plan: &OcrSidecarPlan,
|
|
|
|
|
|
token: String,
|
2026-06-01 10:30:42 +08:00
|
|
|
|
mut entry: OcrIndexEntry,
|
2026-06-01 10:07:42 +08:00
|
|
|
|
) -> Result<String, WebError> {
|
2026-06-01 09:29:12 +08:00
|
|
|
|
let config = MineruClientConfig {
|
|
|
|
|
|
api_base_url: mineru_api_base_url(),
|
|
|
|
|
|
token,
|
|
|
|
|
|
model_version: plan.model_version.clone(),
|
|
|
|
|
|
poll_interval: mineru_poll_interval(),
|
|
|
|
|
|
max_polls: mineru_max_polls(),
|
|
|
|
|
|
};
|
|
|
|
|
|
let source_path = root.join(&plan.source_root_relative_path);
|
|
|
|
|
|
let bytes = fs::read(&source_path).map_err(|error| {
|
|
|
|
|
|
WebError::bad_request_code(
|
|
|
|
|
|
"mineru_source_read_failed",
|
|
|
|
|
|
format!(
|
|
|
|
|
|
"无法读取 MinerU OCR 来源文件 {}: {error}",
|
|
|
|
|
|
source_path.display()
|
|
|
|
|
|
),
|
|
|
|
|
|
)
|
|
|
|
|
|
})?;
|
|
|
|
|
|
let file_name = Path::new(&plan.source_root_relative_path)
|
|
|
|
|
|
.file_name()
|
|
|
|
|
|
.and_then(|value| value.to_str())
|
|
|
|
|
|
.unwrap_or("source.pdf");
|
|
|
|
|
|
let client = reqwest::Client::new();
|
|
|
|
|
|
let upload = create_mineru_upload_task(&client, &config, file_name).await?;
|
|
|
|
|
|
upload_mineru_source(&client, &upload.upload_url, bytes).await?;
|
2026-06-01 10:30:42 +08:00
|
|
|
|
entry = advance_ocr_entry(&entry, "mineru_processing", now_ms(), "", None);
|
|
|
|
|
|
upsert_and_broadcast_ocr_index_entry(state, root, root_uri, entry.clone())?;
|
2026-06-01 09:29:12 +08:00
|
|
|
|
let zip_url = poll_mineru_result_zip_url(&client, &config, &upload.batch_id).await?;
|
2026-06-01 10:30:42 +08:00
|
|
|
|
entry = advance_ocr_entry(&entry, "downloading", now_ms(), "", None);
|
|
|
|
|
|
upsert_and_broadcast_ocr_index_entry(state, root, root_uri, entry)?;
|
2026-06-01 09:29:12 +08:00
|
|
|
|
let zip_bytes = download_mineru_result_zip(&client, &zip_url).await?;
|
2026-06-02 17:17:49 +08:00
|
|
|
|
let extraction = extract_mineru_markdown_and_assets_from_zip(&zip_bytes)?;
|
|
|
|
|
|
write_mineru_zip_assets(plan, &extraction.assets)?;
|
2026-06-04 18:51:16 +08:00
|
|
|
|
if let Some(source_map_input) = extraction.source_map_input.as_ref() {
|
|
|
|
|
|
let source_map = build_mineru_source_map(plan, source_map_input);
|
|
|
|
|
|
write_source_map_sidecar(plan, &source_map)?;
|
|
|
|
|
|
}
|
2026-06-02 17:17:49 +08:00
|
|
|
|
Ok(extraction.markdown)
|
2026-06-01 09:29:12 +08:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[derive(Debug)]
|
|
|
|
|
|
struct MineruUploadTask {
|
|
|
|
|
|
batch_id: String,
|
|
|
|
|
|
upload_url: String,
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
async fn create_mineru_upload_task(
|
|
|
|
|
|
client: &reqwest::Client,
|
|
|
|
|
|
config: &MineruClientConfig,
|
|
|
|
|
|
file_name: &str,
|
|
|
|
|
|
) -> Result<MineruUploadTask, WebError> {
|
|
|
|
|
|
let url = format!(
|
|
|
|
|
|
"{}/api/v4/file-urls/batch",
|
|
|
|
|
|
config.api_base_url.trim_end_matches('/')
|
|
|
|
|
|
);
|
|
|
|
|
|
let payload = json!({
|
|
|
|
|
|
"files": [{ "name": file_name, "data_id": short_hash(file_name) }],
|
|
|
|
|
|
"model_version": config.model_version,
|
|
|
|
|
|
});
|
|
|
|
|
|
let response = client
|
|
|
|
|
|
.post(&url)
|
|
|
|
|
|
.bearer_auth(&config.token)
|
|
|
|
|
|
.json(&payload)
|
|
|
|
|
|
.send()
|
|
|
|
|
|
.await
|
|
|
|
|
|
.map_err(|error| {
|
|
|
|
|
|
WebError::bad_gateway_code(
|
|
|
|
|
|
"mineru_upload_task_failed",
|
2026-06-01 10:07:42 +08:00
|
|
|
|
format!(
|
|
|
|
|
|
"MinerU 上传任务创建失败: {}",
|
|
|
|
|
|
redact_error(&error.to_string())
|
|
|
|
|
|
),
|
2026-06-01 09:29:12 +08:00
|
|
|
|
)
|
|
|
|
|
|
})?;
|
|
|
|
|
|
let status = response.status();
|
|
|
|
|
|
let body = response.text().await.unwrap_or_default();
|
|
|
|
|
|
if !status.is_success() {
|
|
|
|
|
|
return Err(WebError::bad_gateway_code(
|
|
|
|
|
|
"mineru_upload_task_http_failed",
|
|
|
|
|
|
format!("MinerU 上传任务创建失败: HTTP {}", status.as_u16()),
|
|
|
|
|
|
));
|
|
|
|
|
|
}
|
|
|
|
|
|
let value = parse_mineru_json(&body, "mineru_upload_task_json_invalid")?;
|
2026-06-01 10:07:42 +08:00
|
|
|
|
let batch_id =
|
|
|
|
|
|
find_json_string_by_keys(&value, &["batch_id", "batchId", "id"]).ok_or_else(|| {
|
2026-06-01 09:29:12 +08:00
|
|
|
|
WebError::bad_gateway_code(
|
|
|
|
|
|
"mineru_upload_task_batch_missing",
|
|
|
|
|
|
"MinerU 上传任务响应缺少 batch_id",
|
|
|
|
|
|
)
|
|
|
|
|
|
})?;
|
|
|
|
|
|
let upload_url = find_upload_url(&value).ok_or_else(|| {
|
|
|
|
|
|
WebError::bad_gateway_code(
|
|
|
|
|
|
"mineru_upload_url_missing",
|
|
|
|
|
|
"MinerU 上传任务响应缺少 upload_url",
|
|
|
|
|
|
)
|
|
|
|
|
|
})?;
|
|
|
|
|
|
Ok(MineruUploadTask {
|
|
|
|
|
|
batch_id,
|
|
|
|
|
|
upload_url,
|
|
|
|
|
|
})
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
async fn upload_mineru_source(
|
|
|
|
|
|
client: &reqwest::Client,
|
|
|
|
|
|
upload_url: &str,
|
|
|
|
|
|
bytes: Vec<u8>,
|
|
|
|
|
|
) -> Result<(), WebError> {
|
|
|
|
|
|
let response = client
|
|
|
|
|
|
.put(upload_url)
|
|
|
|
|
|
.body(bytes)
|
|
|
|
|
|
.send()
|
|
|
|
|
|
.await
|
|
|
|
|
|
.map_err(|error| {
|
|
|
|
|
|
WebError::bad_gateway_code(
|
|
|
|
|
|
"mineru_source_upload_failed",
|
2026-06-01 10:07:42 +08:00
|
|
|
|
format!(
|
|
|
|
|
|
"MinerU 源文件上传失败: {}",
|
|
|
|
|
|
redact_error(&error.to_string())
|
|
|
|
|
|
),
|
2026-06-01 09:29:12 +08:00
|
|
|
|
)
|
|
|
|
|
|
})?;
|
|
|
|
|
|
if !response.status().is_success() {
|
|
|
|
|
|
return Err(WebError::bad_gateway_code(
|
|
|
|
|
|
"mineru_source_upload_http_failed",
|
|
|
|
|
|
format!("MinerU 源文件上传失败: HTTP {}", response.status().as_u16()),
|
|
|
|
|
|
));
|
|
|
|
|
|
}
|
|
|
|
|
|
Ok(())
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
async fn poll_mineru_result_zip_url(
|
|
|
|
|
|
client: &reqwest::Client,
|
|
|
|
|
|
config: &MineruClientConfig,
|
|
|
|
|
|
batch_id: &str,
|
|
|
|
|
|
) -> Result<String, WebError> {
|
|
|
|
|
|
let url = format!(
|
|
|
|
|
|
"{}/api/v4/extract-results/batch/{}",
|
|
|
|
|
|
config.api_base_url.trim_end_matches('/'),
|
|
|
|
|
|
batch_id
|
|
|
|
|
|
);
|
|
|
|
|
|
for _ in 0..config.max_polls.max(1) {
|
|
|
|
|
|
let response = client
|
|
|
|
|
|
.get(&url)
|
|
|
|
|
|
.bearer_auth(&config.token)
|
|
|
|
|
|
.send()
|
|
|
|
|
|
.await
|
|
|
|
|
|
.map_err(|error| {
|
|
|
|
|
|
WebError::bad_gateway_code(
|
|
|
|
|
|
"mineru_result_poll_failed",
|
|
|
|
|
|
format!("MinerU 结果轮询失败: {}", redact_error(&error.to_string())),
|
|
|
|
|
|
)
|
|
|
|
|
|
})?;
|
|
|
|
|
|
let status = response.status();
|
|
|
|
|
|
let body = response.text().await.unwrap_or_default();
|
|
|
|
|
|
if !status.is_success() {
|
|
|
|
|
|
return Err(WebError::bad_gateway_code(
|
|
|
|
|
|
"mineru_result_poll_http_failed",
|
|
|
|
|
|
format!("MinerU 结果轮询失败: HTTP {}", status.as_u16()),
|
|
|
|
|
|
));
|
|
|
|
|
|
}
|
|
|
|
|
|
let value = parse_mineru_json(&body, "mineru_result_poll_json_invalid")?;
|
|
|
|
|
|
if let Some(error_message) = mineru_result_error(&value) {
|
|
|
|
|
|
return Err(WebError::bad_gateway_code(
|
|
|
|
|
|
"mineru_result_failed",
|
|
|
|
|
|
format!("MinerU 识别失败: {}", redact_error(&error_message)),
|
|
|
|
|
|
));
|
|
|
|
|
|
}
|
|
|
|
|
|
if let Some(zip_url) = find_json_string_by_keys(
|
|
|
|
|
|
&value,
|
2026-06-01 10:07:42 +08:00
|
|
|
|
&[
|
|
|
|
|
|
"full_zip_url",
|
|
|
|
|
|
"fullZipUrl",
|
|
|
|
|
|
"zip_url",
|
|
|
|
|
|
"zipUrl",
|
|
|
|
|
|
"result_url",
|
|
|
|
|
|
"resultUrl",
|
|
|
|
|
|
],
|
2026-06-01 09:29:12 +08:00
|
|
|
|
) {
|
|
|
|
|
|
return Ok(zip_url);
|
|
|
|
|
|
}
|
|
|
|
|
|
tokio::time::sleep(config.poll_interval).await;
|
|
|
|
|
|
}
|
|
|
|
|
|
Err(WebError::gateway_timeout_code(
|
|
|
|
|
|
"mineru_result_poll_timeout",
|
|
|
|
|
|
"MinerU 识别结果轮询超时",
|
|
|
|
|
|
))
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
async fn download_mineru_result_zip(
|
|
|
|
|
|
client: &reqwest::Client,
|
|
|
|
|
|
zip_url: &str,
|
|
|
|
|
|
) -> Result<Vec<u8>, WebError> {
|
|
|
|
|
|
let response = client.get(zip_url).send().await.map_err(|error| {
|
|
|
|
|
|
WebError::bad_gateway_code(
|
|
|
|
|
|
"mineru_result_download_failed",
|
2026-06-01 10:07:42 +08:00
|
|
|
|
format!(
|
|
|
|
|
|
"MinerU 结果包下载失败: {}",
|
|
|
|
|
|
redact_error(&error.to_string())
|
|
|
|
|
|
),
|
2026-06-01 09:29:12 +08:00
|
|
|
|
)
|
|
|
|
|
|
})?;
|
|
|
|
|
|
if !response.status().is_success() {
|
|
|
|
|
|
return Err(WebError::bad_gateway_code(
|
|
|
|
|
|
"mineru_result_download_http_failed",
|
|
|
|
|
|
format!("MinerU 结果包下载失败: HTTP {}", response.status().as_u16()),
|
|
|
|
|
|
));
|
|
|
|
|
|
}
|
2026-06-01 10:07:42 +08:00
|
|
|
|
response
|
|
|
|
|
|
.bytes()
|
|
|
|
|
|
.await
|
|
|
|
|
|
.map(|bytes| bytes.to_vec())
|
|
|
|
|
|
.map_err(|error| {
|
|
|
|
|
|
WebError::bad_gateway_code(
|
|
|
|
|
|
"mineru_result_download_failed",
|
|
|
|
|
|
format!(
|
|
|
|
|
|
"MinerU 结果包读取失败: {}",
|
|
|
|
|
|
redact_error(&error.to_string())
|
|
|
|
|
|
),
|
|
|
|
|
|
)
|
|
|
|
|
|
})
|
2026-06-01 09:29:12 +08:00
|
|
|
|
}
|
|
|
|
|
|
|
2026-06-02 17:17:49 +08:00
|
|
|
|
fn extract_mineru_markdown_and_assets_from_zip(
|
|
|
|
|
|
bytes: &[u8],
|
|
|
|
|
|
) -> Result<MineruZipExtraction, WebError> {
|
2026-06-01 09:29:12 +08:00
|
|
|
|
let cursor = Cursor::new(bytes);
|
|
|
|
|
|
let mut archive = zip::ZipArchive::new(cursor).map_err(|error| {
|
|
|
|
|
|
WebError::bad_gateway_code(
|
|
|
|
|
|
"mineru_result_zip_invalid",
|
|
|
|
|
|
format!("MinerU 结果包不是合法 zip: {error}"),
|
|
|
|
|
|
)
|
|
|
|
|
|
})?;
|
|
|
|
|
|
let mut candidates = Vec::<(String, String)>::new();
|
2026-06-04 18:51:16 +08:00
|
|
|
|
let mut json_candidates = Vec::<(String, Value)>::new();
|
2026-06-02 17:17:49 +08:00
|
|
|
|
let mut assets = Vec::<MineruZipAsset>::new();
|
2026-06-01 09:29:12 +08:00
|
|
|
|
for index in 0..archive.len() {
|
|
|
|
|
|
let mut file = archive.by_index(index).map_err(|error| {
|
|
|
|
|
|
WebError::bad_gateway_code(
|
|
|
|
|
|
"mineru_result_zip_read_failed",
|
|
|
|
|
|
format!("MinerU 结果包读取失败: {error}"),
|
|
|
|
|
|
)
|
|
|
|
|
|
})?;
|
|
|
|
|
|
let name = file.name().replace('\\', "/");
|
2026-06-02 17:17:49 +08:00
|
|
|
|
if file.is_dir() || name.contains("/.") {
|
2026-06-01 09:29:12 +08:00
|
|
|
|
continue;
|
|
|
|
|
|
}
|
2026-06-02 17:17:49 +08:00
|
|
|
|
if name.to_ascii_lowercase().ends_with(".md") {
|
|
|
|
|
|
let mut markdown = String::new();
|
|
|
|
|
|
file.read_to_string(&mut markdown).map_err(|error| {
|
|
|
|
|
|
WebError::bad_gateway_code(
|
|
|
|
|
|
"mineru_result_markdown_read_failed",
|
|
|
|
|
|
format!("MinerU Markdown 读取失败: {error}"),
|
|
|
|
|
|
)
|
|
|
|
|
|
})?;
|
|
|
|
|
|
candidates.push((name, markdown));
|
|
|
|
|
|
continue;
|
|
|
|
|
|
}
|
2026-06-04 18:51:16 +08:00
|
|
|
|
if name.to_ascii_lowercase().ends_with(".json") {
|
|
|
|
|
|
let mut json_text = String::new();
|
|
|
|
|
|
file.read_to_string(&mut json_text).map_err(|error| {
|
|
|
|
|
|
WebError::bad_gateway_code(
|
|
|
|
|
|
"mineru_result_json_read_failed",
|
|
|
|
|
|
format!("MinerU JSON 读取失败: {error}"),
|
|
|
|
|
|
)
|
|
|
|
|
|
})?;
|
|
|
|
|
|
if let Ok(value) = serde_json::from_str::<Value>(&json_text) {
|
|
|
|
|
|
json_candidates.push((name, value));
|
|
|
|
|
|
}
|
|
|
|
|
|
continue;
|
|
|
|
|
|
}
|
2026-06-02 17:17:49 +08:00
|
|
|
|
let Some(relative_path) = safe_mineru_asset_relative_path(&name) else {
|
|
|
|
|
|
continue;
|
|
|
|
|
|
};
|
|
|
|
|
|
let mut asset_bytes = Vec::new();
|
|
|
|
|
|
file.read_to_end(&mut asset_bytes).map_err(|error| {
|
2026-06-01 09:29:12 +08:00
|
|
|
|
WebError::bad_gateway_code(
|
2026-06-02 17:17:49 +08:00
|
|
|
|
"mineru_result_asset_read_failed",
|
|
|
|
|
|
format!("MinerU 资源读取失败: {error}"),
|
2026-06-01 09:29:12 +08:00
|
|
|
|
)
|
|
|
|
|
|
})?;
|
2026-06-02 17:17:49 +08:00
|
|
|
|
assets.push(MineruZipAsset {
|
|
|
|
|
|
relative_path,
|
|
|
|
|
|
bytes: asset_bytes,
|
|
|
|
|
|
});
|
2026-06-01 09:29:12 +08:00
|
|
|
|
}
|
2026-06-02 17:17:49 +08:00
|
|
|
|
let markdown = candidates
|
2026-06-01 09:29:12 +08:00
|
|
|
|
.into_iter()
|
|
|
|
|
|
.max_by_key(|(name, markdown)| {
|
2026-06-01 10:07:42 +08:00
|
|
|
|
let preferred =
|
|
|
|
|
|
name.ends_with("/full.md") || name == "full.md" || name.ends_with("/result.md");
|
2026-06-01 09:29:12 +08:00
|
|
|
|
(preferred, markdown.len())
|
|
|
|
|
|
})
|
|
|
|
|
|
.map(|(_, markdown)| markdown)
|
|
|
|
|
|
.filter(|markdown| !markdown.trim().is_empty())
|
|
|
|
|
|
.ok_or_else(|| {
|
|
|
|
|
|
WebError::bad_gateway_code(
|
|
|
|
|
|
"mineru_result_markdown_missing",
|
|
|
|
|
|
"MinerU 结果包中缺少 Markdown 文件",
|
|
|
|
|
|
)
|
2026-06-02 17:17:49 +08:00
|
|
|
|
})?;
|
2026-06-04 18:51:16 +08:00
|
|
|
|
let source_map_input = pick_mineru_source_map_input(json_candidates);
|
|
|
|
|
|
Ok(MineruZipExtraction {
|
|
|
|
|
|
markdown,
|
|
|
|
|
|
assets,
|
|
|
|
|
|
source_map_input,
|
|
|
|
|
|
})
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn pick_mineru_source_map_input(candidates: Vec<(String, Value)>) -> Option<Value> {
|
|
|
|
|
|
candidates
|
|
|
|
|
|
.into_iter()
|
|
|
|
|
|
.max_by_key(|(name, value)| {
|
|
|
|
|
|
let lower = name.to_ascii_lowercase();
|
|
|
|
|
|
let preferred = lower.ends_with("content_list.json")
|
|
|
|
|
|
|| lower.ends_with("_content_list.json")
|
|
|
|
|
|
|| lower.ends_with("middle.json");
|
|
|
|
|
|
let item_count = mineru_content_items(value)
|
|
|
|
|
|
.map(|items| items.len())
|
|
|
|
|
|
.unwrap_or_default();
|
|
|
|
|
|
(preferred, item_count)
|
|
|
|
|
|
})
|
|
|
|
|
|
.map(|(_, value)| value)
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn build_mineru_source_map(plan: &OcrSidecarPlan, value: &Value) -> ResourceSourceMap {
|
|
|
|
|
|
let mut pages = BTreeMap::<u32, SourceMapPage>::new();
|
|
|
|
|
|
let mut sections = Vec::new();
|
|
|
|
|
|
let mut section_stack: Vec<String> = Vec::new();
|
|
|
|
|
|
if let Some(items) = mineru_content_items(value) {
|
|
|
|
|
|
for (index, item) in items.iter().enumerate() {
|
|
|
|
|
|
let page = mineru_page_number(item).unwrap_or(1);
|
|
|
|
|
|
let text = mineru_item_text(item).unwrap_or_default();
|
|
|
|
|
|
if text.trim().is_empty() {
|
|
|
|
|
|
continue;
|
|
|
|
|
|
}
|
|
|
|
|
|
let bbox = mineru_item_bbox(item);
|
|
|
|
|
|
let block_id = format!("p{page}_b{}", index + 1);
|
|
|
|
|
|
let text_item_id = format!("p{page}_t{}", index + 1);
|
|
|
|
|
|
let block_type = mineru_block_kind(item);
|
|
|
|
|
|
if matches!(block_type, SourceMapBlockKind::Heading) {
|
|
|
|
|
|
let level = mineru_heading_level(item).unwrap_or(1).max(1);
|
|
|
|
|
|
section_stack.truncate(level.saturating_sub(1));
|
|
|
|
|
|
section_stack.push(text.clone());
|
|
|
|
|
|
sections.push(SourceMapSection {
|
|
|
|
|
|
id: format!(
|
|
|
|
|
|
"sec_{}",
|
|
|
|
|
|
short_hash(&format!(
|
|
|
|
|
|
"{}:{}:{}",
|
|
|
|
|
|
plan.source_root_relative_path,
|
|
|
|
|
|
page,
|
|
|
|
|
|
section_stack.join("/")
|
|
|
|
|
|
))
|
|
|
|
|
|
),
|
|
|
|
|
|
title: text.clone(),
|
|
|
|
|
|
path: section_stack.clone(),
|
|
|
|
|
|
page_start: Some(page),
|
|
|
|
|
|
page_end: Some(page),
|
|
|
|
|
|
block_ids: vec![block_id.clone()],
|
|
|
|
|
|
});
|
|
|
|
|
|
} else if let Some(section) = sections.last_mut() {
|
|
|
|
|
|
section.page_end = Some(page);
|
|
|
|
|
|
if !section.block_ids.iter().any(|id| id == &block_id) {
|
|
|
|
|
|
section.block_ids.push(block_id.clone());
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
let page_entry = pages.entry(page).or_insert_with(|| SourceMapPage {
|
|
|
|
|
|
page,
|
|
|
|
|
|
width: None,
|
|
|
|
|
|
height: None,
|
|
|
|
|
|
text_items: Vec::new(),
|
|
|
|
|
|
blocks: Vec::new(),
|
|
|
|
|
|
});
|
|
|
|
|
|
page_entry.text_items.push(SourceMapTextItem {
|
|
|
|
|
|
id: text_item_id,
|
|
|
|
|
|
text: text.clone(),
|
|
|
|
|
|
bbox: bbox.clone(),
|
|
|
|
|
|
char_range: Some(EvidenceRange {
|
|
|
|
|
|
start: 0,
|
|
|
|
|
|
end: text.chars().count() as u64,
|
|
|
|
|
|
}),
|
|
|
|
|
|
});
|
|
|
|
|
|
page_entry.blocks.push(SourceMapBlock {
|
|
|
|
|
|
id: block_id,
|
|
|
|
|
|
block_type,
|
|
|
|
|
|
text: text.clone(),
|
|
|
|
|
|
bbox,
|
|
|
|
|
|
char_range: Some(EvidenceRange {
|
|
|
|
|
|
start: 0,
|
|
|
|
|
|
end: text.chars().count() as u64,
|
|
|
|
|
|
}),
|
|
|
|
|
|
});
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
ResourceSourceMap {
|
|
|
|
|
|
schema: RESOURCE_SOURCE_MAP_SCHEMA.into(),
|
|
|
|
|
|
provider: plan.provider.clone(),
|
|
|
|
|
|
model_version: Some(plan.model_version.clone()),
|
|
|
|
|
|
owner_document_path: plan.owner_document_path.clone(),
|
|
|
|
|
|
source_root_relative_path: plan.source_root_relative_path.clone(),
|
|
|
|
|
|
source_hash: format!("size:{}:mtime:{}", plan.source_size, plan.source_mtime_ms),
|
|
|
|
|
|
page_count: pages.keys().max().copied(),
|
|
|
|
|
|
pages: pages.into_values().collect(),
|
|
|
|
|
|
sections,
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn write_source_map_sidecar(
|
|
|
|
|
|
plan: &OcrSidecarPlan,
|
|
|
|
|
|
source_map: &ResourceSourceMap,
|
|
|
|
|
|
) -> Result<(), WebError> {
|
|
|
|
|
|
let source_map_path = source_map_path_for_ocr_plan(plan);
|
|
|
|
|
|
if let Some(parent) = source_map_path.parent() {
|
|
|
|
|
|
fs::create_dir_all(parent).map_err(|error| {
|
|
|
|
|
|
WebError::bad_request_code(
|
|
|
|
|
|
"local_source_map_create_failed",
|
|
|
|
|
|
format!("无法创建 source-map 目录 {}: {error}", parent.display()),
|
|
|
|
|
|
)
|
|
|
|
|
|
})?;
|
|
|
|
|
|
}
|
|
|
|
|
|
let content = serde_json::to_string_pretty(source_map).map_err(|error| {
|
|
|
|
|
|
WebError::bad_request_code(
|
|
|
|
|
|
"local_source_map_serialize_failed",
|
|
|
|
|
|
format!("source-map 序列化失败: {error}"),
|
|
|
|
|
|
)
|
|
|
|
|
|
})?;
|
|
|
|
|
|
fs::write(&source_map_path, content).map_err(|error| {
|
|
|
|
|
|
WebError::bad_request_code(
|
|
|
|
|
|
"local_source_map_write_failed",
|
|
|
|
|
|
format!("无法写入 source-map {}: {error}", source_map_path.display()),
|
|
|
|
|
|
)
|
|
|
|
|
|
})
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn source_map_path_for_ocr_plan(plan: &OcrSidecarPlan) -> PathBuf {
|
|
|
|
|
|
let source_map_relative = source_map_sidecar_relative_path(&plan.ocr_root_relative_path);
|
|
|
|
|
|
let file_name = Path::new(&source_map_relative)
|
|
|
|
|
|
.file_name()
|
|
|
|
|
|
.unwrap_or_else(|| std::ffi::OsStr::new("source.source-map.json"));
|
|
|
|
|
|
plan.ocr_path
|
|
|
|
|
|
.parent()
|
|
|
|
|
|
.unwrap_or_else(|| Path::new(""))
|
|
|
|
|
|
.join(file_name)
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn parse_sidecar_relative_path(ocr_root_relative_path: &str) -> String {
|
|
|
|
|
|
ocr_root_relative_path
|
|
|
|
|
|
.strip_suffix(".ocr.md")
|
|
|
|
|
|
.map(|prefix| format!("{prefix}.parse.md"))
|
|
|
|
|
|
.unwrap_or_else(|| format!("{ocr_root_relative_path}.parse.md"))
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn source_map_sidecar_relative_path(ocr_root_relative_path: &str) -> String {
|
|
|
|
|
|
ocr_root_relative_path
|
|
|
|
|
|
.strip_suffix(".ocr.md")
|
|
|
|
|
|
.map(|prefix| format!("{prefix}.source-map.json"))
|
|
|
|
|
|
.unwrap_or_else(|| format!("{ocr_root_relative_path}.source-map.json"))
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn mineru_content_items(value: &Value) -> Option<Vec<Value>> {
|
|
|
|
|
|
match value {
|
|
|
|
|
|
Value::Array(items) => Some(items.clone()),
|
|
|
|
|
|
Value::Object(map) => {
|
|
|
|
|
|
for key in ["content_list", "contentList", "items", "blocks", "pages"] {
|
|
|
|
|
|
if let Some(items) = map.get(key).and_then(mineru_content_items) {
|
|
|
|
|
|
return Some(items);
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
map.values().find_map(mineru_content_items)
|
|
|
|
|
|
}
|
|
|
|
|
|
_ => None,
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn mineru_item_text(value: &Value) -> Option<String> {
|
|
|
|
|
|
for key in ["text", "content", "markdown", "md"] {
|
|
|
|
|
|
if let Some(text) = value.get(key).and_then(Value::as_str) {
|
|
|
|
|
|
let trimmed = text.trim();
|
|
|
|
|
|
if !trimmed.is_empty() {
|
|
|
|
|
|
return Some(trimmed.to_string());
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
None
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn mineru_page_number(value: &Value) -> Option<u32> {
|
|
|
|
|
|
if let Some(page_idx) = value.get("page_idx").and_then(Value::as_u64) {
|
|
|
|
|
|
return u32::try_from(page_idx + 1).ok();
|
|
|
|
|
|
}
|
|
|
|
|
|
for key in ["page", "page_no", "pageNo", "page_number", "pageNumber"] {
|
|
|
|
|
|
if let Some(page) = value.get(key).and_then(Value::as_u64) {
|
|
|
|
|
|
return u32::try_from(page.max(1)).ok();
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
None
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn mineru_item_bbox(value: &Value) -> Option<EvidenceBBox> {
|
|
|
|
|
|
let bbox = value.get("bbox").and_then(Value::as_array)?;
|
|
|
|
|
|
if bbox.len() != 4 {
|
|
|
|
|
|
return None;
|
|
|
|
|
|
}
|
|
|
|
|
|
Some(EvidenceBBox {
|
|
|
|
|
|
x0: bbox[0].as_f64()?,
|
|
|
|
|
|
y0: bbox[1].as_f64()?,
|
|
|
|
|
|
x1: bbox[2].as_f64()?,
|
|
|
|
|
|
y1: bbox[3].as_f64()?,
|
|
|
|
|
|
})
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn mineru_block_kind(value: &Value) -> SourceMapBlockKind {
|
|
|
|
|
|
match value
|
|
|
|
|
|
.get("type")
|
|
|
|
|
|
.or_else(|| value.get("block_type"))
|
|
|
|
|
|
.and_then(Value::as_str)
|
|
|
|
|
|
.unwrap_or_default()
|
|
|
|
|
|
.to_ascii_lowercase()
|
|
|
|
|
|
.as_str()
|
|
|
|
|
|
{
|
|
|
|
|
|
"title" | "heading" => SourceMapBlockKind::Heading,
|
|
|
|
|
|
"table" => SourceMapBlockKind::Table,
|
|
|
|
|
|
"image" => SourceMapBlockKind::Image,
|
|
|
|
|
|
"figure" => SourceMapBlockKind::Figure,
|
|
|
|
|
|
"list" => SourceMapBlockKind::List,
|
|
|
|
|
|
"text" | "paragraph" => SourceMapBlockKind::Paragraph,
|
|
|
|
|
|
_ => SourceMapBlockKind::Text,
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn mineru_heading_level(value: &Value) -> Option<usize> {
|
|
|
|
|
|
value
|
|
|
|
|
|
.get("level")
|
|
|
|
|
|
.or_else(|| value.pointer("/props/level"))
|
|
|
|
|
|
.and_then(Value::as_u64)
|
|
|
|
|
|
.map(|value| value as usize)
|
2026-06-02 17:17:49 +08:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn safe_mineru_asset_relative_path(name: &str) -> Option<PathBuf> {
|
|
|
|
|
|
let normalized = name.trim().trim_start_matches('/').replace('\\', "/");
|
|
|
|
|
|
if normalized.is_empty() {
|
|
|
|
|
|
return None;
|
|
|
|
|
|
}
|
|
|
|
|
|
let mut path = PathBuf::new();
|
|
|
|
|
|
for component in Path::new(&normalized).components() {
|
|
|
|
|
|
match component {
|
|
|
|
|
|
Component::Normal(value) => {
|
|
|
|
|
|
let text = value.to_str()?.trim();
|
|
|
|
|
|
if text.is_empty() || text == "." || text == ".." || text.starts_with('.') {
|
|
|
|
|
|
return None;
|
|
|
|
|
|
}
|
|
|
|
|
|
path.push(text);
|
|
|
|
|
|
}
|
|
|
|
|
|
_ => return None,
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
(!path.as_os_str().is_empty()).then_some(path)
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn write_mineru_zip_assets(
|
|
|
|
|
|
plan: &OcrSidecarPlan,
|
|
|
|
|
|
assets: &[MineruZipAsset],
|
|
|
|
|
|
) -> Result<(), WebError> {
|
|
|
|
|
|
let sidecar_dir = plan.ocr_path.parent().unwrap_or_else(|| Path::new(""));
|
|
|
|
|
|
for asset in assets {
|
|
|
|
|
|
let target = sidecar_dir.join(&asset.relative_path);
|
|
|
|
|
|
ensure_target_under_root(sidecar_dir, &target, "local_ocr_asset_root_escape")?;
|
|
|
|
|
|
if let Some(parent) = target.parent() {
|
|
|
|
|
|
fs::create_dir_all(parent).map_err(|error| {
|
|
|
|
|
|
WebError::bad_request_code(
|
|
|
|
|
|
"local_ocr_asset_create_failed",
|
|
|
|
|
|
format!("无法创建 OCR 资源目录 {}: {error}", parent.display()),
|
|
|
|
|
|
)
|
|
|
|
|
|
})?;
|
|
|
|
|
|
}
|
|
|
|
|
|
fs::write(&target, &asset.bytes).map_err(|error| {
|
|
|
|
|
|
WebError::bad_request_code(
|
|
|
|
|
|
"local_ocr_asset_write_failed",
|
|
|
|
|
|
format!("无法写入 OCR 资源 {}: {error}", target.display()),
|
|
|
|
|
|
)
|
|
|
|
|
|
})?;
|
|
|
|
|
|
}
|
|
|
|
|
|
Ok(())
|
2026-06-01 09:29:12 +08:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn parse_mineru_json(body: &str, code: &'static str) -> Result<Value, WebError> {
|
|
|
|
|
|
serde_json::from_str::<Value>(body).map_err(|error| {
|
|
|
|
|
|
WebError::bad_gateway_code(code, format!("MinerU 响应 JSON 解析失败: {error}"))
|
|
|
|
|
|
})
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn find_upload_url(value: &Value) -> Option<String> {
|
|
|
|
|
|
if let Some(url) = find_json_string_by_keys(value, &["upload_url", "uploadUrl"]) {
|
|
|
|
|
|
return Some(url);
|
|
|
|
|
|
}
|
2026-06-02 17:17:49 +08:00
|
|
|
|
find_json_url_array_item_by_keys(
|
|
|
|
|
|
value,
|
|
|
|
|
|
&[
|
|
|
|
|
|
"file_urls",
|
|
|
|
|
|
"fileUrls",
|
|
|
|
|
|
"file_url",
|
|
|
|
|
|
"fileUrl",
|
|
|
|
|
|
"urls",
|
|
|
|
|
|
"upload_urls",
|
|
|
|
|
|
"uploadUrls",
|
|
|
|
|
|
],
|
|
|
|
|
|
)
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn find_json_url_array_item_by_keys(value: &Value, keys: &[&str]) -> Option<String> {
|
|
|
|
|
|
match value {
|
|
|
|
|
|
Value::Object(map) => {
|
|
|
|
|
|
for key in keys {
|
|
|
|
|
|
if let Some(found) = map.get(*key).and_then(find_first_non_empty_json_string) {
|
|
|
|
|
|
return Some(found);
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
for nested in map.values() {
|
|
|
|
|
|
if let Some(found) = find_json_url_array_item_by_keys(nested, keys) {
|
|
|
|
|
|
return Some(found);
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
None
|
|
|
|
|
|
}
|
|
|
|
|
|
Value::Array(items) => items
|
|
|
|
|
|
.iter()
|
|
|
|
|
|
.find_map(|item| find_json_url_array_item_by_keys(item, keys)),
|
|
|
|
|
|
_ => None,
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn find_first_non_empty_json_string(value: &Value) -> Option<String> {
|
|
|
|
|
|
match value {
|
|
|
|
|
|
Value::String(text) => {
|
|
|
|
|
|
let trimmed = text.trim();
|
|
|
|
|
|
(!trimmed.is_empty()).then(|| trimmed.to_string())
|
|
|
|
|
|
}
|
|
|
|
|
|
Value::Array(items) => items.iter().find_map(find_first_non_empty_json_string),
|
|
|
|
|
|
Value::Object(map) => map.values().find_map(find_first_non_empty_json_string),
|
|
|
|
|
|
_ => None,
|
|
|
|
|
|
}
|
2026-06-01 09:29:12 +08:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn find_json_string_by_keys(value: &Value, keys: &[&str]) -> Option<String> {
|
|
|
|
|
|
match value {
|
|
|
|
|
|
Value::Object(map) => {
|
|
|
|
|
|
for key in keys {
|
|
|
|
|
|
if let Some(text) = map.get(*key).and_then(Value::as_str) {
|
|
|
|
|
|
let trimmed = text.trim();
|
|
|
|
|
|
if !trimmed.is_empty() {
|
|
|
|
|
|
return Some(trimmed.to_string());
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
for nested in map.values() {
|
|
|
|
|
|
if let Some(found) = find_json_string_by_keys(nested, keys) {
|
|
|
|
|
|
return Some(found);
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
None
|
|
|
|
|
|
}
|
|
|
|
|
|
Value::Array(items) => items
|
|
|
|
|
|
.iter()
|
|
|
|
|
|
.find_map(|item| find_json_string_by_keys(item, keys)),
|
|
|
|
|
|
_ => None,
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn mineru_result_error(value: &Value) -> Option<String> {
|
|
|
|
|
|
let status = find_json_string_by_keys(value, &["state", "status"])
|
|
|
|
|
|
.unwrap_or_default()
|
|
|
|
|
|
.to_ascii_lowercase();
|
|
|
|
|
|
if matches!(
|
|
|
|
|
|
status.as_str(),
|
|
|
|
|
|
"failed" | "fail" | "error" | "interrupted" | "canceled" | "cancelled"
|
|
|
|
|
|
) {
|
|
|
|
|
|
return find_json_string_by_keys(value, &["error", "message", "msg", "err_msg", "errMsg"])
|
|
|
|
|
|
.or_else(|| Some(status));
|
|
|
|
|
|
}
|
|
|
|
|
|
None
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn ok_json(context: &RequestContext, result: Value) -> (StatusCode, HeaderMap, Json<Value>) {
|
|
|
|
|
|
let mut headers = HeaderMap::new();
|
|
|
|
|
|
if let Ok(value) = axum::http::HeaderValue::from_str(&context.trace.request_id) {
|
|
|
|
|
|
headers.insert("x-request-id", value);
|
|
|
|
|
|
}
|
|
|
|
|
|
(StatusCode::OK, headers, Json(result))
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn plan_ocr_sidecar_path(
|
|
|
|
|
|
root: &Path,
|
|
|
|
|
|
document_id: &str,
|
|
|
|
|
|
source_root_relative_path: &str,
|
|
|
|
|
|
provider: &str,
|
|
|
|
|
|
model_version: &str,
|
2026-06-04 18:51:16 +08:00
|
|
|
|
_force: bool,
|
2026-06-01 09:29:12 +08:00
|
|
|
|
) -> Result<OcrSidecarPlan, WebError> {
|
|
|
|
|
|
let owner_document_path = owner_document_path_from_id(document_id)?;
|
|
|
|
|
|
let source_root_relative_path = normalize_relative_path(source_root_relative_path)?;
|
|
|
|
|
|
let source_path = root.join(&source_root_relative_path);
|
|
|
|
|
|
ensure_target_under_root(root, &source_path, "local_ocr_source_root_escape")?;
|
|
|
|
|
|
if !source_path.is_file() {
|
|
|
|
|
|
return Err(WebError::bad_request_code(
|
|
|
|
|
|
"local_ocr_source_missing",
|
|
|
|
|
|
"OCR 来源文件不存在",
|
|
|
|
|
|
));
|
|
|
|
|
|
}
|
|
|
|
|
|
if !is_supported_ocr_source(&source_path) {
|
|
|
|
|
|
return Err(WebError::bad_request_code(
|
|
|
|
|
|
"local_ocr_source_type_unsupported",
|
|
|
|
|
|
"OCR 仅支持图片和 PDF",
|
|
|
|
|
|
));
|
|
|
|
|
|
}
|
|
|
|
|
|
let owner_path = root.join(&owner_document_path);
|
|
|
|
|
|
ensure_target_under_root(root, &owner_path, "local_ocr_owner_root_escape")?;
|
|
|
|
|
|
if !owner_path.is_file() {
|
|
|
|
|
|
return Err(WebError::bad_request_code(
|
|
|
|
|
|
"local_ocr_owner_missing",
|
|
|
|
|
|
"OCR owner Markdown 不存在",
|
|
|
|
|
|
));
|
|
|
|
|
|
}
|
|
|
|
|
|
let metadata = fs::metadata(&source_path).map_err(|error| {
|
|
|
|
|
|
WebError::bad_request_code(
|
|
|
|
|
|
"local_ocr_source_stat_failed",
|
|
|
|
|
|
format!(
|
|
|
|
|
|
"无法读取 OCR 来源文件状态 {}: {error}",
|
|
|
|
|
|
source_path.display()
|
|
|
|
|
|
),
|
|
|
|
|
|
)
|
|
|
|
|
|
})?;
|
|
|
|
|
|
let source_metadata = SourceMetadata {
|
|
|
|
|
|
size: metadata.len(),
|
|
|
|
|
|
mtime_ms: metadata
|
|
|
|
|
|
.modified()
|
|
|
|
|
|
.ok()
|
|
|
|
|
|
.and_then(|value| value.duration_since(UNIX_EPOCH).ok())
|
|
|
|
|
|
.map(|value| value.as_millis())
|
|
|
|
|
|
.unwrap_or_default(),
|
|
|
|
|
|
};
|
|
|
|
|
|
let owner_parent = Path::new(&owner_document_path)
|
|
|
|
|
|
.parent()
|
|
|
|
|
|
.map(Path::to_path_buf)
|
|
|
|
|
|
.unwrap_or_default();
|
|
|
|
|
|
let owner_stem = Path::new(&owner_document_path)
|
|
|
|
|
|
.file_stem()
|
|
|
|
|
|
.and_then(|value| value.to_str())
|
|
|
|
|
|
.unwrap_or("Page");
|
|
|
|
|
|
let ocr_dir = owner_parent.join(format!("{owner_stem}.ocr"));
|
|
|
|
|
|
let source_leaf = Path::new(&source_root_relative_path)
|
|
|
|
|
|
.file_name()
|
|
|
|
|
|
.and_then(|value| value.to_str())
|
|
|
|
|
|
.unwrap_or("source");
|
|
|
|
|
|
let base_file_name = format!("{source_leaf}.ocr.md");
|
2026-06-04 18:51:16 +08:00
|
|
|
|
let ocr_relative = ocr_dir.join(&base_file_name);
|
2026-06-01 09:29:12 +08:00
|
|
|
|
let ocr_root_relative_path = ocr_relative.to_string_lossy().replace('\\', "/");
|
|
|
|
|
|
let ocr_path = root.join(&ocr_root_relative_path);
|
|
|
|
|
|
ensure_target_under_root(root, &ocr_path, "local_ocr_sidecar_root_escape")?;
|
|
|
|
|
|
Ok(OcrSidecarPlan {
|
|
|
|
|
|
owner_document_path,
|
|
|
|
|
|
source_root_relative_path,
|
|
|
|
|
|
ocr_root_relative_path,
|
|
|
|
|
|
ocr_path,
|
|
|
|
|
|
provider: provider.to_string(),
|
|
|
|
|
|
model_version: model_version.to_string(),
|
|
|
|
|
|
source_size: source_metadata.size,
|
|
|
|
|
|
source_mtime_ms: source_metadata.mtime_ms,
|
|
|
|
|
|
})
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn write_ocr_sidecar(
|
|
|
|
|
|
plan: &OcrSidecarPlan,
|
|
|
|
|
|
markdown_body: &str,
|
|
|
|
|
|
now: u128,
|
|
|
|
|
|
) -> Result<(), WebError> {
|
|
|
|
|
|
if let Some(parent) = plan.ocr_path.parent() {
|
|
|
|
|
|
fs::create_dir_all(parent).map_err(|error| {
|
|
|
|
|
|
WebError::bad_request_code(
|
|
|
|
|
|
"local_ocr_sidecar_create_failed",
|
|
|
|
|
|
format!("无法创建 OCR sidecar 目录 {}: {error}", parent.display()),
|
|
|
|
|
|
)
|
|
|
|
|
|
})?;
|
|
|
|
|
|
}
|
2026-06-04 18:51:16 +08:00
|
|
|
|
let parse_relative = parse_sidecar_relative_path(&plan.ocr_root_relative_path);
|
|
|
|
|
|
let parse_file_name = Path::new(&parse_relative)
|
|
|
|
|
|
.file_name()
|
|
|
|
|
|
.unwrap_or_else(|| std::ffi::OsStr::new("source.parse.md"));
|
|
|
|
|
|
let parse_path = plan.ocr_path.with_file_name(parse_file_name);
|
|
|
|
|
|
fs::write(&parse_path, markdown_body.trim_end()).map_err(|error| {
|
|
|
|
|
|
WebError::bad_request_code(
|
|
|
|
|
|
"local_ocr_sidecar_write_failed",
|
|
|
|
|
|
format!(
|
|
|
|
|
|
"无法写入 OCR Parse Markdown {}: {error}",
|
|
|
|
|
|
parse_path.display()
|
|
|
|
|
|
),
|
|
|
|
|
|
)
|
|
|
|
|
|
})?;
|
2026-06-01 09:29:12 +08:00
|
|
|
|
let content = build_ocr_markdown(plan, markdown_body, "done", now);
|
|
|
|
|
|
fs::write(&plan.ocr_path, content).map_err(|error| {
|
|
|
|
|
|
WebError::bad_request_code(
|
|
|
|
|
|
"local_ocr_sidecar_write_failed",
|
|
|
|
|
|
format!("无法写入 OCR Markdown {}: {error}", plan.ocr_path.display()),
|
|
|
|
|
|
)
|
|
|
|
|
|
})
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn build_ocr_markdown(
|
|
|
|
|
|
plan: &OcrSidecarPlan,
|
|
|
|
|
|
markdown_body: &str,
|
|
|
|
|
|
status: &str,
|
|
|
|
|
|
timestamp_ms: u128,
|
|
|
|
|
|
) -> String {
|
|
|
|
|
|
let owner_document =
|
|
|
|
|
|
relative_from_sidecar_dir(&plan.ocr_root_relative_path, &plan.owner_document_path);
|
|
|
|
|
|
let source_path =
|
|
|
|
|
|
relative_from_owner_dir(&plan.owner_document_path, &plan.source_root_relative_path);
|
|
|
|
|
|
let timestamp = timestamp_ms.to_string();
|
|
|
|
|
|
format!(
|
|
|
|
|
|
"---\nmnote_ocr_version: 1\nprovider: {}\nmodel_version: {}\nowner_document: {}\nsource_path: {}\nsource_root_relative_path: {}\nsource_size: {}\nsource_mtime_ms: {}\nstatus: {}\ncreated_at: {}\nupdated_at: {}\n---\n\n{}\n",
|
|
|
|
|
|
plan.provider,
|
|
|
|
|
|
plan.model_version,
|
|
|
|
|
|
owner_document,
|
|
|
|
|
|
source_path,
|
|
|
|
|
|
plan.source_root_relative_path,
|
|
|
|
|
|
plan.source_size,
|
|
|
|
|
|
plan.source_mtime_ms,
|
|
|
|
|
|
status,
|
|
|
|
|
|
timestamp,
|
|
|
|
|
|
timestamp,
|
|
|
|
|
|
markdown_body.trim_end()
|
|
|
|
|
|
)
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn build_index_entry(
|
|
|
|
|
|
plan: &OcrSidecarPlan,
|
|
|
|
|
|
status: &str,
|
|
|
|
|
|
now: u128,
|
|
|
|
|
|
markdown: &str,
|
|
|
|
|
|
error: Option<String>,
|
|
|
|
|
|
) -> OcrIndexEntry {
|
|
|
|
|
|
OcrIndexEntry {
|
|
|
|
|
|
job_id: format!(
|
|
|
|
|
|
"ocr_{}_{}",
|
|
|
|
|
|
now,
|
|
|
|
|
|
short_hash(&plan.source_root_relative_path)
|
|
|
|
|
|
),
|
|
|
|
|
|
owner_document_id: format!(
|
|
|
|
|
|
"local-md:{}",
|
|
|
|
|
|
encode_local_id_segment(&plan.owner_document_path)
|
|
|
|
|
|
),
|
|
|
|
|
|
owner_document_path: plan.owner_document_path.clone(),
|
|
|
|
|
|
source_root_relative_path: plan.source_root_relative_path.clone(),
|
|
|
|
|
|
ocr_root_relative_path: plan.ocr_root_relative_path.clone(),
|
|
|
|
|
|
provider: plan.provider.clone(),
|
|
|
|
|
|
model_version: plan.model_version.clone(),
|
|
|
|
|
|
status: status.to_string(),
|
|
|
|
|
|
source_size: plan.source_size,
|
|
|
|
|
|
source_mtime_ms: plan.source_mtime_ms,
|
|
|
|
|
|
created_at_ms: now,
|
|
|
|
|
|
updated_at_ms: now,
|
|
|
|
|
|
plain_text_preview: markdown.chars().take(240).collect(),
|
|
|
|
|
|
error,
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-06-01 10:30:42 +08:00
|
|
|
|
fn advance_ocr_entry(
|
|
|
|
|
|
previous: &OcrIndexEntry,
|
|
|
|
|
|
status: &str,
|
|
|
|
|
|
now: u128,
|
|
|
|
|
|
markdown: &str,
|
|
|
|
|
|
error: Option<String>,
|
|
|
|
|
|
) -> OcrIndexEntry {
|
|
|
|
|
|
let mut next = previous.clone();
|
|
|
|
|
|
next.status = status.to_string();
|
|
|
|
|
|
next.updated_at_ms = now;
|
|
|
|
|
|
if !markdown.is_empty() {
|
|
|
|
|
|
next.plain_text_preview = markdown.chars().take(240).collect();
|
|
|
|
|
|
}
|
|
|
|
|
|
next.error = error;
|
|
|
|
|
|
next
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-06-01 09:29:12 +08:00
|
|
|
|
fn read_ocr_index(root: &Path) -> Result<OcrIndex, WebError> {
|
|
|
|
|
|
let path = ocr_index_path(root);
|
|
|
|
|
|
if !path.exists() {
|
|
|
|
|
|
return Ok(OcrIndex {
|
|
|
|
|
|
version: OCR_INDEX_VERSION,
|
|
|
|
|
|
entries: BTreeMap::new(),
|
|
|
|
|
|
});
|
|
|
|
|
|
}
|
|
|
|
|
|
let content = fs::read_to_string(&path).map_err(|error| {
|
|
|
|
|
|
WebError::bad_request_code(
|
|
|
|
|
|
"local_ocr_index_read_failed",
|
|
|
|
|
|
format!("无法读取 OCR 索引 {}: {error}", path.display()),
|
|
|
|
|
|
)
|
|
|
|
|
|
})?;
|
|
|
|
|
|
serde_json::from_str::<OcrIndex>(&content).map_err(|error| {
|
|
|
|
|
|
WebError::bad_request_code(
|
|
|
|
|
|
"local_ocr_index_invalid",
|
|
|
|
|
|
format!("OCR 索引格式非法 {}: {error}", path.display()),
|
|
|
|
|
|
)
|
|
|
|
|
|
})
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn write_ocr_index(root: &Path, index: &OcrIndex) -> Result<(), WebError> {
|
|
|
|
|
|
let path = ocr_index_path(root);
|
|
|
|
|
|
if let Some(parent) = path.parent() {
|
|
|
|
|
|
fs::create_dir_all(parent).map_err(|error| {
|
|
|
|
|
|
WebError::bad_request_code(
|
|
|
|
|
|
"local_ocr_index_create_failed",
|
|
|
|
|
|
format!("无法创建 OCR 索引目录 {}: {error}", parent.display()),
|
|
|
|
|
|
)
|
|
|
|
|
|
})?;
|
|
|
|
|
|
}
|
|
|
|
|
|
let content = serde_json::to_vec_pretty(index)
|
|
|
|
|
|
.map_err(|error| WebError::internal(format!("OCR 索引序列化失败: {error}")))?;
|
|
|
|
|
|
let tmp = path.with_extension("json.tmp");
|
|
|
|
|
|
fs::write(&tmp, [&content[..], b"\n"].concat()).map_err(|error| {
|
|
|
|
|
|
WebError::bad_request_code(
|
|
|
|
|
|
"local_ocr_index_write_failed",
|
|
|
|
|
|
format!("无法写入 OCR 临时索引 {}: {error}", tmp.display()),
|
|
|
|
|
|
)
|
|
|
|
|
|
})?;
|
|
|
|
|
|
fs::rename(&tmp, &path).map_err(|error| {
|
|
|
|
|
|
WebError::bad_request_code(
|
|
|
|
|
|
"local_ocr_index_write_failed",
|
|
|
|
|
|
format!("无法替换 OCR 索引 {}: {error}", path.display()),
|
|
|
|
|
|
)
|
|
|
|
|
|
})
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-06-04 18:51:16 +08:00
|
|
|
|
fn reusable_existing_ocr_entry(
|
|
|
|
|
|
state: &AppState,
|
|
|
|
|
|
root: &Path,
|
|
|
|
|
|
root_uri: &str,
|
|
|
|
|
|
plan: &OcrSidecarPlan,
|
|
|
|
|
|
) -> Result<Option<OcrIndexEntry>, WebError> {
|
|
|
|
|
|
let index = read_ocr_index(root)?;
|
|
|
|
|
|
if let Some(entry) = index.entries.get(&plan.source_root_relative_path) {
|
|
|
|
|
|
if entry.source_size == plan.source_size && entry.source_mtime_ms == plan.source_mtime_ms {
|
|
|
|
|
|
let status = entry.status.as_str();
|
|
|
|
|
|
if status == "done" {
|
|
|
|
|
|
let sidecar = root.join(&entry.ocr_root_relative_path);
|
|
|
|
|
|
if sidecar.is_file() && !source_is_stale(root, entry) {
|
|
|
|
|
|
return Ok(Some(entry.clone()));
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
if matches!(
|
|
|
|
|
|
status,
|
|
|
|
|
|
"queued" | "uploading" | "mineru_processing" | "downloading" | "writing_sidecar"
|
|
|
|
|
|
) {
|
|
|
|
|
|
let key = format!("{root_uri}:{}", entry.source_root_relative_path);
|
|
|
|
|
|
if state
|
|
|
|
|
|
.local_ocr_active_jobs
|
|
|
|
|
|
.read()
|
|
|
|
|
|
.map(|jobs| jobs.contains_key(&key))
|
|
|
|
|
|
.unwrap_or(false)
|
|
|
|
|
|
&& !source_is_stale(root, entry)
|
|
|
|
|
|
{
|
|
|
|
|
|
return Ok(Some(entry.clone()));
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
let default_ocr_path = root.join(&plan.ocr_root_relative_path);
|
|
|
|
|
|
if default_ocr_path.is_file() {
|
|
|
|
|
|
let markdown = fs::read_to_string(&default_ocr_path).map_err(|error| {
|
|
|
|
|
|
WebError::bad_request_code(
|
|
|
|
|
|
"local_ocr_read_failed",
|
|
|
|
|
|
format!(
|
|
|
|
|
|
"无法读取 OCR Markdown {}: {error}",
|
|
|
|
|
|
default_ocr_path.display()
|
|
|
|
|
|
),
|
|
|
|
|
|
)
|
|
|
|
|
|
})?;
|
|
|
|
|
|
if let Some(frontmatter) = parse_ocr_frontmatter(&markdown) {
|
|
|
|
|
|
if frontmatter.source_root_relative_path == plan.source_root_relative_path
|
|
|
|
|
|
&& frontmatter.source_size == plan.source_size
|
|
|
|
|
|
&& frontmatter.source_mtime_ms == plan.source_mtime_ms
|
|
|
|
|
|
&& frontmatter.status == "done"
|
|
|
|
|
|
{
|
|
|
|
|
|
let now = now_ms();
|
|
|
|
|
|
let entry = OcrIndexEntry {
|
|
|
|
|
|
job_id: format!(
|
|
|
|
|
|
"ocr_{}_{}",
|
|
|
|
|
|
now,
|
|
|
|
|
|
short_hash(&plan.source_root_relative_path)
|
|
|
|
|
|
),
|
|
|
|
|
|
owner_document_id: format!(
|
|
|
|
|
|
"local-md:{}",
|
|
|
|
|
|
encode_local_id_segment(&plan.owner_document_path)
|
|
|
|
|
|
),
|
|
|
|
|
|
owner_document_path: plan.owner_document_path.clone(),
|
|
|
|
|
|
source_root_relative_path: plan.source_root_relative_path.clone(),
|
|
|
|
|
|
ocr_root_relative_path: plan.ocr_root_relative_path.clone(),
|
|
|
|
|
|
provider: frontmatter.provider,
|
|
|
|
|
|
model_version: plan.model_version.clone(),
|
|
|
|
|
|
status: frontmatter.status,
|
|
|
|
|
|
source_size: plan.source_size,
|
|
|
|
|
|
source_mtime_ms: plan.source_mtime_ms,
|
|
|
|
|
|
created_at_ms: now,
|
|
|
|
|
|
updated_at_ms: now,
|
|
|
|
|
|
plain_text_preview: strip_ocr_frontmatter(&markdown)
|
|
|
|
|
|
.chars()
|
|
|
|
|
|
.take(240)
|
|
|
|
|
|
.collect(),
|
|
|
|
|
|
error: None,
|
|
|
|
|
|
};
|
|
|
|
|
|
upsert_ocr_index_entry(root, entry.clone())?;
|
|
|
|
|
|
return Ok(Some(entry));
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
Ok(None)
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-06-01 09:29:12 +08:00
|
|
|
|
fn upsert_ocr_index_entry(root: &Path, entry: OcrIndexEntry) -> Result<(), WebError> {
|
|
|
|
|
|
let mut index = read_ocr_index(root)?;
|
|
|
|
|
|
index.version = OCR_INDEX_VERSION;
|
|
|
|
|
|
index
|
|
|
|
|
|
.entries
|
|
|
|
|
|
.insert(entry.source_root_relative_path.clone(), entry);
|
|
|
|
|
|
write_ocr_index(root, &index)
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-06-01 10:30:42 +08:00
|
|
|
|
fn upsert_and_broadcast_ocr_index_entry(
|
|
|
|
|
|
state: &AppState,
|
|
|
|
|
|
root: &Path,
|
|
|
|
|
|
root_uri: &str,
|
|
|
|
|
|
entry: OcrIndexEntry,
|
|
|
|
|
|
) -> Result<(), WebError> {
|
|
|
|
|
|
upsert_ocr_index_entry(root, entry.clone())?;
|
|
|
|
|
|
broadcast_ocr_job_update(state, root, root_uri, &entry);
|
|
|
|
|
|
Ok(())
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-06-02 17:17:49 +08:00
|
|
|
|
fn cleanup_empty_ocr_sidecar_dir(root: &Path, sidecar: &Path) -> Result<(), WebError> {
|
|
|
|
|
|
let Some(parent) = sidecar.parent() else {
|
|
|
|
|
|
return Ok(());
|
|
|
|
|
|
};
|
|
|
|
|
|
ensure_target_under_root(root, parent, "local_ocr_delete_root_escape")?;
|
|
|
|
|
|
let Ok(entries) = fs::read_dir(parent) else {
|
|
|
|
|
|
return Ok(());
|
|
|
|
|
|
};
|
|
|
|
|
|
let has_other_sidecars = entries.filter_map(Result::ok).any(|entry| {
|
|
|
|
|
|
entry
|
|
|
|
|
|
.file_name()
|
|
|
|
|
|
.to_string_lossy()
|
|
|
|
|
|
.to_ascii_lowercase()
|
|
|
|
|
|
.ends_with(".ocr.md")
|
|
|
|
|
|
});
|
|
|
|
|
|
if !has_other_sidecars {
|
|
|
|
|
|
fs::remove_dir_all(parent).map_err(|error| {
|
|
|
|
|
|
WebError::bad_request_code(
|
|
|
|
|
|
"local_ocr_delete_failed",
|
|
|
|
|
|
format!("无法删除 OCR sidecar 目录 {}: {error}", parent.display()),
|
|
|
|
|
|
)
|
|
|
|
|
|
})?;
|
|
|
|
|
|
}
|
|
|
|
|
|
Ok(())
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn broadcast_ocr_job_deleted(
|
|
|
|
|
|
state: &AppState,
|
|
|
|
|
|
root_uri: &str,
|
|
|
|
|
|
source_root_relative_path: &str,
|
|
|
|
|
|
removed: Option<&OcrIndexEntry>,
|
|
|
|
|
|
) {
|
|
|
|
|
|
let now = now_ms();
|
|
|
|
|
|
let job = json!({
|
|
|
|
|
|
"jobId": removed.map(|entry| entry.job_id.as_str()).unwrap_or(""),
|
|
|
|
|
|
"ownerDocumentId": removed.map(|entry| entry.owner_document_id.as_str()).unwrap_or(""),
|
|
|
|
|
|
"ownerDocumentPath": removed.map(|entry| entry.owner_document_path.as_str()).unwrap_or(""),
|
|
|
|
|
|
"sourceRootRelativePath": source_root_relative_path,
|
|
|
|
|
|
"ocrRootRelativePath": removed.map(|entry| entry.ocr_root_relative_path.as_str()).unwrap_or(""),
|
|
|
|
|
|
"provider": removed.map(|entry| entry.provider.as_str()).unwrap_or(""),
|
|
|
|
|
|
"modelVersion": removed.map(|entry| entry.model_version.as_str()).unwrap_or(""),
|
|
|
|
|
|
"status": "deleted",
|
|
|
|
|
|
"stageLabel": "已删除",
|
|
|
|
|
|
"stale": false,
|
|
|
|
|
|
"updatedAtMs": now,
|
|
|
|
|
|
"finishedAtMs": now,
|
|
|
|
|
|
"plainTextPreview": "",
|
|
|
|
|
|
"error": null,
|
|
|
|
|
|
});
|
|
|
|
|
|
let payload = json!({
|
|
|
|
|
|
"schema": "mnote.local_ocr.job.updated.v1",
|
|
|
|
|
|
"kind": "local_ocr_job_updated",
|
|
|
|
|
|
"eventType": "local_ocr.job.updated",
|
|
|
|
|
|
"sourceKind": "local_folder",
|
|
|
|
|
|
"rootUri": root_uri,
|
|
|
|
|
|
"relativePath": source_root_relative_path,
|
|
|
|
|
|
"documentId": removed.map(|entry| entry.owner_document_id.as_str()).unwrap_or(""),
|
|
|
|
|
|
"revision": now.to_string(),
|
|
|
|
|
|
"job": job,
|
|
|
|
|
|
});
|
|
|
|
|
|
let _ = state.local_ocr_job_tx.send(payload.clone());
|
|
|
|
|
|
let _ = state.stream_delta_tx.send(payload);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-06-01 10:30:42 +08:00
|
|
|
|
fn broadcast_ocr_job_update(state: &AppState, root: &Path, root_uri: &str, entry: &OcrIndexEntry) {
|
|
|
|
|
|
let job = ocr_job_payload(root, entry);
|
|
|
|
|
|
let key = format!("{root_uri}:{}", entry.source_root_relative_path);
|
|
|
|
|
|
if matches!(entry.status.as_str(), "done" | "failed" | "interrupted") {
|
|
|
|
|
|
if let Ok(mut jobs) = state.local_ocr_active_jobs.write() {
|
|
|
|
|
|
jobs.remove(&key);
|
|
|
|
|
|
}
|
|
|
|
|
|
} else if let Ok(mut jobs) = state.local_ocr_active_jobs.write() {
|
|
|
|
|
|
jobs.insert(key, job.clone());
|
|
|
|
|
|
}
|
|
|
|
|
|
let payload = json!({
|
|
|
|
|
|
"schema": "mnote.local_ocr.job.updated.v1",
|
|
|
|
|
|
"kind": "local_ocr_job_updated",
|
|
|
|
|
|
"eventType": "local_ocr.job.updated",
|
|
|
|
|
|
"sourceKind": "local_folder",
|
|
|
|
|
|
"rootUri": root_uri,
|
|
|
|
|
|
"relativePath": entry.source_root_relative_path,
|
|
|
|
|
|
"documentId": entry.owner_document_id,
|
|
|
|
|
|
"revision": entry.updated_at_ms.to_string(),
|
|
|
|
|
|
"job": job,
|
|
|
|
|
|
});
|
|
|
|
|
|
let _ = state.local_ocr_job_tx.send(payload.clone());
|
|
|
|
|
|
let _ = state.stream_delta_tx.send(payload);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-06-01 09:29:12 +08:00
|
|
|
|
fn ocr_job_payload(root: &Path, entry: &OcrIndexEntry) -> Value {
|
|
|
|
|
|
let stale = source_is_stale(root, entry);
|
|
|
|
|
|
json!({
|
|
|
|
|
|
"jobId": entry.job_id,
|
|
|
|
|
|
"ownerDocumentId": entry.owner_document_id,
|
|
|
|
|
|
"ownerDocumentPath": entry.owner_document_path,
|
|
|
|
|
|
"sourceRootRelativePath": entry.source_root_relative_path,
|
|
|
|
|
|
"ocrRootRelativePath": entry.ocr_root_relative_path,
|
|
|
|
|
|
"provider": entry.provider,
|
|
|
|
|
|
"modelVersion": entry.model_version,
|
|
|
|
|
|
"status": if stale && entry.status == "done" { "stale" } else { entry.status.as_str() },
|
|
|
|
|
|
"stageLabel": stage_label(if stale && entry.status == "done" { "stale" } else { entry.status.as_str() }),
|
|
|
|
|
|
"stale": stale,
|
|
|
|
|
|
"updatedAtMs": entry.updated_at_ms,
|
|
|
|
|
|
"finishedAtMs": if matches!(entry.status.as_str(), "done" | "failed") { Some(entry.updated_at_ms) } else { None },
|
|
|
|
|
|
"plainTextPreview": entry.plain_text_preview,
|
|
|
|
|
|
"error": entry.error,
|
|
|
|
|
|
})
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn source_is_stale(root: &Path, entry: &OcrIndexEntry) -> bool {
|
|
|
|
|
|
let source = root.join(&entry.source_root_relative_path);
|
|
|
|
|
|
let Ok(metadata) = fs::metadata(source) else {
|
|
|
|
|
|
return true;
|
|
|
|
|
|
};
|
|
|
|
|
|
let mtime_ms = metadata
|
|
|
|
|
|
.modified()
|
|
|
|
|
|
.ok()
|
|
|
|
|
|
.and_then(|value| value.duration_since(UNIX_EPOCH).ok())
|
|
|
|
|
|
.map(|value| value.as_millis())
|
|
|
|
|
|
.unwrap_or_default();
|
|
|
|
|
|
metadata.len() != entry.source_size || mtime_ms != entry.source_mtime_ms
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn stage_label(status: &str) -> &'static str {
|
|
|
|
|
|
match status {
|
|
|
|
|
|
"queued" => "排队中",
|
|
|
|
|
|
"uploading" => "上传中",
|
|
|
|
|
|
"mineru_processing" => "识别中",
|
|
|
|
|
|
"downloading" => "下载中",
|
|
|
|
|
|
"writing_sidecar" => "写入中",
|
|
|
|
|
|
"done" => "已识别",
|
|
|
|
|
|
"failed" => "识别失败",
|
|
|
|
|
|
"interrupted" => "已中断",
|
|
|
|
|
|
"stale" => "来源已变化",
|
|
|
|
|
|
_ => "未知状态",
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn ocr_frontmatter_payload(frontmatter: OcrFrontmatter) -> Value {
|
|
|
|
|
|
json!({
|
|
|
|
|
|
"provider": frontmatter.provider,
|
|
|
|
|
|
"ownerDocument": frontmatter.owner_document,
|
|
|
|
|
|
"sourcePath": frontmatter.source_path,
|
|
|
|
|
|
"sourceRootRelativePath": frontmatter.source_root_relative_path,
|
|
|
|
|
|
"sourceSize": frontmatter.source_size,
|
|
|
|
|
|
"sourceMtimeMs": frontmatter.source_mtime_ms,
|
|
|
|
|
|
"status": frontmatter.status,
|
|
|
|
|
|
})
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn ocr_index_path(root: &Path) -> PathBuf {
|
|
|
|
|
|
root.join(".mnote").join("ocr-index.json")
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn owner_document_path_from_id(document_id: &str) -> Result<String, WebError> {
|
|
|
|
|
|
let encoded = document_id
|
|
|
|
|
|
.trim()
|
|
|
|
|
|
.strip_prefix("local-md:")
|
|
|
|
|
|
.ok_or_else(|| {
|
|
|
|
|
|
WebError::bad_request_code(
|
|
|
|
|
|
"local_ocr_owner_document_invalid",
|
|
|
|
|
|
"OCR owner documentId 必须是 local-md 路径型 ID",
|
|
|
|
|
|
)
|
|
|
|
|
|
})?;
|
|
|
|
|
|
let decoded = decode_local_id_segment(encoded)?;
|
|
|
|
|
|
normalize_relative_path(&decoded)
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn normalize_relative_path(value: &str) -> Result<String, WebError> {
|
|
|
|
|
|
let normalized = value.trim().trim_start_matches('/').replace('\\', "/");
|
|
|
|
|
|
if normalized.is_empty() {
|
|
|
|
|
|
return Err(WebError::bad_request_code(
|
|
|
|
|
|
"local_ocr_path_required",
|
|
|
|
|
|
"OCR 路径不能为空",
|
|
|
|
|
|
));
|
|
|
|
|
|
}
|
|
|
|
|
|
let path = Path::new(&normalized);
|
|
|
|
|
|
if path.is_absolute()
|
|
|
|
|
|
|| path.components().any(|component| {
|
|
|
|
|
|
matches!(
|
|
|
|
|
|
component,
|
|
|
|
|
|
Component::ParentDir | Component::RootDir | Component::Prefix(_)
|
|
|
|
|
|
)
|
|
|
|
|
|
})
|
|
|
|
|
|
{
|
|
|
|
|
|
return Err(WebError::bad_request_code(
|
|
|
|
|
|
"local_ocr_path_escape",
|
|
|
|
|
|
"OCR 相对路径不能越过 root",
|
|
|
|
|
|
));
|
|
|
|
|
|
}
|
|
|
|
|
|
Ok(normalized)
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn ensure_target_under_root(
|
|
|
|
|
|
root: &Path,
|
|
|
|
|
|
target: &Path,
|
|
|
|
|
|
code: &'static str,
|
|
|
|
|
|
) -> Result<(), WebError> {
|
|
|
|
|
|
let parent = target.parent().unwrap_or(root);
|
|
|
|
|
|
let canonical_parent = parent
|
|
|
|
|
|
.canonicalize()
|
|
|
|
|
|
.unwrap_or_else(|_| parent.to_path_buf());
|
|
|
|
|
|
let canonical_root = root.canonicalize().unwrap_or_else(|_| root.to_path_buf());
|
|
|
|
|
|
if !canonical_parent.starts_with(&canonical_root) {
|
|
|
|
|
|
return Err(WebError::bad_request_code(code, "OCR 路径不能越过 root"));
|
|
|
|
|
|
}
|
|
|
|
|
|
Ok(())
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn is_supported_ocr_source(path: &Path) -> bool {
|
|
|
|
|
|
let ext = path
|
|
|
|
|
|
.extension()
|
|
|
|
|
|
.and_then(|value| value.to_str())
|
|
|
|
|
|
.unwrap_or_default()
|
|
|
|
|
|
.to_ascii_lowercase();
|
|
|
|
|
|
matches!(
|
|
|
|
|
|
ext.as_str(),
|
|
|
|
|
|
"pdf" | "png" | "jpg" | "jpeg" | "webp" | "bmp" | "tif" | "tiff"
|
|
|
|
|
|
)
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn relative_from_owner_dir(owner_document_path: &str, source_root_relative_path: &str) -> String {
|
|
|
|
|
|
let owner_parent = Path::new(owner_document_path)
|
|
|
|
|
|
.parent()
|
|
|
|
|
|
.unwrap_or(Path::new(""));
|
|
|
|
|
|
Path::new(source_root_relative_path)
|
|
|
|
|
|
.strip_prefix(owner_parent)
|
|
|
|
|
|
.map(|value| format!("./{}", value.to_string_lossy().replace('\\', "/")))
|
|
|
|
|
|
.unwrap_or_else(|_| source_root_relative_path.to_string())
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn relative_from_sidecar_dir(ocr_root_relative_path: &str, owner_document_path: &str) -> String {
|
|
|
|
|
|
let sidecar_parent = Path::new(ocr_root_relative_path)
|
|
|
|
|
|
.parent()
|
|
|
|
|
|
.unwrap_or_else(|| Path::new(""));
|
|
|
|
|
|
let owner = Path::new(owner_document_path);
|
|
|
|
|
|
let sidecar_components = sidecar_parent
|
|
|
|
|
|
.components()
|
|
|
|
|
|
.map(|component| component.as_os_str().to_string_lossy().to_string())
|
|
|
|
|
|
.collect::<Vec<_>>();
|
|
|
|
|
|
let owner_components = owner
|
|
|
|
|
|
.components()
|
|
|
|
|
|
.map(|component| component.as_os_str().to_string_lossy().to_string())
|
|
|
|
|
|
.collect::<Vec<_>>();
|
|
|
|
|
|
let common = sidecar_components
|
|
|
|
|
|
.iter()
|
|
|
|
|
|
.zip(owner_components.iter())
|
|
|
|
|
|
.take_while(|(left, right)| left == right)
|
|
|
|
|
|
.count();
|
|
|
|
|
|
let mut parts = Vec::<String>::new();
|
|
|
|
|
|
for _ in common..sidecar_components.len() {
|
|
|
|
|
|
parts.push("..".to_string());
|
|
|
|
|
|
}
|
|
|
|
|
|
parts.extend(owner_components.into_iter().skip(common));
|
|
|
|
|
|
if parts.is_empty() {
|
|
|
|
|
|
".".to_string()
|
|
|
|
|
|
} else {
|
|
|
|
|
|
parts.join("/")
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn short_hash(value: &str) -> String {
|
|
|
|
|
|
let mut hasher = std::collections::hash_map::DefaultHasher::new();
|
|
|
|
|
|
value.hash(&mut hasher);
|
|
|
|
|
|
format!("{:x}", hasher.finish()).chars().take(6).collect()
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn now_ms() -> u128 {
|
|
|
|
|
|
SystemTime::now()
|
|
|
|
|
|
.duration_since(UNIX_EPOCH)
|
|
|
|
|
|
.map(|value| value.as_millis())
|
|
|
|
|
|
.unwrap_or_default()
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn redact_error(error: &str) -> String {
|
|
|
|
|
|
let text = error.trim().replace('\n', " ");
|
|
|
|
|
|
let lower = text.to_ascii_lowercase();
|
|
|
|
|
|
if lower.contains("token") || lower.contains("signature") || lower.contains("x-oss-") {
|
|
|
|
|
|
return "provider_error_redacted".to_string();
|
|
|
|
|
|
}
|
|
|
|
|
|
text.chars().take(160).collect()
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn ocr_link_label(ocr_root_relative_path: &str) -> String {
|
|
|
|
|
|
Path::new(ocr_root_relative_path)
|
|
|
|
|
|
.file_name()
|
|
|
|
|
|
.and_then(|value| value.to_str())
|
|
|
|
|
|
.unwrap_or("OCR")
|
|
|
|
|
|
.trim_end_matches(".ocr.md")
|
|
|
|
|
|
.replace(['[', ']'], "")
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn mineru_token() -> Option<String> {
|
|
|
|
|
|
std::env::var("MNOTE_MINERU_API_TOKEN")
|
|
|
|
|
|
.ok()
|
|
|
|
|
|
.or_else(|| std::env::var("MINERU_API_TOKEN").ok())
|
|
|
|
|
|
.map(|value| value.trim().to_string())
|
|
|
|
|
|
.filter(|value| !value.is_empty())
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn mineru_api_base_url() -> String {
|
|
|
|
|
|
std::env::var("MNOTE_MINERU_API_BASE_URL")
|
|
|
|
|
|
.ok()
|
|
|
|
|
|
.or_else(|| std::env::var("MINERU_API_BASE_URL").ok())
|
|
|
|
|
|
.map(|value| value.trim().trim_end_matches('/').to_string())
|
|
|
|
|
|
.filter(|value| !value.is_empty())
|
|
|
|
|
|
.unwrap_or_else(|| DEFAULT_MINERU_API_BASE_URL.to_string())
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn mineru_poll_interval() -> Duration {
|
|
|
|
|
|
std::env::var("MNOTE_MINERU_POLL_INTERVAL_MS")
|
|
|
|
|
|
.ok()
|
|
|
|
|
|
.and_then(|value| value.trim().parse::<u64>().ok())
|
|
|
|
|
|
.filter(|value| *value > 0)
|
|
|
|
|
|
.map(Duration::from_millis)
|
|
|
|
|
|
.unwrap_or_else(|| Duration::from_secs(2))
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn mineru_max_polls() -> usize {
|
|
|
|
|
|
std::env::var("MNOTE_MINERU_MAX_POLLS")
|
|
|
|
|
|
.ok()
|
|
|
|
|
|
.and_then(|value| value.trim().parse::<usize>().ok())
|
|
|
|
|
|
.filter(|value| *value > 0)
|
|
|
|
|
|
.unwrap_or(90)
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[cfg(test)]
|
|
|
|
|
|
mod tests {
|
|
|
|
|
|
use super::*;
|
|
|
|
|
|
use crate::app::{build_app, AppConfig, AppState};
|
|
|
|
|
|
use axum::body::{to_bytes, Body};
|
|
|
|
|
|
use axum::http::Request;
|
|
|
|
|
|
use std::io::Write as _;
|
|
|
|
|
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
|
|
|
|
|
use std::sync::Arc;
|
|
|
|
|
|
use tower::util::ServiceExt;
|
|
|
|
|
|
|
|
|
|
|
|
fn temp_root(name: &str) -> PathBuf {
|
|
|
|
|
|
let root = std::env::temp_dir().join(format!("{name}-{}", std::process::id()));
|
|
|
|
|
|
let _ = fs::remove_dir_all(&root);
|
|
|
|
|
|
fs::create_dir_all(root.join("docs").join("Page.assets")).expect("create assets");
|
|
|
|
|
|
fs::create_dir_all(root.join(".mnote")).expect("create metadata");
|
|
|
|
|
|
root
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-06-01 10:30:42 +08:00
|
|
|
|
fn test_app_state() -> AppState {
|
|
|
|
|
|
AppState::new(AppConfig {
|
2026-06-01 09:29:12 +08:00
|
|
|
|
service_name: "mnote-web".into(),
|
|
|
|
|
|
service_version: "0.1.0".into(),
|
|
|
|
|
|
bind_addr: "127.0.0.1:0".into(),
|
|
|
|
|
|
public_bind_addr: "127.0.0.1:3000".into(),
|
|
|
|
|
|
legacy_next_base_url: Some("http://127.0.0.1:3100".into()),
|
|
|
|
|
|
enable_legacy_next_compat: true,
|
|
|
|
|
|
enable_debug_shell_routes: false,
|
|
|
|
|
|
enable_editor_actor: true,
|
|
|
|
|
|
hermes_base_path: "/api/hermes".into(),
|
|
|
|
|
|
compat_next_base_path: "/api/compat/next".into(),
|
|
|
|
|
|
convex_url: None,
|
|
|
|
|
|
convex_admin_key: None,
|
|
|
|
|
|
allow_dev_fixtures: true,
|
|
|
|
|
|
query_fixtures_json: None,
|
|
|
|
|
|
mutation_fixtures_json: None,
|
|
|
|
|
|
dev_user_id: "dev-user".into(),
|
|
|
|
|
|
dev_user_name: "开发用户".into(),
|
|
|
|
|
|
dev_user_email: "dev@mnote.local".into(),
|
2026-06-01 10:30:42 +08:00
|
|
|
|
})
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn app() -> axum::Router {
|
|
|
|
|
|
build_app(test_app_state())
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn app_with_state(state: AppState) -> axum::Router {
|
|
|
|
|
|
build_app(state)
|
2026-06-01 09:29:12 +08:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn write_workspace_manifest(root: &Path) {
|
|
|
|
|
|
fs::write(
|
|
|
|
|
|
root.join(".mnote").join("workspace.json"),
|
|
|
|
|
|
r#"{"workspaceId":"local-ws-ocr","ownerId":"user_test","createdAt":"2026-06-01T00:00:00Z","capabilities":["local_files"]}"#,
|
|
|
|
|
|
)
|
|
|
|
|
|
.expect("manifest");
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
async fn post_ocr_job(root: &Path, body: Value) -> (StatusCode, Value) {
|
|
|
|
|
|
let response = app()
|
|
|
|
|
|
.oneshot(
|
|
|
|
|
|
Request::builder()
|
|
|
|
|
|
.method("POST")
|
|
|
|
|
|
.uri("/api/local-folder/ocr/jobs")
|
|
|
|
|
|
.header("content-type", "application/json")
|
|
|
|
|
|
.header("x-mnote-actor-id", "user_test")
|
|
|
|
|
|
.header("x-mnote-actor-type", "user")
|
|
|
|
|
|
.body(Body::from(body.to_string()))
|
|
|
|
|
|
.expect("request"),
|
|
|
|
|
|
)
|
|
|
|
|
|
.await
|
|
|
|
|
|
.expect("response");
|
|
|
|
|
|
let status = response.status();
|
|
|
|
|
|
let body = to_bytes(response.into_body(), usize::MAX)
|
|
|
|
|
|
.await
|
|
|
|
|
|
.expect("response body");
|
|
|
|
|
|
let payload = serde_json::from_slice(&body).expect("response json");
|
|
|
|
|
|
let _ = fs::remove_dir_all(root);
|
|
|
|
|
|
(status, payload)
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn query_escape(value: &str) -> String {
|
|
|
|
|
|
value
|
|
|
|
|
|
.bytes()
|
|
|
|
|
|
.map(|byte| match byte {
|
|
|
|
|
|
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => {
|
|
|
|
|
|
(byte as char).to_string()
|
|
|
|
|
|
}
|
|
|
|
|
|
_ => format!("%{byte:02X}"),
|
|
|
|
|
|
})
|
|
|
|
|
|
.collect()
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-06-02 17:17:49 +08:00
|
|
|
|
fn build_test_mineru_zip_with_files(markdown: &str, files: &[(&str, &[u8])]) -> Vec<u8> {
|
2026-06-01 09:29:12 +08:00
|
|
|
|
let mut bytes = Cursor::new(Vec::<u8>::new());
|
|
|
|
|
|
{
|
|
|
|
|
|
let mut writer = zip::ZipWriter::new(&mut bytes);
|
|
|
|
|
|
writer
|
|
|
|
|
|
.start_file("full.md", zip::write::SimpleFileOptions::default())
|
|
|
|
|
|
.expect("zip start file");
|
2026-06-01 10:07:42 +08:00
|
|
|
|
writer.write_all(markdown.as_bytes()).expect("zip markdown");
|
2026-06-02 17:17:49 +08:00
|
|
|
|
for (name, content) in files {
|
|
|
|
|
|
writer
|
|
|
|
|
|
.start_file(*name, zip::write::SimpleFileOptions::default())
|
|
|
|
|
|
.expect("zip asset start file");
|
|
|
|
|
|
writer.write_all(content).expect("zip asset");
|
|
|
|
|
|
}
|
2026-06-01 09:29:12 +08:00
|
|
|
|
writer.finish().expect("zip finish");
|
|
|
|
|
|
}
|
|
|
|
|
|
bytes.into_inner()
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
|
|
fn local_ocr_sidecar_path_and_frontmatter_contract() {
|
|
|
|
|
|
let root = temp_root("mnote-local-ocr-sidecar");
|
|
|
|
|
|
fs::write(root.join("docs").join("Page.md"), "# Page\n").expect("page");
|
|
|
|
|
|
fs::write(
|
|
|
|
|
|
root.join("docs").join("Page.assets").join("photo.png"),
|
|
|
|
|
|
b"png",
|
|
|
|
|
|
)
|
|
|
|
|
|
.expect("photo");
|
|
|
|
|
|
let plan = plan_ocr_sidecar_path(
|
|
|
|
|
|
&root,
|
|
|
|
|
|
"local-md:docs~2FPage.md",
|
|
|
|
|
|
"docs/Page.assets/photo.png",
|
|
|
|
|
|
"mineru",
|
|
|
|
|
|
"vlm",
|
|
|
|
|
|
false,
|
|
|
|
|
|
)
|
|
|
|
|
|
.expect("plan");
|
|
|
|
|
|
assert_eq!(
|
|
|
|
|
|
plan.ocr_root_relative_path,
|
|
|
|
|
|
"docs/Page.ocr/photo.png.ocr.md"
|
|
|
|
|
|
);
|
|
|
|
|
|
let markdown = build_ocr_markdown(&plan, "识别文本", "done", 1780000000000);
|
|
|
|
|
|
assert!(markdown.contains("mnote_ocr_version: 1"));
|
|
|
|
|
|
assert!(markdown.contains("provider: mineru"));
|
|
|
|
|
|
assert!(markdown.contains("model_version: vlm"));
|
|
|
|
|
|
assert!(markdown.contains("owner_document: ../Page.md"));
|
|
|
|
|
|
assert!(markdown.contains("source_path: ./Page.assets/photo.png"));
|
|
|
|
|
|
assert!(markdown.contains("source_root_relative_path: docs/Page.assets/photo.png"));
|
|
|
|
|
|
assert!(markdown.contains("status: done"));
|
|
|
|
|
|
let parsed = parse_ocr_frontmatter(&markdown).expect("frontmatter");
|
|
|
|
|
|
assert_eq!(
|
|
|
|
|
|
parsed.source_root_relative_path,
|
|
|
|
|
|
"docs/Page.assets/photo.png"
|
|
|
|
|
|
);
|
|
|
|
|
|
let _ = fs::remove_dir_all(root);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
|
|
fn local_ocr_index_marks_stale_and_redacts_error() {
|
|
|
|
|
|
let root = temp_root("mnote-local-ocr-index");
|
|
|
|
|
|
fs::write(root.join("docs").join("Page.md"), "# Page\n").expect("page");
|
|
|
|
|
|
let source = root.join("docs").join("Page.assets").join("scan.pdf");
|
|
|
|
|
|
fs::write(&source, b"pdf").expect("pdf");
|
|
|
|
|
|
let plan = plan_ocr_sidecar_path(
|
|
|
|
|
|
&root,
|
|
|
|
|
|
"local-md:docs~2FPage.md",
|
|
|
|
|
|
"docs/Page.assets/scan.pdf",
|
|
|
|
|
|
"mock",
|
|
|
|
|
|
"vlm",
|
|
|
|
|
|
false,
|
|
|
|
|
|
)
|
|
|
|
|
|
.expect("plan");
|
|
|
|
|
|
let entry = build_index_entry(
|
|
|
|
|
|
&plan,
|
|
|
|
|
|
"failed",
|
|
|
|
|
|
1780000000000,
|
|
|
|
|
|
"",
|
|
|
|
|
|
Some(redact_error("token=secret upload_url=https://example.test")),
|
|
|
|
|
|
);
|
|
|
|
|
|
upsert_ocr_index_entry(&root, entry.clone()).expect("write index");
|
|
|
|
|
|
let stored = read_ocr_index_entry(&root, "docs/Page.assets/scan.pdf")
|
|
|
|
|
|
.expect("read index")
|
|
|
|
|
|
.expect("entry");
|
|
|
|
|
|
assert_eq!(stored.error.as_deref(), Some("provider_error_redacted"));
|
|
|
|
|
|
fs::write(&source, b"pdf changed").expect("change source");
|
|
|
|
|
|
assert!(source_is_stale(&root, &stored));
|
|
|
|
|
|
let _ = fs::remove_dir_all(root);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[tokio::test]
|
|
|
|
|
|
async fn local_ocr_jobs_route_writes_mock_sidecar_and_reads_status() {
|
|
|
|
|
|
let root = temp_root("mnote-local-ocr-route");
|
|
|
|
|
|
write_workspace_manifest(&root);
|
|
|
|
|
|
fs::write(root.join("docs").join("Page.md"), "# Page\n").expect("page");
|
|
|
|
|
|
fs::write(
|
|
|
|
|
|
root.join("docs").join("Page.assets").join("photo.png"),
|
|
|
|
|
|
b"png",
|
|
|
|
|
|
)
|
|
|
|
|
|
.expect("photo");
|
|
|
|
|
|
let root_uri = format!("file://{}", root.display());
|
|
|
|
|
|
let response = app()
|
|
|
|
|
|
.oneshot(
|
|
|
|
|
|
Request::builder()
|
|
|
|
|
|
.method("POST")
|
|
|
|
|
|
.uri("/api/local-folder/ocr/jobs")
|
|
|
|
|
|
.header("content-type", "application/json")
|
|
|
|
|
|
.header("x-mnote-actor-id", "user_test")
|
|
|
|
|
|
.header("x-mnote-actor-type", "user")
|
|
|
|
|
|
.body(Body::from(
|
|
|
|
|
|
json!({
|
|
|
|
|
|
"rootUri": root_uri,
|
|
|
|
|
|
"documentId": "local-md:docs~2FPage.md",
|
|
|
|
|
|
"sourceRootRelativePath": "docs/Page.assets/photo.png",
|
|
|
|
|
|
"provider": "mock",
|
|
|
|
|
|
"mockMarkdown": "Route OCR Token"
|
|
|
|
|
|
})
|
|
|
|
|
|
.to_string(),
|
|
|
|
|
|
))
|
|
|
|
|
|
.expect("request"),
|
|
|
|
|
|
)
|
|
|
|
|
|
.await
|
|
|
|
|
|
.expect("response");
|
|
|
|
|
|
assert_eq!(response.status(), StatusCode::OK);
|
|
|
|
|
|
assert!(root
|
|
|
|
|
|
.join("docs")
|
|
|
|
|
|
.join("Page.ocr")
|
|
|
|
|
|
.join("photo.png.ocr.md")
|
|
|
|
|
|
.is_file());
|
2026-06-04 18:51:16 +08:00
|
|
|
|
assert!(root
|
|
|
|
|
|
.join("docs")
|
|
|
|
|
|
.join("Page.ocr")
|
|
|
|
|
|
.join("photo.png.parse.md")
|
|
|
|
|
|
.is_file());
|
2026-06-01 09:29:12 +08:00
|
|
|
|
|
|
|
|
|
|
let escaped_root = query_escape(&root_uri);
|
|
|
|
|
|
let status_response = app()
|
|
|
|
|
|
.oneshot(
|
|
|
|
|
|
Request::builder()
|
|
|
|
|
|
.uri(format!(
|
|
|
|
|
|
"/api/local-folder/ocr/status?rootUri={escaped_root}&sourceRootRelativePath=docs%2FPage.assets%2Fphoto.png"
|
|
|
|
|
|
))
|
|
|
|
|
|
.header("x-mnote-actor-id", "user_test")
|
|
|
|
|
|
.header("x-mnote-actor-type", "user")
|
|
|
|
|
|
.body(Body::empty())
|
|
|
|
|
|
.expect("request"),
|
|
|
|
|
|
)
|
|
|
|
|
|
.await
|
|
|
|
|
|
.expect("status response");
|
|
|
|
|
|
assert_eq!(status_response.status(), StatusCode::OK);
|
|
|
|
|
|
let status_body = to_bytes(status_response.into_body(), usize::MAX)
|
|
|
|
|
|
.await
|
|
|
|
|
|
.expect("status body");
|
|
|
|
|
|
let status_payload: Value = serde_json::from_slice(&status_body).expect("status json");
|
|
|
|
|
|
assert_eq!(status_payload["job"]["status"].as_str(), Some("done"));
|
|
|
|
|
|
|
|
|
|
|
|
let read_response = app()
|
|
|
|
|
|
.oneshot(
|
|
|
|
|
|
Request::builder()
|
|
|
|
|
|
.uri(format!(
|
|
|
|
|
|
"/api/local-folder/ocr/read?rootUri={escaped_root}&ocrRootRelativePath=docs%2FPage.ocr%2Fphoto.png.ocr.md"
|
|
|
|
|
|
))
|
|
|
|
|
|
.header("x-mnote-actor-id", "user_test")
|
|
|
|
|
|
.header("x-mnote-actor-type", "user")
|
|
|
|
|
|
.body(Body::empty())
|
|
|
|
|
|
.expect("request"),
|
|
|
|
|
|
)
|
|
|
|
|
|
.await
|
|
|
|
|
|
.expect("read response");
|
|
|
|
|
|
assert_eq!(read_response.status(), StatusCode::OK);
|
|
|
|
|
|
let read_body = to_bytes(read_response.into_body(), usize::MAX)
|
|
|
|
|
|
.await
|
|
|
|
|
|
.expect("read body");
|
|
|
|
|
|
let read_payload: Value = serde_json::from_slice(&read_body).expect("read json");
|
|
|
|
|
|
assert!(read_payload["markdown"]
|
|
|
|
|
|
.as_str()
|
|
|
|
|
|
.is_some_and(|markdown| markdown.contains("Route OCR Token")));
|
2026-06-02 17:17:49 +08:00
|
|
|
|
let delete_response = app()
|
|
|
|
|
|
.oneshot(
|
|
|
|
|
|
Request::builder()
|
|
|
|
|
|
.method("DELETE")
|
|
|
|
|
|
.uri("/api/local-folder/ocr/jobs")
|
|
|
|
|
|
.header("content-type", "application/json")
|
|
|
|
|
|
.header("x-mnote-actor-id", "user_test")
|
|
|
|
|
|
.header("x-mnote-actor-type", "user")
|
|
|
|
|
|
.body(Body::from(
|
|
|
|
|
|
json!({
|
|
|
|
|
|
"rootUri": root_uri,
|
|
|
|
|
|
"sourceRootRelativePath": "docs/Page.assets/photo.png"
|
|
|
|
|
|
})
|
|
|
|
|
|
.to_string(),
|
|
|
|
|
|
))
|
|
|
|
|
|
.expect("delete request"),
|
|
|
|
|
|
)
|
|
|
|
|
|
.await
|
|
|
|
|
|
.expect("delete response");
|
|
|
|
|
|
assert_eq!(delete_response.status(), StatusCode::OK);
|
|
|
|
|
|
assert!(!root
|
|
|
|
|
|
.join("docs")
|
|
|
|
|
|
.join("Page.ocr")
|
|
|
|
|
|
.join("photo.png.ocr.md")
|
|
|
|
|
|
.exists());
|
2026-06-04 18:51:16 +08:00
|
|
|
|
assert!(!root
|
|
|
|
|
|
.join("docs")
|
|
|
|
|
|
.join("Page.ocr")
|
|
|
|
|
|
.join("photo.png.parse.md")
|
|
|
|
|
|
.exists());
|
2026-06-02 17:17:49 +08:00
|
|
|
|
assert!(read_ocr_index(&root)
|
|
|
|
|
|
.expect("index after delete")
|
|
|
|
|
|
.entries
|
|
|
|
|
|
.is_empty());
|
2026-06-01 09:29:12 +08:00
|
|
|
|
let _ = fs::remove_dir_all(root);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-06-04 18:51:16 +08:00
|
|
|
|
#[tokio::test]
|
|
|
|
|
|
async fn local_ocr_jobs_route_reuses_existing_done_sidecar_without_reprocessing() {
|
|
|
|
|
|
let root = temp_root("mnote-local-ocr-dedup-done");
|
|
|
|
|
|
write_workspace_manifest(&root);
|
|
|
|
|
|
fs::write(root.join("docs").join("Page.md"), "# Page\n").expect("page");
|
|
|
|
|
|
fs::write(
|
|
|
|
|
|
root.join("docs").join("Page.assets").join("photo.png"),
|
|
|
|
|
|
b"png",
|
|
|
|
|
|
)
|
|
|
|
|
|
.expect("photo");
|
|
|
|
|
|
let root_uri = format!("file://{}", root.display());
|
|
|
|
|
|
let create_payload = |markdown: &str| {
|
|
|
|
|
|
json!({
|
|
|
|
|
|
"rootUri": root_uri,
|
|
|
|
|
|
"documentId": "local-md:docs~2FPage.md",
|
|
|
|
|
|
"sourceRootRelativePath": "docs/Page.assets/photo.png",
|
|
|
|
|
|
"provider": "mock",
|
|
|
|
|
|
"mockMarkdown": markdown
|
|
|
|
|
|
})
|
|
|
|
|
|
};
|
|
|
|
|
|
let first_response = app()
|
|
|
|
|
|
.oneshot(
|
|
|
|
|
|
Request::builder()
|
|
|
|
|
|
.method("POST")
|
|
|
|
|
|
.uri("/api/local-folder/ocr/jobs")
|
|
|
|
|
|
.header("content-type", "application/json")
|
|
|
|
|
|
.header("x-mnote-actor-id", "user_test")
|
|
|
|
|
|
.header("x-mnote-actor-type", "user")
|
|
|
|
|
|
.body(Body::from(create_payload("First OCR Token").to_string()))
|
|
|
|
|
|
.expect("first request"),
|
|
|
|
|
|
)
|
|
|
|
|
|
.await
|
|
|
|
|
|
.expect("first response");
|
|
|
|
|
|
assert_eq!(first_response.status(), StatusCode::OK);
|
|
|
|
|
|
let first_body = to_bytes(first_response.into_body(), usize::MAX)
|
|
|
|
|
|
.await
|
|
|
|
|
|
.expect("first body");
|
|
|
|
|
|
let first_payload: Value = serde_json::from_slice(&first_body).expect("first json");
|
|
|
|
|
|
assert_eq!(first_payload["job"]["status"].as_str(), Some("done"));
|
|
|
|
|
|
|
|
|
|
|
|
let second_response = app()
|
|
|
|
|
|
.oneshot(
|
|
|
|
|
|
Request::builder()
|
|
|
|
|
|
.method("POST")
|
|
|
|
|
|
.uri("/api/local-folder/ocr/jobs")
|
|
|
|
|
|
.header("content-type", "application/json")
|
|
|
|
|
|
.header("x-mnote-actor-id", "user_test")
|
|
|
|
|
|
.header("x-mnote-actor-type", "user")
|
|
|
|
|
|
.body(Body::from(create_payload("Second OCR Token").to_string()))
|
|
|
|
|
|
.expect("second request"),
|
|
|
|
|
|
)
|
|
|
|
|
|
.await
|
|
|
|
|
|
.expect("second response");
|
|
|
|
|
|
assert_eq!(second_response.status(), StatusCode::OK);
|
|
|
|
|
|
let second_body = to_bytes(second_response.into_body(), usize::MAX)
|
|
|
|
|
|
.await
|
|
|
|
|
|
.expect("second body");
|
|
|
|
|
|
let second_payload: Value = serde_json::from_slice(&second_body).expect("second json");
|
|
|
|
|
|
assert_eq!(second_payload["deduplicated"].as_bool(), Some(true));
|
|
|
|
|
|
assert_eq!(
|
|
|
|
|
|
second_payload["job"]["ocrRootRelativePath"].as_str(),
|
|
|
|
|
|
Some("docs/Page.ocr/photo.png.ocr.md")
|
|
|
|
|
|
);
|
|
|
|
|
|
let sidecar =
|
|
|
|
|
|
fs::read_to_string(root.join("docs").join("Page.ocr").join("photo.png.ocr.md"))
|
|
|
|
|
|
.expect("sidecar");
|
|
|
|
|
|
assert!(sidecar.contains("First OCR Token"));
|
|
|
|
|
|
assert!(!sidecar.contains("Second OCR Token"));
|
|
|
|
|
|
assert!(!root
|
|
|
|
|
|
.join("docs")
|
|
|
|
|
|
.join("Page.ocr")
|
|
|
|
|
|
.join("photo.png-")
|
|
|
|
|
|
.exists());
|
|
|
|
|
|
let _ = fs::remove_dir_all(root);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[tokio::test]
|
|
|
|
|
|
async fn local_ocr_jobs_route_recovers_existing_done_sidecar_when_index_is_missing() {
|
|
|
|
|
|
let root = temp_root("mnote-local-ocr-dedup-sidecar-recover");
|
|
|
|
|
|
write_workspace_manifest(&root);
|
|
|
|
|
|
fs::write(root.join("docs").join("Page.md"), "# Page\n").expect("page");
|
|
|
|
|
|
fs::write(
|
|
|
|
|
|
root.join("docs").join("Page.assets").join("photo.png"),
|
|
|
|
|
|
b"png",
|
|
|
|
|
|
)
|
|
|
|
|
|
.expect("photo");
|
|
|
|
|
|
let root_uri = format!("file://{}", root.display());
|
|
|
|
|
|
let create_payload = |markdown: &str| {
|
|
|
|
|
|
json!({
|
|
|
|
|
|
"rootUri": root_uri,
|
|
|
|
|
|
"documentId": "local-md:docs~2FPage.md",
|
|
|
|
|
|
"sourceRootRelativePath": "docs/Page.assets/photo.png",
|
|
|
|
|
|
"provider": "mock",
|
|
|
|
|
|
"mockMarkdown": markdown
|
|
|
|
|
|
})
|
|
|
|
|
|
};
|
|
|
|
|
|
let first_response = app()
|
|
|
|
|
|
.oneshot(
|
|
|
|
|
|
Request::builder()
|
|
|
|
|
|
.method("POST")
|
|
|
|
|
|
.uri("/api/local-folder/ocr/jobs")
|
|
|
|
|
|
.header("content-type", "application/json")
|
|
|
|
|
|
.header("x-mnote-actor-id", "user_test")
|
|
|
|
|
|
.header("x-mnote-actor-type", "user")
|
|
|
|
|
|
.body(Body::from(
|
|
|
|
|
|
create_payload("Recovered OCR Token").to_string(),
|
|
|
|
|
|
))
|
|
|
|
|
|
.expect("first request"),
|
|
|
|
|
|
)
|
|
|
|
|
|
.await
|
|
|
|
|
|
.expect("first response");
|
|
|
|
|
|
assert_eq!(first_response.status(), StatusCode::OK);
|
|
|
|
|
|
fs::remove_file(ocr_index_path(&root)).expect("remove index");
|
|
|
|
|
|
|
|
|
|
|
|
let second_response = app()
|
|
|
|
|
|
.oneshot(
|
|
|
|
|
|
Request::builder()
|
|
|
|
|
|
.method("POST")
|
|
|
|
|
|
.uri("/api/local-folder/ocr/jobs")
|
|
|
|
|
|
.header("content-type", "application/json")
|
|
|
|
|
|
.header("x-mnote-actor-id", "user_test")
|
|
|
|
|
|
.header("x-mnote-actor-type", "user")
|
|
|
|
|
|
.body(Body::from(
|
|
|
|
|
|
create_payload("Should Not Reprocess").to_string(),
|
|
|
|
|
|
))
|
|
|
|
|
|
.expect("second request"),
|
|
|
|
|
|
)
|
|
|
|
|
|
.await
|
|
|
|
|
|
.expect("second response");
|
|
|
|
|
|
assert_eq!(second_response.status(), StatusCode::OK);
|
|
|
|
|
|
let second_body = to_bytes(second_response.into_body(), usize::MAX)
|
|
|
|
|
|
.await
|
|
|
|
|
|
.expect("second body");
|
|
|
|
|
|
let second_payload: Value = serde_json::from_slice(&second_body).expect("second json");
|
|
|
|
|
|
assert_eq!(second_payload["deduplicated"].as_bool(), Some(true));
|
|
|
|
|
|
assert_eq!(
|
|
|
|
|
|
read_ocr_index(&root)
|
|
|
|
|
|
.expect("recovered index")
|
|
|
|
|
|
.entries
|
|
|
|
|
|
.len(),
|
|
|
|
|
|
1
|
|
|
|
|
|
);
|
|
|
|
|
|
let sidecar =
|
|
|
|
|
|
fs::read_to_string(root.join("docs").join("Page.ocr").join("photo.png.ocr.md"))
|
|
|
|
|
|
.expect("sidecar");
|
|
|
|
|
|
assert!(sidecar.contains("Recovered OCR Token"));
|
|
|
|
|
|
assert!(!sidecar.contains("Should Not Reprocess"));
|
|
|
|
|
|
let _ = fs::remove_dir_all(root);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[tokio::test]
|
|
|
|
|
|
async fn local_ocr_jobs_route_reuses_existing_done_before_mineru_token_check() {
|
|
|
|
|
|
let old_mnote_token = std::env::var("MNOTE_MINERU_API_TOKEN").ok();
|
|
|
|
|
|
let old_mineru_token = std::env::var("MINERU_API_TOKEN").ok();
|
|
|
|
|
|
std::env::remove_var("MNOTE_MINERU_API_TOKEN");
|
|
|
|
|
|
std::env::remove_var("MINERU_API_TOKEN");
|
|
|
|
|
|
|
|
|
|
|
|
let root = temp_root("mnote-local-ocr-dedup-before-token");
|
|
|
|
|
|
write_workspace_manifest(&root);
|
|
|
|
|
|
fs::write(root.join("docs").join("Page.md"), "# Page\n").expect("page");
|
|
|
|
|
|
fs::write(
|
|
|
|
|
|
root.join("docs").join("Page.assets").join("photo.png"),
|
|
|
|
|
|
b"png",
|
|
|
|
|
|
)
|
|
|
|
|
|
.expect("photo");
|
|
|
|
|
|
let root_uri = format!("file://{}", root.display());
|
|
|
|
|
|
let first_response = app()
|
|
|
|
|
|
.oneshot(
|
|
|
|
|
|
Request::builder()
|
|
|
|
|
|
.method("POST")
|
|
|
|
|
|
.uri("/api/local-folder/ocr/jobs")
|
|
|
|
|
|
.header("content-type", "application/json")
|
|
|
|
|
|
.header("x-mnote-actor-id", "user_test")
|
|
|
|
|
|
.header("x-mnote-actor-type", "user")
|
|
|
|
|
|
.body(Body::from(
|
|
|
|
|
|
json!({
|
|
|
|
|
|
"rootUri": root_uri,
|
|
|
|
|
|
"documentId": "local-md:docs~2FPage.md",
|
|
|
|
|
|
"sourceRootRelativePath": "docs/Page.assets/photo.png",
|
|
|
|
|
|
"provider": "mock",
|
|
|
|
|
|
"mockMarkdown": "Existing OCR Token"
|
|
|
|
|
|
})
|
|
|
|
|
|
.to_string(),
|
|
|
|
|
|
))
|
|
|
|
|
|
.expect("first request"),
|
|
|
|
|
|
)
|
|
|
|
|
|
.await
|
|
|
|
|
|
.expect("first response");
|
|
|
|
|
|
assert_eq!(first_response.status(), StatusCode::OK);
|
|
|
|
|
|
|
|
|
|
|
|
let second_response = app()
|
|
|
|
|
|
.oneshot(
|
|
|
|
|
|
Request::builder()
|
|
|
|
|
|
.method("POST")
|
|
|
|
|
|
.uri("/api/local-folder/ocr/jobs")
|
|
|
|
|
|
.header("content-type", "application/json")
|
|
|
|
|
|
.header("x-mnote-actor-id", "user_test")
|
|
|
|
|
|
.header("x-mnote-actor-type", "user")
|
|
|
|
|
|
.body(Body::from(
|
|
|
|
|
|
json!({
|
|
|
|
|
|
"rootUri": root_uri,
|
|
|
|
|
|
"documentId": "local-md:docs~2FPage.md",
|
|
|
|
|
|
"sourceRootRelativePath": "docs/Page.assets/photo.png",
|
|
|
|
|
|
"provider": "mineru"
|
|
|
|
|
|
})
|
|
|
|
|
|
.to_string(),
|
|
|
|
|
|
))
|
|
|
|
|
|
.expect("second request"),
|
|
|
|
|
|
)
|
|
|
|
|
|
.await
|
|
|
|
|
|
.expect("second response");
|
|
|
|
|
|
|
|
|
|
|
|
if let Some(value) = old_mnote_token {
|
|
|
|
|
|
std::env::set_var("MNOTE_MINERU_API_TOKEN", value);
|
|
|
|
|
|
}
|
|
|
|
|
|
if let Some(value) = old_mineru_token {
|
|
|
|
|
|
std::env::set_var("MINERU_API_TOKEN", value);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
assert_eq!(second_response.status(), StatusCode::OK);
|
|
|
|
|
|
let body = to_bytes(second_response.into_body(), usize::MAX)
|
|
|
|
|
|
.await
|
|
|
|
|
|
.expect("second body");
|
|
|
|
|
|
let payload: Value = serde_json::from_slice(&body).expect("second json");
|
|
|
|
|
|
assert_eq!(payload["deduplicated"].as_bool(), Some(true));
|
|
|
|
|
|
assert_eq!(payload["job"]["status"].as_str(), Some("done"));
|
|
|
|
|
|
let _ = fs::remove_dir_all(root);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-06-01 10:30:42 +08:00
|
|
|
|
#[tokio::test]
|
|
|
|
|
|
async fn local_ocr_jobs_route_broadcasts_stage_updates() {
|
|
|
|
|
|
let root = temp_root("mnote-local-ocr-events");
|
|
|
|
|
|
write_workspace_manifest(&root);
|
|
|
|
|
|
fs::write(root.join("docs").join("Page.md"), "# Page\n").expect("page");
|
|
|
|
|
|
fs::write(
|
|
|
|
|
|
root.join("docs").join("Page.assets").join("photo.png"),
|
|
|
|
|
|
b"png",
|
|
|
|
|
|
)
|
|
|
|
|
|
.expect("photo");
|
|
|
|
|
|
let root_uri = format!("file://{}", root.display());
|
|
|
|
|
|
let state = test_app_state();
|
|
|
|
|
|
let mut events = state.local_ocr_job_tx.subscribe();
|
|
|
|
|
|
let response = app_with_state(state.clone())
|
|
|
|
|
|
.oneshot(
|
|
|
|
|
|
Request::builder()
|
|
|
|
|
|
.method("POST")
|
|
|
|
|
|
.uri("/api/local-folder/ocr/jobs")
|
|
|
|
|
|
.header("content-type", "application/json")
|
|
|
|
|
|
.header("x-mnote-actor-id", "user_test")
|
|
|
|
|
|
.header("x-mnote-actor-type", "user")
|
|
|
|
|
|
.body(Body::from(
|
|
|
|
|
|
json!({
|
|
|
|
|
|
"rootUri": root_uri,
|
|
|
|
|
|
"documentId": "local-md:docs~2FPage.md",
|
|
|
|
|
|
"sourceRootRelativePath": "docs/Page.assets/photo.png",
|
|
|
|
|
|
"provider": "mock",
|
|
|
|
|
|
"mockMarkdown": "Event OCR Token"
|
|
|
|
|
|
})
|
|
|
|
|
|
.to_string(),
|
|
|
|
|
|
))
|
|
|
|
|
|
.expect("request"),
|
|
|
|
|
|
)
|
|
|
|
|
|
.await
|
|
|
|
|
|
.expect("response");
|
|
|
|
|
|
assert_eq!(response.status(), StatusCode::OK);
|
|
|
|
|
|
|
|
|
|
|
|
let mut statuses = Vec::new();
|
|
|
|
|
|
for _ in 0..3 {
|
|
|
|
|
|
let payload = tokio::time::timeout(Duration::from_secs(1), events.recv())
|
|
|
|
|
|
.await
|
|
|
|
|
|
.expect("ocr event timeout")
|
|
|
|
|
|
.expect("ocr event");
|
|
|
|
|
|
assert_eq!(payload["eventType"].as_str(), Some("local_ocr.job.updated"));
|
|
|
|
|
|
assert_eq!(
|
|
|
|
|
|
payload["job"]["sourceRootRelativePath"].as_str(),
|
|
|
|
|
|
Some("docs/Page.assets/photo.png")
|
|
|
|
|
|
);
|
|
|
|
|
|
statuses.push(
|
|
|
|
|
|
payload["job"]["status"]
|
|
|
|
|
|
.as_str()
|
|
|
|
|
|
.unwrap_or_default()
|
|
|
|
|
|
.to_string(),
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
assert_eq!(statuses, vec!["queued", "writing_sidecar", "done"]);
|
|
|
|
|
|
assert!(state
|
|
|
|
|
|
.local_ocr_active_jobs
|
|
|
|
|
|
.read()
|
|
|
|
|
|
.expect("active jobs lock")
|
|
|
|
|
|
.is_empty());
|
|
|
|
|
|
let _ = fs::remove_dir_all(root);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-06-01 09:29:12 +08:00
|
|
|
|
#[tokio::test]
|
|
|
|
|
|
async fn local_ocr_jobs_route_rejects_missing_mineru_token() {
|
|
|
|
|
|
let old_mnote_token = std::env::var("MNOTE_MINERU_API_TOKEN").ok();
|
|
|
|
|
|
let old_mineru_token = std::env::var("MINERU_API_TOKEN").ok();
|
|
|
|
|
|
std::env::remove_var("MNOTE_MINERU_API_TOKEN");
|
|
|
|
|
|
std::env::remove_var("MINERU_API_TOKEN");
|
|
|
|
|
|
|
|
|
|
|
|
let root = temp_root("mnote-local-ocr-token");
|
|
|
|
|
|
write_workspace_manifest(&root);
|
2026-06-04 18:51:16 +08:00
|
|
|
|
fs::write(root.join("docs").join("Page.md"), "# Page\n").expect("page");
|
|
|
|
|
|
fs::write(
|
|
|
|
|
|
root.join("docs").join("Page.assets").join("photo.png"),
|
|
|
|
|
|
b"png",
|
|
|
|
|
|
)
|
|
|
|
|
|
.expect("photo");
|
2026-06-01 09:29:12 +08:00
|
|
|
|
let root_uri = format!("file://{}", root.display());
|
|
|
|
|
|
let (status, payload) = post_ocr_job(
|
|
|
|
|
|
&root,
|
|
|
|
|
|
json!({
|
|
|
|
|
|
"rootUri": root_uri,
|
|
|
|
|
|
"documentId": "local-md:docs~2FPage.md",
|
|
|
|
|
|
"sourceRootRelativePath": "docs/Page.assets/photo.png",
|
|
|
|
|
|
"provider": "mineru"
|
|
|
|
|
|
}),
|
|
|
|
|
|
)
|
|
|
|
|
|
.await;
|
|
|
|
|
|
if let Some(value) = old_mnote_token {
|
|
|
|
|
|
std::env::set_var("MNOTE_MINERU_API_TOKEN", value);
|
|
|
|
|
|
}
|
|
|
|
|
|
if let Some(value) = old_mineru_token {
|
|
|
|
|
|
std::env::set_var("MINERU_API_TOKEN", value);
|
|
|
|
|
|
}
|
|
|
|
|
|
assert_eq!(status, StatusCode::UNAUTHORIZED);
|
|
|
|
|
|
assert_eq!(payload["code"].as_str(), Some("mineru_token_missing"));
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[tokio::test]
|
|
|
|
|
|
async fn local_ocr_jobs_route_runs_mineru_runtime_against_http_mock() {
|
|
|
|
|
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
|
|
|
|
|
.await
|
|
|
|
|
|
.expect("mock mineru bind");
|
|
|
|
|
|
let base_url = format!("http://{}", listener.local_addr().expect("mock addr"));
|
|
|
|
|
|
let upload_count = Arc::new(AtomicUsize::new(0));
|
|
|
|
|
|
let poll_count = Arc::new(AtomicUsize::new(0));
|
2026-06-02 17:17:49 +08:00
|
|
|
|
let zip_bytes = Arc::new(build_test_mineru_zip_with_files(
|
|
|
|
|
|
"# MinerU Result\n\n\n\n识别文本",
|
2026-06-04 18:51:16 +08:00
|
|
|
|
&[
|
|
|
|
|
|
("images/ocr.png", b"png-bytes"),
|
|
|
|
|
|
(
|
|
|
|
|
|
"content_list.json",
|
|
|
|
|
|
r#"[{"type":"title","level":1,"page_idx":0,"text":"MinerU Result","bbox":[0,0,100,18]},{"type":"text","page_idx":0,"text":"识别文本","bbox":[10,20,110,40]}]"#
|
|
|
|
|
|
.as_bytes(),
|
|
|
|
|
|
),
|
|
|
|
|
|
],
|
2026-06-02 17:17:49 +08:00
|
|
|
|
));
|
2026-06-01 09:29:12 +08:00
|
|
|
|
|
|
|
|
|
|
let mock_mineru = axum::Router::new()
|
|
|
|
|
|
.route(
|
|
|
|
|
|
"/api/v4/file-urls/batch",
|
|
|
|
|
|
axum::routing::post({
|
|
|
|
|
|
let base_url = base_url.clone();
|
|
|
|
|
|
|| async move {
|
|
|
|
|
|
Json(json!({
|
2026-06-02 17:17:49 +08:00
|
|
|
|
"code": 0,
|
|
|
|
|
|
"msg": "ok",
|
|
|
|
|
|
"data": {
|
|
|
|
|
|
"batch_id": "batch_1",
|
|
|
|
|
|
"file_urls": [format!("{base_url}/upload/source")]
|
|
|
|
|
|
}
|
2026-06-01 09:29:12 +08:00
|
|
|
|
}))
|
|
|
|
|
|
}
|
|
|
|
|
|
}),
|
|
|
|
|
|
)
|
|
|
|
|
|
.route(
|
|
|
|
|
|
"/upload/source",
|
|
|
|
|
|
axum::routing::put({
|
|
|
|
|
|
let upload_count = upload_count.clone();
|
|
|
|
|
|
move |body: axum::body::Bytes| {
|
|
|
|
|
|
let upload_count = upload_count.clone();
|
|
|
|
|
|
async move {
|
|
|
|
|
|
assert!(!body.is_empty());
|
|
|
|
|
|
upload_count.fetch_add(1, Ordering::SeqCst);
|
|
|
|
|
|
StatusCode::OK
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}),
|
|
|
|
|
|
)
|
|
|
|
|
|
.route(
|
|
|
|
|
|
"/api/v4/extract-results/batch/batch_1",
|
|
|
|
|
|
axum::routing::get({
|
|
|
|
|
|
let base_url = base_url.clone();
|
|
|
|
|
|
let poll_count = poll_count.clone();
|
|
|
|
|
|
move || {
|
|
|
|
|
|
let poll_count = poll_count.clone();
|
|
|
|
|
|
async move {
|
|
|
|
|
|
poll_count.fetch_add(1, Ordering::SeqCst);
|
|
|
|
|
|
Json(json!({
|
|
|
|
|
|
"status": "done",
|
|
|
|
|
|
"full_zip_url": format!("{base_url}/result.zip")
|
|
|
|
|
|
}))
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}),
|
|
|
|
|
|
)
|
|
|
|
|
|
.route(
|
|
|
|
|
|
"/result.zip",
|
|
|
|
|
|
axum::routing::get({
|
|
|
|
|
|
let zip_bytes = zip_bytes.clone();
|
|
|
|
|
|
move || {
|
|
|
|
|
|
let zip_bytes = zip_bytes.clone();
|
|
|
|
|
|
async move {
|
|
|
|
|
|
(
|
|
|
|
|
|
[(axum::http::header::CONTENT_TYPE, "application/zip")],
|
|
|
|
|
|
(*zip_bytes).clone(),
|
|
|
|
|
|
)
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}),
|
|
|
|
|
|
);
|
|
|
|
|
|
let mock_handle = tokio::spawn(async move {
|
2026-06-01 10:07:42 +08:00
|
|
|
|
axum::serve(listener, mock_mineru)
|
|
|
|
|
|
.await
|
|
|
|
|
|
.expect("mock mineru server");
|
2026-06-01 09:29:12 +08:00
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
|
|
let old_token = std::env::var("MNOTE_MINERU_API_TOKEN").ok();
|
|
|
|
|
|
let old_base = std::env::var("MNOTE_MINERU_API_BASE_URL").ok();
|
|
|
|
|
|
let old_interval = std::env::var("MNOTE_MINERU_POLL_INTERVAL_MS").ok();
|
|
|
|
|
|
let old_max_polls = std::env::var("MNOTE_MINERU_MAX_POLLS").ok();
|
|
|
|
|
|
std::env::set_var("MNOTE_MINERU_API_TOKEN", "test-mineru-token");
|
|
|
|
|
|
std::env::set_var("MNOTE_MINERU_API_BASE_URL", &base_url);
|
|
|
|
|
|
std::env::set_var("MNOTE_MINERU_POLL_INTERVAL_MS", "1");
|
|
|
|
|
|
std::env::set_var("MNOTE_MINERU_MAX_POLLS", "2");
|
|
|
|
|
|
|
|
|
|
|
|
let root = temp_root("mnote-local-ocr-mineru-runtime");
|
|
|
|
|
|
write_workspace_manifest(&root);
|
|
|
|
|
|
fs::write(root.join("docs").join("Page.md"), "# Page\n").expect("page");
|
|
|
|
|
|
fs::write(
|
|
|
|
|
|
root.join("docs").join("Page.assets").join("photo.png"),
|
|
|
|
|
|
b"png",
|
|
|
|
|
|
)
|
|
|
|
|
|
.expect("photo");
|
|
|
|
|
|
let root_uri = format!("file://{}", root.display());
|
|
|
|
|
|
let response = app()
|
|
|
|
|
|
.oneshot(
|
|
|
|
|
|
Request::builder()
|
|
|
|
|
|
.method("POST")
|
|
|
|
|
|
.uri("/api/local-folder/ocr/jobs")
|
|
|
|
|
|
.header("content-type", "application/json")
|
|
|
|
|
|
.header("x-mnote-actor-id", "user_test")
|
|
|
|
|
|
.header("x-mnote-actor-type", "user")
|
|
|
|
|
|
.body(Body::from(
|
|
|
|
|
|
json!({
|
|
|
|
|
|
"rootUri": root_uri,
|
|
|
|
|
|
"documentId": "local-md:docs~2FPage.md",
|
|
|
|
|
|
"sourceRootRelativePath": "docs/Page.assets/photo.png",
|
|
|
|
|
|
"provider": "mineru"
|
|
|
|
|
|
})
|
|
|
|
|
|
.to_string(),
|
|
|
|
|
|
))
|
|
|
|
|
|
.expect("request"),
|
|
|
|
|
|
)
|
|
|
|
|
|
.await
|
|
|
|
|
|
.expect("response");
|
|
|
|
|
|
|
|
|
|
|
|
if let Some(value) = old_token {
|
|
|
|
|
|
std::env::set_var("MNOTE_MINERU_API_TOKEN", value);
|
|
|
|
|
|
} else {
|
|
|
|
|
|
std::env::remove_var("MNOTE_MINERU_API_TOKEN");
|
|
|
|
|
|
}
|
|
|
|
|
|
if let Some(value) = old_base {
|
|
|
|
|
|
std::env::set_var("MNOTE_MINERU_API_BASE_URL", value);
|
|
|
|
|
|
} else {
|
|
|
|
|
|
std::env::remove_var("MNOTE_MINERU_API_BASE_URL");
|
|
|
|
|
|
}
|
|
|
|
|
|
if let Some(value) = old_interval {
|
|
|
|
|
|
std::env::set_var("MNOTE_MINERU_POLL_INTERVAL_MS", value);
|
|
|
|
|
|
} else {
|
|
|
|
|
|
std::env::remove_var("MNOTE_MINERU_POLL_INTERVAL_MS");
|
|
|
|
|
|
}
|
|
|
|
|
|
if let Some(value) = old_max_polls {
|
|
|
|
|
|
std::env::set_var("MNOTE_MINERU_MAX_POLLS", value);
|
|
|
|
|
|
} else {
|
|
|
|
|
|
std::env::remove_var("MNOTE_MINERU_MAX_POLLS");
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
assert_eq!(response.status(), StatusCode::OK);
|
|
|
|
|
|
let body = to_bytes(response.into_body(), usize::MAX)
|
|
|
|
|
|
.await
|
|
|
|
|
|
.expect("response body");
|
|
|
|
|
|
let payload: Value = serde_json::from_slice(&body).expect("response json");
|
|
|
|
|
|
assert_eq!(payload["job"]["status"].as_str(), Some("done"));
|
|
|
|
|
|
assert_eq!(payload["job"]["provider"].as_str(), Some("mineru"));
|
|
|
|
|
|
assert_eq!(upload_count.load(Ordering::SeqCst), 1);
|
|
|
|
|
|
assert_eq!(poll_count.load(Ordering::SeqCst), 1);
|
2026-06-01 10:07:42 +08:00
|
|
|
|
let sidecar =
|
|
|
|
|
|
fs::read_to_string(root.join("docs").join("Page.ocr").join("photo.png.ocr.md"))
|
|
|
|
|
|
.expect("sidecar");
|
2026-06-01 09:29:12 +08:00
|
|
|
|
assert!(sidecar.contains("provider: mineru"));
|
2026-06-02 17:17:49 +08:00
|
|
|
|
assert!(sidecar.contains(""));
|
2026-06-01 09:29:12 +08:00
|
|
|
|
assert!(sidecar.contains("识别文本"));
|
2026-06-04 18:51:16 +08:00
|
|
|
|
let parse_markdown = fs::read_to_string(
|
|
|
|
|
|
root.join("docs")
|
|
|
|
|
|
.join("Page.ocr")
|
|
|
|
|
|
.join("photo.png.parse.md"),
|
|
|
|
|
|
)
|
|
|
|
|
|
.expect("parse markdown");
|
|
|
|
|
|
assert_eq!(
|
|
|
|
|
|
parse_markdown.trim(),
|
|
|
|
|
|
"# MinerU Result\n\n\n\n识别文本"
|
|
|
|
|
|
);
|
|
|
|
|
|
let source_map = fs::read_to_string(
|
|
|
|
|
|
root.join("docs")
|
|
|
|
|
|
.join("Page.ocr")
|
|
|
|
|
|
.join("photo.png.source-map.json"),
|
|
|
|
|
|
)
|
|
|
|
|
|
.expect("source map");
|
|
|
|
|
|
let source_map_json: Value = serde_json::from_str(&source_map).expect("source map json");
|
|
|
|
|
|
assert_eq!(source_map_json["schema"], RESOURCE_SOURCE_MAP_SCHEMA);
|
|
|
|
|
|
assert_eq!(source_map_json["provider"], "mineru");
|
|
|
|
|
|
assert_eq!(source_map_json["pages"][0]["page"], 1);
|
|
|
|
|
|
assert_eq!(source_map_json["pages"][0]["blocks"][1]["text"], "识别文本");
|
|
|
|
|
|
assert_eq!(source_map_json["pages"][0]["blocks"][1]["bbox"]["x0"], 10.0);
|
|
|
|
|
|
assert_eq!(source_map_json["sections"][0]["path"][0], "MinerU Result");
|
|
|
|
|
|
assert_eq!(source_map_json["sections"][0]["blockIds"][1], "p1_b2");
|
2026-06-02 17:17:49 +08:00
|
|
|
|
assert_eq!(
|
|
|
|
|
|
fs::read(
|
|
|
|
|
|
root.join("docs")
|
|
|
|
|
|
.join("Page.ocr")
|
|
|
|
|
|
.join("images")
|
|
|
|
|
|
.join("ocr.png")
|
|
|
|
|
|
)
|
|
|
|
|
|
.expect("sidecar image"),
|
|
|
|
|
|
b"png-bytes"
|
|
|
|
|
|
);
|
2026-06-01 09:29:12 +08:00
|
|
|
|
|
|
|
|
|
|
mock_handle.abort();
|
|
|
|
|
|
let _ = fs::remove_dir_all(root);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
|
|
fn local_ocr_mineru_runtime_config_uses_safe_defaults_and_env_overrides() {
|
|
|
|
|
|
let old_base = std::env::var("MNOTE_MINERU_API_BASE_URL").ok();
|
|
|
|
|
|
let old_legacy_base = std::env::var("MINERU_API_BASE_URL").ok();
|
|
|
|
|
|
let old_interval = std::env::var("MNOTE_MINERU_POLL_INTERVAL_MS").ok();
|
|
|
|
|
|
let old_max_polls = std::env::var("MNOTE_MINERU_MAX_POLLS").ok();
|
|
|
|
|
|
std::env::remove_var("MNOTE_MINERU_API_BASE_URL");
|
|
|
|
|
|
std::env::remove_var("MINERU_API_BASE_URL");
|
|
|
|
|
|
std::env::remove_var("MNOTE_MINERU_POLL_INTERVAL_MS");
|
|
|
|
|
|
std::env::remove_var("MNOTE_MINERU_MAX_POLLS");
|
|
|
|
|
|
|
|
|
|
|
|
assert_eq!(mineru_api_base_url(), DEFAULT_MINERU_API_BASE_URL);
|
|
|
|
|
|
assert_eq!(mineru_poll_interval(), Duration::from_secs(2));
|
|
|
|
|
|
assert_eq!(mineru_max_polls(), 90);
|
|
|
|
|
|
|
|
|
|
|
|
std::env::set_var("MNOTE_MINERU_API_BASE_URL", "https://mineru.example.test/");
|
|
|
|
|
|
std::env::set_var("MNOTE_MINERU_POLL_INTERVAL_MS", "25");
|
|
|
|
|
|
std::env::set_var("MNOTE_MINERU_MAX_POLLS", "3");
|
|
|
|
|
|
assert_eq!(mineru_api_base_url(), "https://mineru.example.test");
|
|
|
|
|
|
assert_eq!(mineru_poll_interval(), Duration::from_millis(25));
|
|
|
|
|
|
assert_eq!(mineru_max_polls(), 3);
|
|
|
|
|
|
|
|
|
|
|
|
if let Some(value) = old_base {
|
|
|
|
|
|
std::env::set_var("MNOTE_MINERU_API_BASE_URL", value);
|
|
|
|
|
|
} else {
|
|
|
|
|
|
std::env::remove_var("MNOTE_MINERU_API_BASE_URL");
|
|
|
|
|
|
}
|
|
|
|
|
|
if let Some(value) = old_legacy_base {
|
|
|
|
|
|
std::env::set_var("MINERU_API_BASE_URL", value);
|
|
|
|
|
|
} else {
|
|
|
|
|
|
std::env::remove_var("MINERU_API_BASE_URL");
|
|
|
|
|
|
}
|
|
|
|
|
|
if let Some(value) = old_interval {
|
|
|
|
|
|
std::env::set_var("MNOTE_MINERU_POLL_INTERVAL_MS", value);
|
|
|
|
|
|
} else {
|
|
|
|
|
|
std::env::remove_var("MNOTE_MINERU_POLL_INTERVAL_MS");
|
|
|
|
|
|
}
|
|
|
|
|
|
if let Some(value) = old_max_polls {
|
|
|
|
|
|
std::env::set_var("MNOTE_MINERU_MAX_POLLS", value);
|
|
|
|
|
|
} else {
|
|
|
|
|
|
std::env::remove_var("MNOTE_MINERU_MAX_POLLS");
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[tokio::test]
|
|
|
|
|
|
async fn local_ocr_jobs_route_rejects_root_escape_source() {
|
|
|
|
|
|
let root = temp_root("mnote-local-ocr-escape");
|
|
|
|
|
|
write_workspace_manifest(&root);
|
|
|
|
|
|
fs::write(root.join("docs").join("Page.md"), "# Page\n").expect("page");
|
|
|
|
|
|
let root_uri = format!("file://{}", root.display());
|
|
|
|
|
|
let (status, payload) = post_ocr_job(
|
|
|
|
|
|
&root,
|
|
|
|
|
|
json!({
|
|
|
|
|
|
"rootUri": root_uri,
|
|
|
|
|
|
"documentId": "local-md:docs~2FPage.md",
|
|
|
|
|
|
"sourceRootRelativePath": "../outside.png",
|
|
|
|
|
|
"provider": "mock",
|
|
|
|
|
|
"mockMarkdown": "should not write"
|
|
|
|
|
|
}),
|
|
|
|
|
|
)
|
|
|
|
|
|
.await;
|
|
|
|
|
|
assert_eq!(status, StatusCode::BAD_REQUEST);
|
|
|
|
|
|
assert_eq!(payload["code"].as_str(), Some("local_ocr_path_escape"));
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[tokio::test]
|
|
|
|
|
|
async fn local_ocr_jobs_route_rejects_non_image_pdf_source() {
|
|
|
|
|
|
let root = temp_root("mnote-local-ocr-unsupported");
|
|
|
|
|
|
write_workspace_manifest(&root);
|
|
|
|
|
|
fs::write(root.join("docs").join("Page.md"), "# Page\n").expect("page");
|
|
|
|
|
|
fs::write(
|
|
|
|
|
|
root.join("docs").join("Page.assets").join("notes.txt"),
|
|
|
|
|
|
b"text",
|
|
|
|
|
|
)
|
|
|
|
|
|
.expect("text");
|
|
|
|
|
|
let root_uri = format!("file://{}", root.display());
|
|
|
|
|
|
let (status, payload) = post_ocr_job(
|
|
|
|
|
|
&root,
|
|
|
|
|
|
json!({
|
|
|
|
|
|
"rootUri": root_uri,
|
|
|
|
|
|
"documentId": "local-md:docs~2FPage.md",
|
|
|
|
|
|
"sourceRootRelativePath": "docs/Page.assets/notes.txt",
|
|
|
|
|
|
"provider": "mock",
|
|
|
|
|
|
"mockMarkdown": "should not write"
|
|
|
|
|
|
}),
|
|
|
|
|
|
)
|
|
|
|
|
|
.await;
|
|
|
|
|
|
assert_eq!(status, StatusCode::BAD_REQUEST);
|
|
|
|
|
|
assert_eq!(
|
|
|
|
|
|
payload["code"].as_str(),
|
|
|
|
|
|
Some("local_ocr_source_type_unsupported")
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[tokio::test]
|
|
|
|
|
|
async fn local_ocr_jobs_route_writes_failed_mock_entry_with_redacted_error() {
|
|
|
|
|
|
let root = temp_root("mnote-local-ocr-failed");
|
|
|
|
|
|
write_workspace_manifest(&root);
|
|
|
|
|
|
fs::write(root.join("docs").join("Page.md"), "# Page\n").expect("page");
|
|
|
|
|
|
fs::write(
|
|
|
|
|
|
root.join("docs").join("Page.assets").join("photo.png"),
|
|
|
|
|
|
b"png",
|
|
|
|
|
|
)
|
|
|
|
|
|
.expect("photo");
|
|
|
|
|
|
let root_uri = format!("file://{}", root.display());
|
|
|
|
|
|
let response = app()
|
|
|
|
|
|
.oneshot(
|
|
|
|
|
|
Request::builder()
|
|
|
|
|
|
.method("POST")
|
|
|
|
|
|
.uri("/api/local-folder/ocr/jobs")
|
|
|
|
|
|
.header("content-type", "application/json")
|
|
|
|
|
|
.header("x-mnote-actor-id", "user_test")
|
|
|
|
|
|
.header("x-mnote-actor-type", "user")
|
|
|
|
|
|
.body(Body::from(
|
|
|
|
|
|
json!({
|
|
|
|
|
|
"rootUri": root_uri,
|
|
|
|
|
|
"documentId": "local-md:docs~2FPage.md",
|
|
|
|
|
|
"sourceRootRelativePath": "docs/Page.assets/photo.png",
|
|
|
|
|
|
"provider": "mock",
|
|
|
|
|
|
"mockError": "token=secret https://signed.example.test/full.zip?X-Oss-Signature=abc"
|
|
|
|
|
|
})
|
|
|
|
|
|
.to_string(),
|
|
|
|
|
|
))
|
|
|
|
|
|
.expect("request"),
|
|
|
|
|
|
)
|
|
|
|
|
|
.await
|
|
|
|
|
|
.expect("response");
|
|
|
|
|
|
assert_eq!(response.status(), StatusCode::OK);
|
|
|
|
|
|
let body = to_bytes(response.into_body(), usize::MAX)
|
|
|
|
|
|
.await
|
|
|
|
|
|
.expect("response body");
|
|
|
|
|
|
let payload: Value = serde_json::from_slice(&body).expect("response json");
|
|
|
|
|
|
assert_eq!(payload["job"]["status"].as_str(), Some("failed"));
|
|
|
|
|
|
assert_eq!(
|
|
|
|
|
|
payload["job"]["error"].as_str(),
|
|
|
|
|
|
Some("provider_error_redacted")
|
|
|
|
|
|
);
|
|
|
|
|
|
let stored = read_ocr_index_entry(&root, "docs/Page.assets/photo.png")
|
|
|
|
|
|
.expect("read index")
|
|
|
|
|
|
.expect("entry");
|
|
|
|
|
|
assert_eq!(stored.status, "failed");
|
|
|
|
|
|
assert_eq!(stored.error.as_deref(), Some("provider_error_redacted"));
|
|
|
|
|
|
assert!(!root
|
|
|
|
|
|
.join("docs")
|
|
|
|
|
|
.join("Page.ocr")
|
|
|
|
|
|
.join("photo.png.ocr.md")
|
|
|
|
|
|
.exists());
|
|
|
|
|
|
let _ = fs::remove_dir_all(root);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[tokio::test]
|
|
|
|
|
|
async fn local_ocr_insert_route_appends_explicit_ocr_link() {
|
|
|
|
|
|
let root = temp_root("mnote-local-ocr-insert");
|
|
|
|
|
|
write_workspace_manifest(&root);
|
|
|
|
|
|
fs::write(root.join("docs").join("Page.md"), "# Page\n\n正文\n").expect("page");
|
|
|
|
|
|
fs::write(
|
|
|
|
|
|
root.join("docs").join("Page.assets").join("photo.png"),
|
|
|
|
|
|
b"png",
|
|
|
|
|
|
)
|
|
|
|
|
|
.expect("photo");
|
|
|
|
|
|
let root_uri = format!("file://{}", root.display());
|
|
|
|
|
|
let create_response = app()
|
|
|
|
|
|
.oneshot(
|
|
|
|
|
|
Request::builder()
|
|
|
|
|
|
.method("POST")
|
|
|
|
|
|
.uri("/api/local-folder/ocr/jobs")
|
|
|
|
|
|
.header("content-type", "application/json")
|
|
|
|
|
|
.header("x-mnote-actor-id", "user_test")
|
|
|
|
|
|
.header("x-mnote-actor-type", "user")
|
|
|
|
|
|
.body(Body::from(
|
|
|
|
|
|
json!({
|
|
|
|
|
|
"rootUri": root_uri,
|
|
|
|
|
|
"documentId": "local-md:docs~2FPage.md",
|
|
|
|
|
|
"sourceRootRelativePath": "docs/Page.assets/photo.png",
|
|
|
|
|
|
"provider": "mock",
|
|
|
|
|
|
"mockMarkdown": "识别文本"
|
|
|
|
|
|
})
|
|
|
|
|
|
.to_string(),
|
|
|
|
|
|
))
|
|
|
|
|
|
.expect("request"),
|
|
|
|
|
|
)
|
|
|
|
|
|
.await
|
|
|
|
|
|
.expect("create response");
|
|
|
|
|
|
assert_eq!(create_response.status(), StatusCode::OK);
|
|
|
|
|
|
|
|
|
|
|
|
let insert_response = app()
|
|
|
|
|
|
.oneshot(
|
|
|
|
|
|
Request::builder()
|
|
|
|
|
|
.method("POST")
|
|
|
|
|
|
.uri("/api/local-folder/ocr/insert")
|
|
|
|
|
|
.header("content-type", "application/json")
|
|
|
|
|
|
.header("x-mnote-actor-id", "user_test")
|
|
|
|
|
|
.header("x-mnote-actor-type", "user")
|
|
|
|
|
|
.body(Body::from(
|
|
|
|
|
|
json!({
|
|
|
|
|
|
"rootUri": root_uri,
|
|
|
|
|
|
"documentId": "local-md:docs~2FPage.md",
|
|
|
|
|
|
"ocrRootRelativePath": "docs/Page.ocr/photo.png.ocr.md",
|
|
|
|
|
|
"mode": "link"
|
|
|
|
|
|
})
|
|
|
|
|
|
.to_string(),
|
|
|
|
|
|
))
|
|
|
|
|
|
.expect("request"),
|
|
|
|
|
|
)
|
|
|
|
|
|
.await
|
|
|
|
|
|
.expect("insert response");
|
|
|
|
|
|
assert_eq!(insert_response.status(), StatusCode::OK);
|
|
|
|
|
|
let body = to_bytes(insert_response.into_body(), usize::MAX)
|
|
|
|
|
|
.await
|
|
|
|
|
|
.expect("insert body");
|
|
|
|
|
|
let payload: Value = serde_json::from_slice(&body).expect("insert json");
|
|
|
|
|
|
assert_eq!(
|
|
|
|
|
|
payload["insertedMarkdown"].as_str(),
|
|
|
|
|
|
Some("[OCR:photo.png](./Page.ocr/photo.png.ocr.md)")
|
|
|
|
|
|
);
|
|
|
|
|
|
let owner = fs::read_to_string(root.join("docs").join("Page.md")).expect("owner");
|
|
|
|
|
|
assert_eq!(
|
|
|
|
|
|
owner,
|
|
|
|
|
|
"# Page\n\n正文\n\n[OCR:photo.png](./Page.ocr/photo.png.ocr.md)\n"
|
|
|
|
|
|
);
|
|
|
|
|
|
let _ = fs::remove_dir_all(root);
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|