推进 OCR UI 与 provider 删除验收
This commit is contained in:
@@ -199,8 +199,13 @@ pub(crate) async fn create_job(
|
||||
match run_mineru_ocr(&root, &plan, token.unwrap_or_default()).await {
|
||||
Ok(markdown) => markdown,
|
||||
Err(error) => {
|
||||
let failed_entry =
|
||||
build_index_entry(&plan, "failed", now, "", Some(redact_error(error.message())));
|
||||
let failed_entry = build_index_entry(
|
||||
&plan,
|
||||
"failed",
|
||||
now,
|
||||
"",
|
||||
Some(redact_error(error.message())),
|
||||
);
|
||||
upsert_ocr_index_entry(&root, failed_entry)?;
|
||||
return Err(error.with_context(&context));
|
||||
}
|
||||
@@ -457,7 +462,11 @@ pub(crate) fn strip_ocr_frontmatter(markdown: &str) -> &str {
|
||||
trimmed[end + 4..].trim_start_matches('\n')
|
||||
}
|
||||
|
||||
async fn run_mineru_ocr(root: &Path, plan: &OcrSidecarPlan, token: String) -> Result<String, WebError> {
|
||||
async fn run_mineru_ocr(
|
||||
root: &Path,
|
||||
plan: &OcrSidecarPlan,
|
||||
token: String,
|
||||
) -> Result<String, WebError> {
|
||||
let config = MineruClientConfig {
|
||||
api_base_url: mineru_api_base_url(),
|
||||
token,
|
||||
@@ -515,7 +524,10 @@ async fn create_mineru_upload_task(
|
||||
.map_err(|error| {
|
||||
WebError::bad_gateway_code(
|
||||
"mineru_upload_task_failed",
|
||||
format!("MinerU 上传任务创建失败: {}", redact_error(&error.to_string())),
|
||||
format!(
|
||||
"MinerU 上传任务创建失败: {}",
|
||||
redact_error(&error.to_string())
|
||||
),
|
||||
)
|
||||
})?;
|
||||
let status = response.status();
|
||||
@@ -527,8 +539,8 @@ async fn create_mineru_upload_task(
|
||||
));
|
||||
}
|
||||
let value = parse_mineru_json(&body, "mineru_upload_task_json_invalid")?;
|
||||
let batch_id = find_json_string_by_keys(&value, &["batch_id", "batchId", "id"])
|
||||
.ok_or_else(|| {
|
||||
let batch_id =
|
||||
find_json_string_by_keys(&value, &["batch_id", "batchId", "id"]).ok_or_else(|| {
|
||||
WebError::bad_gateway_code(
|
||||
"mineru_upload_task_batch_missing",
|
||||
"MinerU 上传任务响应缺少 batch_id",
|
||||
@@ -559,7 +571,10 @@ async fn upload_mineru_source(
|
||||
.map_err(|error| {
|
||||
WebError::bad_gateway_code(
|
||||
"mineru_source_upload_failed",
|
||||
format!("MinerU 源文件上传失败: {}", redact_error(&error.to_string())),
|
||||
format!(
|
||||
"MinerU 源文件上传失败: {}",
|
||||
redact_error(&error.to_string())
|
||||
),
|
||||
)
|
||||
})?;
|
||||
if !response.status().is_success() {
|
||||
@@ -610,7 +625,14 @@ async fn poll_mineru_result_zip_url(
|
||||
}
|
||||
if let Some(zip_url) = find_json_string_by_keys(
|
||||
&value,
|
||||
&["full_zip_url", "fullZipUrl", "zip_url", "zipUrl", "result_url", "resultUrl"],
|
||||
&[
|
||||
"full_zip_url",
|
||||
"fullZipUrl",
|
||||
"zip_url",
|
||||
"zipUrl",
|
||||
"result_url",
|
||||
"resultUrl",
|
||||
],
|
||||
) {
|
||||
return Ok(zip_url);
|
||||
}
|
||||
@@ -629,7 +651,10 @@ async fn download_mineru_result_zip(
|
||||
let response = client.get(zip_url).send().await.map_err(|error| {
|
||||
WebError::bad_gateway_code(
|
||||
"mineru_result_download_failed",
|
||||
format!("MinerU 结果包下载失败: {}", redact_error(&error.to_string())),
|
||||
format!(
|
||||
"MinerU 结果包下载失败: {}",
|
||||
redact_error(&error.to_string())
|
||||
),
|
||||
)
|
||||
})?;
|
||||
if !response.status().is_success() {
|
||||
@@ -638,12 +663,19 @@ async fn download_mineru_result_zip(
|
||||
format!("MinerU 结果包下载失败: HTTP {}", response.status().as_u16()),
|
||||
));
|
||||
}
|
||||
response.bytes().await.map(|bytes| bytes.to_vec()).map_err(|error| {
|
||||
WebError::bad_gateway_code(
|
||||
"mineru_result_download_failed",
|
||||
format!("MinerU 结果包读取失败: {}", redact_error(&error.to_string())),
|
||||
)
|
||||
})
|
||||
response
|
||||
.bytes()
|
||||
.await
|
||||
.map(|bytes| bytes.to_vec())
|
||||
.map_err(|error| {
|
||||
WebError::bad_gateway_code(
|
||||
"mineru_result_download_failed",
|
||||
format!(
|
||||
"MinerU 结果包读取失败: {}",
|
||||
redact_error(&error.to_string())
|
||||
),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn extract_mineru_markdown_from_zip(bytes: &[u8]) -> Result<String, WebError> {
|
||||
@@ -678,7 +710,8 @@ fn extract_mineru_markdown_from_zip(bytes: &[u8]) -> Result<String, WebError> {
|
||||
candidates
|
||||
.into_iter()
|
||||
.max_by_key(|(name, markdown)| {
|
||||
let preferred = name.ends_with("/full.md") || name == "full.md" || name.ends_with("/result.md");
|
||||
let preferred =
|
||||
name.ends_with("/full.md") || name == "full.md" || name.ends_with("/result.md");
|
||||
(preferred, markdown.len())
|
||||
})
|
||||
.map(|(_, markdown)| markdown)
|
||||
@@ -1307,9 +1340,7 @@ mod tests {
|
||||
writer
|
||||
.start_file("full.md", zip::write::SimpleFileOptions::default())
|
||||
.expect("zip start file");
|
||||
writer
|
||||
.write_all(markdown.as_bytes())
|
||||
.expect("zip markdown");
|
||||
writer.write_all(markdown.as_bytes()).expect("zip markdown");
|
||||
writer.finish().expect("zip finish");
|
||||
}
|
||||
bytes.into_inner()
|
||||
@@ -1570,7 +1601,9 @@ mod tests {
|
||||
}),
|
||||
);
|
||||
let mock_handle = tokio::spawn(async move {
|
||||
axum::serve(listener, mock_mineru).await.expect("mock mineru server");
|
||||
axum::serve(listener, mock_mineru)
|
||||
.await
|
||||
.expect("mock mineru server");
|
||||
});
|
||||
|
||||
let old_token = std::env::var("MNOTE_MINERU_API_TOKEN").ok();
|
||||
@@ -1643,12 +1676,9 @@ mod tests {
|
||||
assert_eq!(payload["job"]["provider"].as_str(), Some("mineru"));
|
||||
assert_eq!(upload_count.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(poll_count.load(Ordering::SeqCst), 1);
|
||||
let sidecar = fs::read_to_string(
|
||||
root.join("docs")
|
||||
.join("Page.ocr")
|
||||
.join("photo.png.ocr.md"),
|
||||
)
|
||||
.expect("sidecar");
|
||||
let sidecar =
|
||||
fs::read_to_string(root.join("docs").join("Page.ocr").join("photo.png.ocr.md"))
|
||||
.expect("sidecar");
|
||||
assert!(sidecar.contains("provider: mineru"));
|
||||
assert!(sidecar.contains("识别文本"));
|
||||
|
||||
|
||||
@@ -185,6 +185,18 @@ pub(crate) fn refresh_local_search_index_for_path(
|
||||
index
|
||||
.resources
|
||||
.retain(|resource| resource.path != relative_path);
|
||||
if local_ocr::is_local_ocr_sidecar_relative_path(&relative_path) {
|
||||
index.built_at = now_ms();
|
||||
write_local_search_index(root_path, &index)?;
|
||||
return Ok(json!({
|
||||
"version": index.version,
|
||||
"rootUri": index.root_uri,
|
||||
"workspaceId": index.workspace_id,
|
||||
"builtAt": index.built_at,
|
||||
"documentCount": index.documents.len(),
|
||||
"resourceCount": index.resources.len()
|
||||
}));
|
||||
}
|
||||
let absolute_path = root_path.join(&relative_path);
|
||||
if absolute_path.exists() && absolute_path.is_file() && is_markdown_path(&absolute_path) {
|
||||
index
|
||||
@@ -1168,6 +1180,11 @@ mod tests {
|
||||
"---\nmnote_ocr_version: 1\nprovider: mock\nmodel_version: vlm\nowner_document: ../Page.md\nsource_path: ./Page.assets/photo.png\nsource_root_relative_path: docs/Page.assets/photo.png\nsource_size: 3\nsource_mtime_ms: 1\nstatus: done\ncreated_at: 1\nupdated_at: 1\n---\n\nOCR-only-token 识别正文\n",
|
||||
)
|
||||
.expect("ocr markdown");
|
||||
fs::write(
|
||||
root.join("docs").join("Page.ocr").join("photo.png-704905.ocr.md"),
|
||||
"---\nmnote_ocr_version: 1\nprovider: mock\nmodel_version: vlm\nowner_document: ../Page.md\nsource_path: ./Page.assets/photo.png\nsource_root_relative_path: docs/Page.assets/photo.png\nsource_size: 3\nsource_mtime_ms: 2\nstatus: done\ncreated_at: 2\nupdated_at: 2\n---\n\nOCR-hash-token 识别正文\n",
|
||||
)
|
||||
.expect("hashed ocr markdown");
|
||||
fs::create_dir_all(root.join(".mnote")).expect("mnote dir");
|
||||
fs::write(
|
||||
root.join(".mnote").join("ocr-index.json"),
|
||||
@@ -1208,6 +1225,22 @@ mod tests {
|
||||
)
|
||||
.expect("without ocr");
|
||||
assert_eq!(without_ocr["results"].as_array().map(Vec::len), Some(0));
|
||||
let without_hashed_ocr = query_local_search_index(
|
||||
&root,
|
||||
&root_uri,
|
||||
workspace_id,
|
||||
"OCR-hash-token",
|
||||
None,
|
||||
10,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
)
|
||||
.expect("without hashed ocr");
|
||||
assert_eq!(
|
||||
without_hashed_ocr["results"].as_array().map(Vec::len),
|
||||
Some(0)
|
||||
);
|
||||
|
||||
let with_ocr = query_local_search_index(
|
||||
&root,
|
||||
@@ -1245,6 +1278,24 @@ mod tests {
|
||||
.documents
|
||||
.iter()
|
||||
.any(|document| document.path == "docs/Page.ocr/photo.png.ocr.md"));
|
||||
assert!(!index
|
||||
.documents
|
||||
.iter()
|
||||
.any(|document| document.path == "docs/Page.ocr/photo.png-704905.ocr.md"));
|
||||
refresh_local_search_index_for_path(
|
||||
&root,
|
||||
&root_uri,
|
||||
workspace_id,
|
||||
"docs/Page.ocr/photo.png-704905.ocr.md",
|
||||
)
|
||||
.expect("refresh hashed ocr sidecar");
|
||||
let refreshed_index = read_local_search_index(&root)
|
||||
.expect("read refreshed search index")
|
||||
.expect("refreshed search index");
|
||||
assert!(!refreshed_index
|
||||
.documents
|
||||
.iter()
|
||||
.any(|document| document.path == "docs/Page.ocr/photo.png-704905.ocr.md"));
|
||||
let _ = fs::remove_dir_all(&root);
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user