From 1d58b9332b577e000945fdcd2653abc7764ab1f3 Mon Sep 17 00:00:00 2001 From: Steven Huang <134844631+remote-controlled-man@users.noreply.github.com> Date: Wed, 11 Nov 2026 16:02:03 +0800 Subject: [PATCH] fix(cache): reparse documents when cached media files are missing --- raganything/processor.py | 16 +++++ tests/testparse_cache_key.py | 116 +++++++++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+) diff --git a/raganything/processor.py b/raganything/processor.py index d9ad71f8a..3526b07ee 100644 --- a/raganything/processor.py +++ b/raganything/processor.py @@ -13,6 +13,7 @@ from typing import Dict, List, Any, Tuple, Optional from pathlib import Path +from raganything.asset_urls import MEDIA_PATH_FIELDS from raganything.base import DocStatus from raganything.document_context import document_context from raganything.parser import ( @@ -543,6 +544,21 @@ async def _get_cached_result( doc_id = cached_data.get("doc_id") if content_list and doc_id: + # Cached blocks reference local parser outputs, not their bytes. + for item in content_list: + if not isinstance(item, dict): + continue + for field in (*MEDIA_PATH_FIELDS, "audio_path", "video_path"): + media_path = item.get(field) + if not isinstance(media_path, str) or not media_path: + continue + if media_path.startswith(("http://", "https://", "s3://")): + continue + if not Path(media_path).is_file(): + self.logger.debug( + f"Cache invalid - missing media file: {media_path}" + ) + return None self.logger.debug( f"Found valid cached parsing result for key: {cache_key}" ) diff --git a/tests/testparse_cache_key.py b/tests/testparse_cache_key.py index 1fe04d39f..71432dd3b 100644 --- a/tests/testparse_cache_key.py +++ b/tests/testparse_cache_key.py @@ -216,3 +216,119 @@ def run_mineru(input_path, output_dir, method="auto", vlm_url=None, **kwargs): [entry["parse_config"] for entry in processor.parse_cache.entries.values()] ) assert "secret" not in persisted and "vlm-a" not in persisted + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "media_type,path_field", + [ + ("image", "img_path"), + ("table", "table_img_path"), + ("equation", "equation_img_path"), + ("audio", "audio_path"), + ("video", "video_path"), + ], +) +@pytest.mark.parametrize("missing_parent", [False, True]) +async def test_parse_cache_recreates_missing_media( + tmp_path, media_type, path_field, missing_parent +): + processor = _make_processor(tmp_path) + document = tmp_path / "document.txt" + document.write_text("source document", encoding="utf-8") + asset = tmp_path / "output" / "asset.bin" + + class AssetParser: + calls = 0 + + def parse_document(self, file_path, **kwargs): + self.calls += 1 + asset.parent.mkdir(parents=True, exist_ok=True) + asset.write_bytes(b"parsed media") + return [ + { + "type": media_type, + path_field: str(asset), + path_field + "_public_url": "https://cdn.example.com/asset.bin", + } + ] + + processor.doc_parser = AssetParser() + first = await processor.parse_document(str(document)) + assert await processor.parse_document(str(document)) == first + assert processor.doc_parser.calls == 1 + + asset.unlink() + if missing_parent: + asset.parent.rmdir() + + restored = await processor.parse_document(str(document)) + + assert asset.is_file() + assert processor.doc_parser.calls == 2 + assert restored == first + assert await processor.parse_document(str(document)) == restored + assert processor.doc_parser.calls == 2 + + +@pytest.mark.asyncio +async def test_mineru_parse_cache_recreates_removed_image(tmp_path, monkeypatch): + processor = _make_processor(tmp_path) + processor.config.parser = "mineru" + processor.doc_parser = MineruParser() + calls = [] + + def run_mineru(input_path, output_dir, method="auto", **kwargs): + calls.append(input_path) + stem = Path(input_path).stem + out = Path(output_dir) / stem / method + (out / "images").mkdir(parents=True, exist_ok=True) + (out / "images" / "figure.png").write_bytes(b"parsed image") + (out / f"{stem}_content_list.json").write_text( + json.dumps([{"type": "image", "img_path": "images/figure.png"}]), + encoding="utf-8", + ) + + monkeypatch.setattr(processor.doc_parser, "_run_mineru_command", run_mineru) + document = tmp_path / "document.pdf" + document.write_bytes(b"synthetic PDF for a stubbed MinerU process") + first = await processor.parse_document(str(document)) + image = Path(first[0][0]["img_path"]) + assert image.is_file() + assert await processor.parse_document(str(document)) == first + assert len(calls) == 1 + + image.unlink() + restored = await processor.parse_document(str(document)) + + assert image.is_file() + assert restored == first + assert len(calls) == 2 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "media_path", + [ + "https://cdn.example.com/figure.png", + "http://cdn.example.com/figure.png", + "s3://bucket/figure.png", + ], +) +async def test_parse_cache_preserves_remote_media_references(tmp_path, media_path): + processor = _make_processor(tmp_path) + document = tmp_path / "document.txt" + document.write_text("source document", encoding="utf-8") + + class RemoteParser: + calls = 0 + + def parse_document(self, file_path, **kwargs): + self.calls += 1 + return [{"type": "image", "img_path": media_path}] + + processor.doc_parser = RemoteParser() + first = await processor.parse_document(str(document)) + + assert await processor.parse_document(str(document)) == first + assert processor.doc_parser.calls == 1