from email.mime import image
from hashlib import md5
import csv
import json
import os
import httpx
import ssl
import trace
import zipfile
import PyPDF2
import base64
import requests
from concurrent.futures import ThreadPoolExecutor
import time
import re
import pandas as pd
from docx import Document
import subprocess
import tempfile
import textract
import shutil  # 添加导入
from pptx import Presentation  # 添加导入用于处理pptx文件
import fitz  # PyMuPDF，用于PDF图片截取
from PIL import Image, ImageOps, ImageStat  # 用于图片处理
import io
from html.parser import HTMLParser
import database
import value
import mistral_ocr_balancer
import gc
import traceback
from docx import Document
from docx.oxml.ns import qn
from xml.etree import ElementTree as ET
import agents
from libreoffice_pool import libreoffice_pool, convert_with_retry
import chardet
from typing import Optional

os.environ.setdefault("HTTPX_FORCE_HTTP1", "1")
ocr_balancer = None
deepseek_ocr_balancer = None
paddleocrvl_balancer = None


class TransientParseError(RuntimeError):
    """瞬时解析失败（OCR 限流/网络抖动等）。

    必须区别于"请检查文件格式是否正确"这类源文档错误：后者会被重试链路
    直接软删除任务（不再重试），瞬时失败则应标记 failed 走正常重试。
    """
    pass


class SourceDocumentParseError(RuntimeError):
    """源文件本身无法解析，不应进入 OCR 瞬时重试链路。"""
    pass


def _ocr_provider() -> str:
    return os.getenv("TASK_OCR_PROVIDER", "mistral").strip().lower()


def _ocr_fallback_provider() -> str:
    return os.getenv("TASK_OCR_FALLBACK_PROVIDER", "").strip().lower()


def _is_mistral_ocr_provider(provider: str) -> bool:
    return provider in {
        "mistral",
        "mistral_ocr",
        "mistralocr",
        "mistral-ai",
        "mistral_ai",
        "mistral_ocr_lite",
        "mistral_ocr_latest",
    }


def _get_mistral_ocr_balancer():
    global ocr_balancer
    if ocr_balancer is None:
        ocr_balancer = mistral_ocr_balancer.get_shared_balancer()
    return ocr_balancer


def _env_bool(name: str, default: bool = False) -> bool:
    value = os.getenv(name)
    if value is None:
        return default
    return value.strip().lower() in {"1", "true", "yes", "y", "on"}

def _env_int(name: str, default: int) -> int:
    value = os.getenv(name)
    if value is None or value.strip() == "":
        return default
    try:
        return int(value)
    except ValueError:
        return default


def _env_float(name: str, default: float) -> float:
    value = os.getenv(name)
    if value is None or value.strip() == "":
        return default
    try:
        return float(value)
    except ValueError:
        return default


def _clamp_float(value: float, low: float, high: float) -> float:
    return max(low, min(high, value))


def _call_mistral_ocr(client, document, include_image_base64=False):
    kwargs = {
        "model": "mistral-ocr-latest",
        "document": document,
    }
    if include_image_base64:
        kwargs["include_image_base64"] = True
    try:
        return client.ocr.process(**kwargs)
    except TypeError:
        if include_image_base64:
            kwargs.pop("include_image_base64", None)
            return client.ocr.process(**kwargs)
        raise


def _strip_data_url_base64(image_base64):
    image_base64 = str(image_base64 or "")
    if image_base64.startswith("data:") and "," in image_base64:
        return image_base64.split(",", 1)[1]
    return image_base64


def _pdf_image_marker_variants(image_id):
    marker = str(image_id or "").strip()
    if not marker:
        return []
    variants = []
    for value_item in (
        marker,
        os.path.basename(marker),
        f"mistral://{marker}",
        f"mistral-ocr://{marker}",
        f"ocr://{marker}",
    ):
        if value_item and value_item not in variants:
            variants.append(value_item)
    base, ext = os.path.splitext(marker)
    if base and not ext:
        for suffix in (".jpeg", ".jpg", ".png"):
            candidate = marker + suffix
            if candidate not in variants:
                variants.append(candidate)
    return variants


def _replace_pdf_image_markers(markdown, image_id, image_url):
    text = str(markdown or "")
    marker_variants = set(_pdf_image_marker_variants(image_id))
    if not marker_variants:
        return text

    replaced = False

    def _replace_match(match):
        nonlocal replaced
        alt_text = match.group(1) or str(image_id or "image")
        target = (match.group(2) or "").strip()
        if target in marker_variants or os.path.basename(target) in marker_variants:
            replaced = True
            return f"![{alt_text}]({image_url})"
        return match.group(0)

    text = re.sub(r"!\[([^\]]*)\]\(([^)]*)\)", _replace_match, text)
    for marker in marker_variants:
        wrapped = f"({marker})"
        if wrapped in text:
            text = text.replace(wrapped, f"({image_url})")
            replaced = True

    if not replaced and image_url not in text:
        text = text.rstrip() + f"\n\n![{image_id}]({image_url})\n"
    return text


def _pdf_image_storage_dir(pdf_md5, output_dir=None):
    if output_dir:
        return os.path.abspath(output_dir)
    return os.path.abspath(
        os.path.join(os.path.dirname(__file__), "..", "..", "storage", "pdf_images", pdf_md5)
    )


def _normalize_pdf_image_bytes(image_bytes, *, max_side=2400, quality=90):
    with Image.open(io.BytesIO(image_bytes)) as img:
        img = ImageOps.exif_transpose(img)
        width, height = img.size

        if img.mode not in ("RGB", "RGBA", "L", "CMYK"):
            img = img.convert("RGBA" if "A" in img.getbands() else "RGB")
        if img.mode == "RGBA":
            background = Image.new("RGB", img.size, (255, 255, 255))
            background.paste(img, mask=img.getchannel("A"))
            img = background
        elif img.mode in ("L", "CMYK"):
            img = img.convert("RGB")

        if max(img.width, img.height) > max_side:
            resampling = getattr(Image, "Resampling", Image)
            resample = getattr(resampling, "LANCZOS", getattr(Image, "LANCZOS", 1))
            img.thumbnail((max_side, max_side), resample)

        buffer = io.BytesIO()
        img.save(buffer, format="JPEG", quality=quality, optimize=True)
        return buffer.getvalue(), {"width": width, "height": height, "format": "jpeg"}


def _save_pdf_image_bytes(
    pdf_md5,
    output_dir,
    image_filename,
    image_bytes,
    *,
    page_number,
    chunk_index,
    image_id,
    file_remark="",
    coordinates=None,
    dimensions=None,
):
    if not output_dir:
        return None
    os.makedirs(output_dir, exist_ok=True)
    try:
        normalized_bytes, image_info = _normalize_pdf_image_bytes(image_bytes)
    except Exception as img_error:
        print(f"PDF图片标准化失败: {img_error}")
        return None

    image_path = os.path.join(output_dir, image_filename)
    with open(image_path, "wb") as img_file:
        img_file.write(normalized_bytes)

    image_size = len(normalized_bytes)
    image_md5 = md5(normalized_bytes).hexdigest()
    obj_key = f"/pdf_images/{pdf_md5}/{image_filename}"

    try:
        exist_file = database.get_file_by_obj_key(obj_key)
        if exist_file is None:
            database.insert_file(
                f"storage/pdf_images/{pdf_md5}/{image_filename}",
                obj_key,
                image_filename,
                "jpeg",
                image_size,
                image_md5,
                None,
                False,
                file_remark=file_remark,
            )
    except Exception as e:
        print(f"插入文件记录失败: {e}")
        traceback.print_exc()

    return {
        "image_id": image_id,
        "page_number": page_number,
        "chunk_index": chunk_index,
        "image_path": image_path,
        "url": value.storageUrl + obj_key,
        "obj_key": obj_key,
        "md5": image_md5,
        "coordinates": coordinates or {
            "top_left_x": 0,
            "top_left_y": 0,
            "bottom_right_x": 0,
            "bottom_right_y": 0,
        },
        "dimensions": dimensions or image_info,
    }


def _append_pdf_embedded_images(
    pdf_path,
    chunk_index,
    pdf_md5,
    output_dir,
    pages,
    extracted_images,
    *,
    file_remark="",
    image_stats=None,
):
    if not output_dir or fitz is None:
        return

    min_side = max(1, _env_int("PDF_EMBEDDED_IMAGE_MIN_SIDE", 24))
    min_area = max(1, _env_int("PDF_EMBEDDED_IMAGE_MIN_AREA", 1024))
    existing_hashes = set()
    for image in extracted_images or []:
        if image.get("md5"):
            existing_hashes.add(image["md5"])
            continue
        image_path = image.get("image_path")
        if image_path and os.path.exists(image_path):
            try:
                with open(image_path, "rb") as fp:
                    existing_bytes = fp.read()
                normalized_bytes, _ = _normalize_pdf_image_bytes(existing_bytes)
                existing_hashes.add(md5(normalized_bytes).hexdigest())
            except Exception:
                pass

    page_by_index = {}
    for fallback_index, page in enumerate(pages or []):
        try:
            page_index = int(getattr(page, "index", fallback_index))
        except Exception:
            page_index = fallback_index
        page_by_index[page_index] = page

    pdf_document = None
    try:
        pdf_document = fitz.open(pdf_path)
        saved_count = 0
        for page_index, page in page_by_index.items():
            if page_index < 0 or page_index >= len(pdf_document):
                continue
            pdf_page = pdf_document.load_page(page_index)
            seen_xrefs = set()
            for image_pos, image_entry in enumerate(pdf_page.get_images(full=True) or [], start=1):
                if not image_entry:
                    continue
                xref = image_entry[0]
                if xref in seen_xrefs:
                    continue
                seen_xrefs.add(xref)

                try:
                    base_image = pdf_document.extract_image(xref)
                    image_bytes = base_image.get("image")
                except Exception as extract_error:
                    print(f"PDF内嵌图片提取失败 page={page_index} xref={xref}: {extract_error}")
                    continue
                if not image_bytes:
                    continue

                try:
                    normalized_bytes, image_info = _normalize_pdf_image_bytes(image_bytes)
                except Exception:
                    continue
                width = int(image_info.get("width") or 0)
                height = int(image_info.get("height") or 0)
                if min(width, height) < min_side or width * height < min_area:
                    continue

                image_hash = md5(normalized_bytes).hexdigest()
                if image_hash in existing_hashes:
                    continue

                rects = []
                try:
                    rects = pdf_page.get_image_rects(xref) or []
                except Exception:
                    rects = []
                rect = rects[0] if rects else None
                coordinates = {
                    "top_left_x": rect.x0 if rect else 0,
                    "top_left_y": rect.y0 if rect else 0,
                    "bottom_right_x": rect.x1 if rect else 0,
                    "bottom_right_y": rect.y1 if rect else 0,
                }
                image_id = f"embedded-{xref}-{image_pos}"
                image_filename = f"page_{page_index}_chunk_{chunk_index}_embedded_{xref}_{image_pos}.jpeg"
                saved = _save_pdf_image_bytes(
                    pdf_md5,
                    output_dir,
                    image_filename,
                    image_bytes,
                    page_number=page_index,
                    chunk_index=chunk_index,
                    image_id=image_id,
                    file_remark=file_remark,
                    coordinates=coordinates,
                    dimensions=image_info,
                )
                if not saved:
                    continue

                extracted_images.append(saved)
                existing_hashes.add(saved.get("md5") or image_hash)
                saved_count += 1
                image_url = saved["url"]
                page_markdown = str(getattr(page, "markdown", "") or "")
                if image_url not in page_markdown:
                    page.markdown = page_markdown.rstrip() + f"\n\n![{image_id}]({image_url})\n"
                if image_stats is not None:
                    image_stats["total"] = image_stats.get("total", 0) + 1
                    image_stats["success"] = image_stats.get("success", 0) + 1
                    image_stats["embedded_success"] = image_stats.get("embedded_success", 0) + 1
        if saved_count:
            print(f"PDF内嵌图片兜底提取: 保存 {saved_count} 张")
    except Exception as e:
        print(f"PDF内嵌图片兜底提取失败: {e}")
        traceback.print_exc()
    finally:
        try:
            if pdf_document is not None:
                pdf_document.close()
        except Exception:
            pass

IMAGE_PARSE_EXTENSIONS = {
    ".jpg",
    ".jpeg",
    ".png",
    ".gif",
    ".bmp",
    ".webp",
    ".tif",
    ".tiff",
}


def _is_image_file(file_path: str) -> bool:
    _, ext = os.path.splitext(file_path or "")
    return ext.lower() in IMAGE_PARSE_EXTENSIONS


def _normalize_image_file(file_path: str, *, max_side: int = 2048, quality: int = 88):
    """
    将任意上传图片转为统一的 JPEG data URL，便于调用视觉模型。

    返回:
        (data_url, info_dict)
    """
    with Image.open(file_path) as img:
        img = ImageOps.exif_transpose(img)
        info = {
            "original_mode": img.mode,
            "width": img.width,
            "height": img.height,
            "format": (img.format or "").upper(),
        }

        if img.mode not in ("RGB", "RGBA", "L"):
            img = img.convert("RGBA" if "A" in img.getbands() else "RGB")

        if img.mode == "RGBA":
            background = Image.new("RGB", img.size, (255, 255, 255))
            alpha = img.getchannel("A")
            background.paste(img, mask=alpha)
            img = background
        elif img.mode == "L":
            img = img.convert("RGB")
        elif img.mode != "RGB":
            img = img.convert("RGB")

        if max(img.width, img.height) > max_side:
            resampling = getattr(Image, "Resampling", Image)
            resample = getattr(resampling, "LANCZOS", getattr(Image, "LANCZOS", 1))
            img.thumbnail((max_side, max_side), resample)

        buffer = io.BytesIO()
        img.save(buffer, format="JPEG", quality=quality, optimize=True)
        jpeg_bytes = buffer.getvalue()
        image_data_url = "data:image/jpeg;base64," + base64.b64encode(jpeg_bytes).decode("utf-8")
        info.update({
            "normalized_format": "JPEG",
            "normalized_width": img.width,
            "normalized_height": img.height,
            "normalized_size": len(jpeg_bytes),
        })
        return image_data_url, info


def _storage_root():
    return os.path.abspath(os.getenv(
        "KNOWLEDGE_STORAGE_ROOT",
        os.path.join(os.path.dirname(__file__), "..", "..", "storage"),
    ))


def _image_url_to_local_path(image_url: str):
    if not image_url or image_url.startswith("data:"):
        return None
    try:
        from urllib.parse import urlparse

        parsed = urlparse(image_url)
    except Exception:
        parsed = None

    path = parsed.path if parsed and parsed.scheme else image_url
    if not path:
        return None
    if path.startswith("/storage/"):
        path = path[len("/storage/"):]
    elif path.startswith("storage/"):
        path = path[len("storage/"):]
    elif path.startswith("/"):
        return path if os.path.exists(path) else None
    else:
        return image_url if os.path.exists(image_url) else None

    candidate = os.path.abspath(os.path.join(_storage_root(), path.lstrip("/")))
    if os.path.exists(candidate):
        return candidate
    return None


def _summarize_markdown_image(url: str, context: str, title: str):
    image_payload = url
    local_path = _image_url_to_local_path(url)
    if local_path:
        image_payload, _ = _normalize_image_file(local_path)
    image_title, image_description = agents.image_summary(
        image_payload,
        context=context[:4000],
        local=False,
        prompt=(
            "你是一个文档图片解析器。请结合图片附近的文本上下文，描述这张图片的可见内容。"
            "如果图片中包含人像、截图、图表、表格、徽标或文字，请尽量提取关键可见信息；"
            "不要编造看不见的内容。直接返回 JSON："
            "{\"title\":\"图片标题\",\"description\":\"图片详细内容概括\"}。\n\n"
            f"附近文本上下文:\n{context[:4000]}"
        ),
    )
    return (image_title or title or "图片").strip(), (image_description or "").strip()


def _describe_markdown_images_for_direct_ocr(text):
    image_links = re.findall(r"!\[([^\]]*)\]\(([^)]+)\)", text or "")
    if not image_links:
        return text

    processed_text = text
    for title, url in image_links:
        pos = processed_text.find(url)
        if "(图片描述:" in processed_text[max(0, pos - 120): pos + len(url) + 120]:
            continue
        try:
            image_title, image_description = _summarize_markdown_image(
                url,
                processed_text,
                title or "图片",
            )
        except Exception as e:
            print(f"直接OCR图片描述失败: {e}")
            image_title, image_description = title or "图片", ""

        if image_title and len(image_title) > 120:
            image_title = image_title[:120]
        if image_description and len(image_description) > 1200:
            image_description = image_description[:1200]
        if not image_description:
            image_description = "无法获取图片描述"
        original = f"![{title}]({url})"
        replacement = f"![{image_title or title or '图片'}]({url})\n(图片描述: {image_description})"
        processed_text = processed_text.replace(original, replacement)
    return processed_text


def _image_content_prompt(file_name: str, image_info: Optional[dict] = None) -> str:
    image_info = image_info or {}
    prompt_lines = [
        f"文件名: {file_name or '未知图片'}",
        f"原始格式: {image_info.get('format') or 'unknown'}",
        f"原始尺寸: {image_info.get('width') or 'unknown'}x{image_info.get('height') or 'unknown'}",
    ]
    if image_info.get("normalized_width") and image_info.get("normalized_height"):
        prompt_lines.append(
            f"标准化尺寸: {image_info.get('normalized_width')}x{image_info.get('normalized_height')}"
        )
    return "\n".join(prompt_lines)


def _extract_image_paragraphs(file_path: str):
    image_data_url, image_info = _normalize_image_file(file_path)
    title, description = agents.uploaded_image_summary(
        image_data_url,
        file_name=os.path.basename(file_path),
        image_info=image_info,
    )

    title = (title or "").strip()
    description = (description or "").strip()
    file_name = os.path.basename(file_path)
    if not description:
        raise TransientParseError(f"图片视觉模型概括失败（可重试）: {file_path}")
    if not title or title.startswith("data:") or len(title) > 200:
        title = os.path.splitext(file_name)[0] or file_name or "image"

    summary_lines = [
        f"图片文件名: {file_name}",
        f"图片标题: {title}",
        f"图片描述: {description}",
        f"图片格式: JPEG(标准化自 {image_info.get('format') or 'unknown'})",
        f"图片尺寸: {image_info.get('normalized_width')}x{image_info.get('normalized_height')}",
    ]
    image_text = "\n".join(summary_lines)
    return [[image_text]], [[image_text]]


def _is_local_ocr_provider(provider: str) -> bool:
    return provider in {"ppocr_vl", "paddleocr_vl"}


def _is_siliconflow_ocr_provider(provider: str) -> bool:
    return provider in {
        "siliconflow",
        "siliconflow_ocr",
        "siliconflow_ppocr",
        "siliconflow_paddleocr_vl",
    }


def _is_siliconflow_deepseek_ocr_provider(provider: str) -> bool:
    return provider in {
        "deepseek_ocr",
        "siliconflow_deepseek_ocr",
        "siliconflow_deepseek",
        "siliconflow_deepseekocr",
    }


def _is_siliconflow_paddleocrvl_provider(provider: str) -> bool:
    return provider in {
        "paddleocrvl",
        "paddle_ocr_vl",
        "siliconflow_paddleocrvl",
        "siliconflow_paddle_ocr",
        "siliconflow_paddleocr_vl",
        "siliconflow_ppocrvl",
        "siliconflow_ppocr_vl",
    }


def _is_siliconflow_hybrid_ocr_provider(provider: str) -> bool:
    return provider in {
        "hybrid_ocr",
        "siliconflow_hybrid_ocr",
        "deepseek_paddle_ocr",
        "siliconflow_deepseek_paddle_ocr",
    }


def _is_modern_siliconflow_ocr_provider(provider: str) -> bool:
    return (
        _is_siliconflow_deepseek_ocr_provider(provider)
        or _is_siliconflow_paddleocrvl_provider(provider)
        or _is_siliconflow_hybrid_ocr_provider(provider)
    )


def _deepseek_ocr_retry_forever() -> bool:
    return _env_bool("SILICONFLOW_DEEPSEEK_OCR_RETRY_FOREVER", True)


def _paddleocrvl_retry_forever() -> bool:
    return _env_bool(
        "SILICONFLOW_PADDLEOCRVL_RETRY_FOREVER",
        _deepseek_ocr_retry_forever(),
    )


def _mistral_ocr_retry_forever() -> bool:
    return _env_bool("MISTRAL_OCR_RETRY_FOREVER", True)


def _mistral_ocr_retry_wait_seconds(attempt: int) -> float:
    initial = _env_float("MISTRAL_OCR_RETRY_INITIAL_WAIT_SEC", 2.0)
    max_wait = _env_float("MISTRAL_OCR_RETRY_MAX_WAIT_SEC", 120.0)
    wait = initial * (2 ** min(max(0, attempt - 1), 6))
    return _clamp_float(wait, 0.5, max_wait)


def _is_mistral_ocr_non_retryable_error(error: BaseException) -> bool:
    status_code = getattr(error, "status_code", None)
    response = getattr(error, "response", None)
    response_status = getattr(response, "status_code", None)
    if isinstance(response_status, int):
        status_code = response_status
    if status_code in {400, 404}:
        return True

    message = str(error).lower()
    non_retryable_markers = (
        "status 400",
        "status 404",
        "invalid document",
        "unsupported",
        "document_url is invalid",
    )
    return any(marker in message for marker in non_retryable_markers)


def _is_mistral_ocr_retryable_error(error: BaseException) -> bool:
    if isinstance(error, TimeoutError):
        return True
    if isinstance(error, (httpx.ConnectError, httpx.ReadError, httpx.RemoteProtocolError, httpx.TimeoutException)):
        return True
    if isinstance(error, ssl.SSLError) and "eof occurred in violation of protocol" in str(error).lower():
        return True

    status_code = getattr(error, "status_code", None)
    response = getattr(error, "response", None)
    response_status = getattr(response, "status_code", None)
    if isinstance(response_status, int):
        status_code = response_status
    if status_code in {401, 403, 429} or (isinstance(status_code, int) and 500 <= status_code < 600):
        return True

    message = str(error).lower()
    retryable_markers = (
        "status 401",
        "status 403",
        "unauthorized",
        "forbidden",
        "quota",
        "monthly quota",
        "429",
        "rate limit",
        "too many requests",
        "timeout",
        "timed out",
        "connection",
        "connect error",
        "read error",
        "remote protocol",
        "server disconnected",
        "unexpected_eof",
        "eof occurred in violation of protocol",
        "temporarily unavailable",
        "service unavailable",
        "system is too busy",
        "mistral key manager",
        "zero active keys",
        "no active mistral api keys",
        "mistral ocr key",
        "global mistral ocr key slot",
        "redis mistral ocr key-pool",
    )
    return any(marker in message for marker in retryable_markers)


def _deepseek_ocr_request_balancing() -> bool:
    return _env_bool("SILICONFLOW_DEEPSEEK_OCR_REQUEST_BALANCING", True)


def _is_deepseek_non_retryable_error(error: BaseException) -> bool:
    if isinstance(error, (FileNotFoundError, PermissionError)):
        return True
    fitz_errors = tuple(
        error_type
        for error_type in (
            getattr(fitz, "FileDataError", None),
            getattr(fitz, "EmptyFileError", None),
            getattr(fitz, "FileNotFoundError", None),
        )
        if isinstance(error_type, type)
    )
    if fitz_errors and isinstance(error, fitz_errors):
        return True

    message = str(error).lower()
    non_retryable_markers = (
        "pdf requires a password",
        "pdf has no pages",
        "cannot open broken document",
        "cannot open empty document",
        "no such file or directory",
        "failed to open file",
        "is not a pdf",
        "not a pdf",
        "bad file descriptor",
    )
    return any(marker in message for marker in non_retryable_markers)


def _deepseek_retry_wait_seconds(attempt: int) -> float:
    return min(300.0, 5.0 * max(1, attempt))


def _is_paddleocrvl_non_retryable_error(error: BaseException) -> bool:
    if _is_deepseek_non_retryable_error(error):
        return True
    message = str(error).lower()
    non_retryable_markers = (
        "paddleocr is required",
        "model disabled",
        "invalid model",
        "model not found",
        "unauthorized",
        "forbidden",
        "http 401",
        "http 403",
    )
    return any(marker in message for marker in non_retryable_markers)


def _is_rate_limited_or_busy_error(error: BaseException) -> bool:
    message = str(error).lower()
    return any(
        marker in message
        for marker in (
            "429",
            "rate limit",
            "too many",
            "tpm limit",
            "rpm limit",
            "system is too busy",
        )
    )


def _mark_ocr_client_rate_limited(client, error: BaseException) -> None:
    callback = getattr(client, "rate_limit_callback", None)
    if not callback:
        return
    try:
        callback(
            {
                "status_code": 429,
                "error": str(error)[:500],
                "provider": "siliconflow_paddleocrvl",
            }
        )
    except Exception as callback_error:
        print(f"PaddleOCR-VL rate-limit callback failed: {callback_error}", flush=True)


def _stable_route_score(value: str) -> float:
    digest = md5(str(value or "").encode("utf-8", errors="ignore")).hexdigest()
    return int(digest[:8], 16) / 0xFFFFFFFF


def _pdf_native_markdown_enabled() -> bool:
    return _env_bool("PDF_NATIVE_MARKDOWN_ENABLED", True)


def _pdf_native_text_min_chars() -> int:
    return max(0, _env_int("PDF_NATIVE_MIN_TEXT_CHARS", 200))


def _pdf_native_min_text_pages_ratio() -> float:
    return _clamp_float(_env_float("PDF_NATIVE_MIN_TEXT_PAGES_RATIO", 0.45), 0.0, 1.0)


def _pdf_native_min_page_text_chars() -> int:
    return max(0, _env_int("PDF_NATIVE_MIN_PAGE_TEXT_CHARS", 20))


def _pdf_native_large_image_min_text_chars() -> int:
    return max(
        _pdf_native_min_page_text_chars(),
        _env_int("PDF_NATIVE_LARGE_IMAGE_MIN_TEXT_CHARS", 120),
    )


def _pdf_native_scanned_image_area_ratio() -> float:
    return _clamp_float(_env_float("PDF_NATIVE_SCANNED_IMAGE_AREA_RATIO", 0.55), 0.0, 1.0)


def _pdf_native_blank_page_area_ratio() -> float:
    return _clamp_float(_env_float("PDF_NATIVE_BLANK_PAGE_AREA_RATIO", 0.003), 0.0, 1.0)


def _pdf_native_max_unsafe_pages() -> int:
    return max(0, _env_int("PDF_NATIVE_MAX_UNSAFE_PAGES", 0))


def _markdown_escape_table_cell(value_item) -> str:
    text = str(value_item or "")
    text = text.replace("\r", " ").replace("\n", "<br>")
    text = text.replace("|", "\\|")
    return text.strip()


def _table_to_markdown(table_rows) -> str:
    rows = table_rows or []
    normalized = []
    max_cols = 0
    for row in rows:
        if row is None:
            continue
        if not isinstance(row, (list, tuple)):
            row = [row]
        values = [_markdown_escape_table_cell(cell) for cell in row]
        if any(values):
            normalized.append(values)
            max_cols = max(max_cols, len(values))

    if not normalized or max_cols <= 0:
        return ""

    for row in normalized:
        if len(row) < max_cols:
            row.extend([""] * (max_cols - len(row)))

    header = normalized[0]
    if not any(header):
        header = [f"列{i + 1}" for i in range(max_cols)]
    separator = ["---"] * max_cols
    body_rows = normalized[1:] if len(normalized) > 1 else []

    lines = [
        "| " + " | ".join(header) + " |",
        "| " + " | ".join(separator) + " |",
    ]
    for row in body_rows:
        lines.append("| " + " | ".join(row) + " |")
    return "\n".join(lines)


def _fitz_rect_from_bbox(bbox):
    if not bbox or len(bbox) < 4:
        return None
    try:
        return fitz.Rect(float(bbox[0]), float(bbox[1]), float(bbox[2]), float(bbox[3]))
    except Exception:
        return None


def _native_rect_area(rect) -> float:
    if rect is None:
        return 0.0
    return max(0.0, float(rect.width)) * max(0.0, float(rect.height))


def _native_rect_intersection_area(rect_a, rect_b) -> float:
    if rect_a is None or rect_b is None:
        return 0.0
    x0 = max(rect_a.x0, rect_b.x0)
    y0 = max(rect_a.y0, rect_b.y0)
    x1 = min(rect_a.x1, rect_b.x1)
    y1 = min(rect_a.y1, rect_b.y1)
    if x1 <= x0 or y1 <= y0:
        return 0.0
    return (x1 - x0) * (y1 - y0)


def _native_rect_overlap_ratio(rect_a, rect_b) -> float:
    smaller = min(_native_rect_area(rect_a), _native_rect_area(rect_b))
    if smaller <= 0:
        return 0.0
    return _native_rect_intersection_area(rect_a, rect_b) / smaller


def _native_rect_iou(rect_a, rect_b) -> float:
    inter = _native_rect_intersection_area(rect_a, rect_b)
    if inter <= 0:
        return 0.0
    union = _native_rect_area(rect_a) + _native_rect_area(rect_b) - inter
    return inter / max(union, 1e-6)


def _clip_native_rect_to_page(rect, page_rect):
    if rect is None or page_rect is None:
        return None
    try:
        clipped = fitz.Rect(
            max(page_rect.x0, rect.x0),
            max(page_rect.y0, rect.y0),
            min(page_rect.x1, rect.x1),
            min(page_rect.y1, rect.y1),
        )
        if clipped.x1 <= clipped.x0 or clipped.y1 <= clipped.y0:
            return None
        return clipped
    except Exception:
        return None


def _dedupe_native_rects(rects):
    result = []
    for rect in sorted(rects or [], key=lambda item: _native_rect_area(item), reverse=True):
        if any(_native_rect_iou(rect, existing) > 0.85 for existing in result):
            continue
        result.append(rect)
    return result


def _native_page_visual_area_stats(page):
    page_rect = page.rect
    page_area = max(1.0, float(page_rect.width * page_rect.height))
    image_rects = []
    drawing_rects = []

    try:
        for image_entry in page.get_images(full=True) or []:
            if not image_entry:
                continue
            xref = image_entry[0]
            try:
                rects = page.get_image_rects(xref) or []
            except Exception:
                rects = []
            for rect in rects:
                clipped = _clip_native_rect_to_page(rect, page_rect)
                if clipped is not None:
                    image_rects.append(clipped)
    except Exception:
        image_rects = []

    try:
        for drawing in page.get_drawings() or []:
            rect = drawing.get("rect") if isinstance(drawing, dict) else None
            clipped = _clip_native_rect_to_page(rect, page_rect)
            if clipped is not None:
                drawing_rects.append(clipped)
    except Exception:
        drawing_rects = []

    image_rects = _dedupe_native_rects(image_rects)
    drawing_rects = _dedupe_native_rects(drawing_rects)
    image_area_ratio = min(1.0, sum(_native_rect_area(rect) for rect in image_rects) / page_area)
    drawing_area_ratio = min(1.0, sum(_native_rect_area(rect) for rect in drawing_rects) / page_area)
    return {
        "image_count": len(image_rects),
        "drawing_count": len(drawing_rects),
        "image_area_ratio": image_area_ratio,
        "drawing_area_ratio": drawing_area_ratio,
        "max_image_area_ratio": (
            max((_native_rect_area(rect) for rect in image_rects), default=0.0) / page_area
        ),
        "max_drawing_area_ratio": (
            max((_native_rect_area(rect) for rect in drawing_rects), default=0.0) / page_area
        ),
    }


def _native_pdf_page_safety_profile(page, page_index):
    try:
        text = page.get_text("text") or ""
    except Exception:
        text = ""
    compact_text_len = len(re.sub(r"\s+", "", text))
    visual_stats = _native_page_visual_area_stats(page)
    visual_area_ratio = min(
        1.0,
        float(visual_stats.get("image_area_ratio") or 0.0)
        + float(visual_stats.get("drawing_area_ratio") or 0.0),
    )
    max_image_ratio = float(visual_stats.get("max_image_area_ratio") or 0.0)
    max_drawing_ratio = float(visual_stats.get("max_drawing_area_ratio") or 0.0)

    has_text_layer = compact_text_len >= _pdf_native_min_page_text_chars()
    looks_blank = (
        compact_text_len == 0
        and visual_area_ratio <= _pdf_native_blank_page_area_ratio()
        and int(visual_stats.get("image_count") or 0) == 0
    )
    large_scanned_image = max_image_ratio >= _pdf_native_scanned_image_area_ratio()
    large_sparse_drawing = (
        max_drawing_ratio >= _pdf_native_scanned_image_area_ratio()
        and compact_text_len < _pdf_native_large_image_min_text_chars()
    )

    reason = "text"
    safe = True
    if looks_blank:
        reason = "blank"
    elif not has_text_layer:
        safe = False
        reason = "content_without_text_layer"
    elif large_scanned_image:
        safe = False
        reason = "large_image_page"
    elif large_sparse_drawing:
        safe = False
        reason = "large_drawing_with_sparse_text"

    profile = {
        "page_number": page_index + 1,
        "text_chars": compact_text_len,
        "has_text_layer": has_text_layer,
        "blank": looks_blank,
        "safe": safe,
        "reason": reason,
    }
    profile.update(visual_stats)
    return profile


def _pdf_image_content_profile(pdf_document, xref, cache):
    if xref in cache:
        return cache[xref]

    pil_image = None
    thumb = None
    try:
        extracted = pdf_document.extract_image(xref)
        image_bytes = extracted.get("image")
        if not image_bytes:
            cache[xref] = None
            return None
        pil_image = Image.open(io.BytesIO(image_bytes)).convert("RGB")
        thumb = pil_image.copy()
        resampling = getattr(Image, "Resampling", Image)
        thumb.thumbnail((256, 256), resampling.LANCZOS)
        gray = thumb.convert("L")
        hist = gray.histogram()
        total = max(1, sum(hist))
        saturation = thumb.convert("HSV").split()[1]
        profile = {
            "white_ratio": sum(hist[245:]) / total,
            "dark_ratio": sum(hist[:100]) / total,
            "saturation_mean": ImageStat.Stat(saturation).mean[0],
        }
        cache[xref] = profile
        return profile
    except Exception:
        cache[xref] = None
        return None
    finally:
        for image in (thumb, pil_image):
            if image is not None:
                try:
                    image.close()
                except Exception:
                    pass


def _looks_like_text_raster_image(profile) -> bool:
    if not profile:
        return False
    return (
        profile.get("white_ratio", 0.0) >= 0.55
        and profile.get("dark_ratio", 1.0) <= 0.38
        and profile.get("saturation_mean", 255.0) <= 24.0
    )


def _should_keep_native_pdf_image(pdf_document, image_entry, rect, page_rect, profile_cache, text_raster_strip_count=0) -> bool:
    if rect is None:
        return False

    display_width = max(0.0, float(rect.width))
    display_height = max(0.0, float(rect.height))
    display_area = display_width * display_height
    page_area = max(1.0, float(page_rect.width * page_rect.height))
    area_ratio = display_area / page_area
    width_ratio = display_width / max(1.0, float(page_rect.width))
    aspect = max(display_width / max(display_height, 1e-6), display_height / max(display_width, 1e-6))

    min_side = max(1, _env_int("PDF_NATIVE_IMAGE_MIN_SIDE", 24))
    min_area = max(1, _env_int("PDF_NATIVE_IMAGE_MIN_AREA", 1024))
    if min(display_width, display_height) < min_side or display_area < min_area:
        return False

    try:
        pixel_width = int(image_entry[2] or 0)
        pixel_height = int(image_entry[3] or 0)
    except Exception:
        pixel_width = 0
        pixel_height = 0
    if pixel_width and pixel_height and min(pixel_width, pixel_height) < 18 and max(pixel_width, pixel_height) < 80:
        return False

    if aspect >= 8.0:
        return False
    if width_ratio >= 0.65 and aspect >= 3.5 and area_ratio < 0.22:
        return False

    xref = int(image_entry[0])
    profile = _pdf_image_content_profile(pdf_document, xref, profile_cache)
    if _looks_like_text_raster_image(profile):
        if text_raster_strip_count >= 2 and width_ratio >= 0.55 and aspect >= 1.55:
            return False
        if text_raster_strip_count >= 2 and area_ratio >= 0.045 and aspect >= 1.35:
            return False
        if area_ratio < 0.02 and aspect >= 2.2:
            return False

    if max(display_width, display_height) >= 120 and aspect >= 4.5 and area_ratio < 0.035:
        return False

    full_page_threshold = _env_float("PDF_NATIVE_FULL_PAGE_IMAGE_AREA_RATIO", 0.82)
    if area_ratio >= full_page_threshold and width_ratio >= 0.82:
        return _env_bool("PDF_NATIVE_SAVE_FULL_PAGE_IMAGES", False)

    return True


def _merge_native_pdf_image_candidates(candidates):
    if len(candidates) <= 1:
        return candidates

    gap_limit = max(0.0, _env_float("PDF_NATIVE_IMAGE_MERGE_GAP", 12.0))
    merged = list(candidates)
    changed = True
    while changed:
        changed = False
        result = []
        used = [False] * len(merged)
        for idx, current in enumerate(merged):
            if used[idx]:
                continue
            current_rect = current["rect"]
            current["rects"] = list(current.get("rects") or [current_rect])
            used[idx] = True

            for other_idx in range(idx + 1, len(merged)):
                if used[other_idx]:
                    continue
                other = merged[other_idx]
                other_rect = other["rect"]
                overlap = _native_rect_overlap_ratio(current_rect, other_rect)
                iou = _native_rect_iou(current_rect, other_rect)
                horizontal_gap = max(0.0, max(other_rect.x0 - current_rect.x1, current_rect.x0 - other_rect.x1))
                vertical_gap = max(0.0, max(other_rect.y0 - current_rect.y1, current_rect.y0 - other_rect.y1))
                overlap_x = max(0.0, min(current_rect.x1, other_rect.x1) - max(current_rect.x0, other_rect.x0))
                overlap_y = max(0.0, min(current_rect.y1, other_rect.y1) - max(current_rect.y0, other_rect.y0))
                min_width = max(1.0, min(current_rect.width, other_rect.width))
                min_height = max(1.0, min(current_rect.height, other_rect.height))
                should_merge = (
                    overlap >= 0.62
                    or iou >= 0.12
                    or (horizontal_gap <= gap_limit and overlap_y / min_height >= 0.42)
                    or (vertical_gap <= gap_limit and overlap_x / min_width >= 0.42)
                )
                if not should_merge:
                    continue
                current_rect = fitz.Rect(
                    min(current_rect.x0, other_rect.x0),
                    min(current_rect.y0, other_rect.y0),
                    max(current_rect.x1, other_rect.x1),
                    max(current_rect.y1, other_rect.y1),
                )
                current["rect"] = current_rect
                current["rects"].extend(other.get("rects") or [other_rect])
                used[other_idx] = True
                changed = True

            result.append(current)
        merged = result

    return sorted(merged, key=lambda item: (item["rect"].y0, item["rect"].x0, item["rect"].y1, item["rect"].x1))


def _render_native_pdf_region(page, rect):
    if rect is None or rect.x1 <= rect.x0 or rect.y1 <= rect.y0:
        return None
    dpi = max(96, _env_int("PDF_NATIVE_IMAGE_RENDER_DPI", 180))
    matrix = fitz.Matrix(dpi / 72.0, dpi / 72.0)
    pix = None
    try:
        pix = page.get_pixmap(matrix=matrix, clip=rect, alpha=False)
        return pix.tobytes("jpeg", jpg_quality=90)
    except Exception:
        try:
            if pix is not None:
                return pix.tobytes("png")
        except Exception:
            return None
        return None
    finally:
        pix = None


def _find_native_pdf_tables(page):
    if not hasattr(page, "find_tables"):
        return []

    tables = []
    try:
        table_result = page.find_tables()
    except Exception as e:
        print(f"PDF原生表格检测失败 page={getattr(page, 'number', '?')}: {e}")
        return []

    for table_index, table in enumerate(getattr(table_result, "tables", []) or []):
        try:
            rows = table.extract()
        except Exception:
            rows = []
        markdown = _table_to_markdown(rows)
        if not markdown:
            continue
        rect = getattr(table, "bbox", None)
        if rect and not isinstance(rect, fitz.Rect):
            rect = _fitz_rect_from_bbox(rect)
        tables.append({
            "type": "table",
            "rect": rect,
            "markdown": markdown,
            "index": table_index,
        })
    return tables


def _extract_native_pdf_text_blocks(page, table_rects):
    blocks = []
    try:
        raw_blocks = page.get_text("blocks", sort=True) or []
    except Exception:
        raw_blocks = []

    for block_index, block in enumerate(raw_blocks):
        if len(block) < 5:
            continue
        if len(block) > 6:
            try:
                if int(block[6]) != 0:
                    continue
            except Exception:
                pass
        rect = _fitz_rect_from_bbox(block[:4])
        text = _normalize_native_pdf_text(block[4])
        if not text:
            continue
        if any(_native_rect_overlap_ratio(rect, table_rect) > 0.65 for table_rect in table_rects if table_rect is not None):
            continue
        blocks.append({
            "type": "text",
            "rect": rect,
            "text": text,
            "index": block_index,
        })
    return blocks


def _normalize_native_pdf_text(text) -> str:
    lines = [line.strip() for line in str(text or "").splitlines()]
    lines = [line for line in lines if line]
    if not lines:
        return ""

    normalized = ""
    for line in lines:
        if normalized.endswith("-") and line[:1].islower():
            normalized = normalized[:-1] + line
        elif normalized:
            normalized += " " + line
        else:
            normalized = line
    return re.sub(r"\s+", " ", normalized).strip()


def _native_pdf_image_candidates(pdf_document, page, pdf_md5, output_dir, page_number, existing_hashes, *, file_remark=""):
    if not output_dir:
        return [], []

    page_rect = page.rect
    profile_cache = {}
    raw_candidates = []
    for image_pos, image_entry in enumerate(page.get_images(full=True) or [], start=1):
        if not image_entry:
            continue
        xref = image_entry[0]
        try:
            image_rects = page.get_image_rects(xref) or []
        except Exception:
            image_rects = []
        for rect_pos, rect in enumerate(image_rects, start=1):
            if rect is None:
                continue
            clipped = fitz.Rect(
                max(page_rect.x0, rect.x0),
                max(page_rect.y0, rect.y0),
                min(page_rect.x1, rect.x1),
                min(page_rect.y1, rect.y1),
            )
            if clipped.x1 <= clipped.x0 or clipped.y1 <= clipped.y0:
                continue
            raw_candidates.append({
                "xref": xref,
                "image_entry": image_entry,
                "rect": clipped,
                "rects": [clipped],
                "image_pos": image_pos,
                "rect_pos": rect_pos,
            })

    text_raster_strip_count = 0
    page_area = max(1.0, float(page_rect.width * page_rect.height))
    for candidate in raw_candidates:
        rect = candidate["rect"]
        width_ratio = rect.width / max(1.0, float(page_rect.width))
        area_ratio = (rect.width * rect.height) / page_area
        aspect = max(rect.width / max(rect.height, 1e-6), rect.height / max(rect.width, 1e-6))
        profile = _pdf_image_content_profile(pdf_document, int(candidate["xref"]), profile_cache)
        if (
            _looks_like_text_raster_image(profile)
            and width_ratio >= 0.55
            and aspect >= 1.55
            and area_ratio >= 0.025
        ):
            text_raster_strip_count += 1

    kept = []
    for candidate in raw_candidates:
        if _should_keep_native_pdf_image(
            pdf_document,
            candidate["image_entry"],
            candidate["rect"],
            page_rect,
            profile_cache,
            text_raster_strip_count=text_raster_strip_count,
        ):
            kept.append(candidate)

    merged = _merge_native_pdf_image_candidates(kept)
    saved_images = []
    image_blocks = []
    for image_index, candidate in enumerate(merged, start=1):
        xref = int(candidate["xref"])
        rect = candidate["rect"]
        image_bytes = _render_native_pdf_region(page, rect)
        try:
            if not image_bytes:
                base_image = pdf_document.extract_image(xref)
                image_bytes = base_image.get("image")
        except Exception as e:
            print(f"PDF原生图片提取失败 page={page_number} xref={xref}: {e}")
            continue
        if not image_bytes:
            continue
        try:
            normalized_bytes, image_info = _normalize_pdf_image_bytes(image_bytes)
            image_hash = md5(normalized_bytes).hexdigest()
        except Exception:
            continue
        if image_hash in existing_hashes:
            continue

        image_id = f"native-{page_number}-{image_index}-{xref}"
        image_filename = f"page_{page_number}_chunk_0_native_{image_index}_{xref}.jpeg"
        coordinates = {
            "top_left_x": rect.x0,
            "top_left_y": rect.y0,
            "bottom_right_x": rect.x1,
            "bottom_right_y": rect.y1,
        }
        saved = _save_pdf_image_bytes(
            pdf_md5,
            output_dir,
            image_filename,
            image_bytes,
            page_number=page_number,
            chunk_index=0,
            image_id=image_id,
            file_remark=file_remark,
            coordinates=coordinates,
            dimensions=image_info,
        )
        if not saved:
            continue
        saved_images.append(saved)
        existing_hashes.add(saved.get("md5") or image_hash)
        image_blocks.append({
            "type": "image",
            "rect": rect,
            "markdown": f"![{image_id}]({saved['url']})",
            "index": image_index,
            "image": saved,
        })

    return image_blocks, saved_images


def _native_pdf_page_markdown(pdf_document, page, pdf_md5, output_dir, page_number, existing_hashes, save_images=True, file_remark=""):
    tables = _find_native_pdf_tables(page)
    table_rects = [table.get("rect") for table in tables if table.get("rect") is not None]
    text_blocks = _extract_native_pdf_text_blocks(page, table_rects)
    image_blocks = []
    page_images = []
    if save_images:
        image_blocks, page_images = _native_pdf_image_candidates(
            pdf_document,
            page,
            pdf_md5,
            output_dir,
            page_number,
            existing_hashes,
            file_remark=file_remark,
        )

    elements = []
    elements.extend(text_blocks)
    elements.extend(tables)
    elements.extend(image_blocks)
    elements.sort(key=lambda item: (
        item["rect"].y0 if item.get("rect") is not None else 0,
        item["rect"].x0 if item.get("rect") is not None else 0,
        item.get("index", 0),
    ))

    parts = []
    for element in elements:
        if element["type"] == "text":
            parts.append(element["text"])
        else:
            parts.append(element["markdown"])

    markdown = "\n\n".join(part.strip() for part in parts if str(part or "").strip())
    return markdown, page_images


def _pdf_native_safety_analysis(pdf_document):
    total_pages = len(pdf_document)
    total_text_chars = 0
    text_pages = 0
    blank_pages = 0
    unsafe_pages = []
    profiles = []

    for page_index in range(total_pages):
        try:
            profile = _native_pdf_page_safety_profile(pdf_document.load_page(page_index), page_index)
        except Exception:
            profile = {
                "page_number": page_index + 1,
                "text_chars": 0,
                "has_text_layer": False,
                "blank": False,
                "safe": False,
                "reason": "page_analysis_failed",
            }
        profiles.append(profile)
        compact_text_len = int(profile.get("text_chars") or 0)
        total_text_chars += compact_text_len
        if profile.get("has_text_layer"):
            text_pages += 1
        if profile.get("blank"):
            blank_pages += 1
        if not profile.get("safe"):
            unsafe_pages.append(profile)

    content_pages = max(0, total_pages - blank_pages)
    safe_pages = max(0, content_pages - len(unsafe_pages))
    return {
        "page_count": total_pages,
        "content_pages": content_pages,
        "total_text_chars": total_text_chars,
        "text_pages": text_pages,
        "blank_pages": blank_pages,
        "safe_pages": safe_pages,
        "unsafe_pages": unsafe_pages[:20],
        "unsafe_page_count": len(unsafe_pages),
        "text_page_ratio": (text_pages / content_pages) if content_pages else 1.0,
        "safe_page_ratio": (safe_pages / content_pages) if content_pages else 1.0,
        "profiles": profiles,
    }


def _pdf_is_native_text_candidate(analysis) -> bool:
    page_count = int(analysis.get("page_count") or 0)
    if page_count <= 0:
        return False
    content_pages = int(analysis.get("content_pages") or 0)
    if content_pages <= 0:
        return False
    if int(analysis.get("total_text_chars") or 0) < _pdf_native_text_min_chars():
        return False
    if int(analysis.get("unsafe_page_count") or 0) > _pdf_native_max_unsafe_pages():
        return False
    text_page_ratio = float(analysis.get("text_page_ratio") or 0.0)
    return text_page_ratio >= _pdf_native_min_text_pages_ratio()


def perform_pdf_native_markdown(pdf_path, save_images=True, output_dir=None, user_id=None):
    """
    Fast path for digital PDFs with embedded text.

    It preserves table markdown where PyMuPDF supports table detection and stores
    extracted embedded figures in the same storage layout as the OCR path.
    Scanned or mostly raster PDFs return None so callers can fall back to OCR.
    """
    if not _pdf_native_markdown_enabled():
        return None
    if not os.path.exists(pdf_path):
        raise FileNotFoundError(f"PDF文件不存在: {pdf_path}")

    pdf_document = None
    try:
        pdf_document = fitz.open(pdf_path)
        if pdf_document.needs_pass and not pdf_document.authenticate(""):
            raise SourceDocumentParseError(f"PDF需要密码: {pdf_path}")
        if len(pdf_document) <= 0:
            raise SourceDocumentParseError(f"PDF没有有效页面: {pdf_path}")

        analysis = _pdf_native_safety_analysis(pdf_document)
        if not _pdf_is_native_text_candidate(analysis):
            unsafe_summary = [
                {
                    "page": item.get("page_number"),
                    "reason": item.get("reason"),
                    "text_chars": item.get("text_chars"),
                    "max_image_area_ratio": round(float(item.get("max_image_area_ratio") or 0.0), 3),
                    "max_drawing_area_ratio": round(float(item.get("max_drawing_area_ratio") or 0.0), 3),
                }
                for item in (analysis.get("unsafe_pages") or [])[:8]
            ]
            print(
                "PDF原生解析跳过，疑似扫描/混合PDF或文本层不足: "
                f"pages={analysis.get('page_count')}, content_pages={analysis.get('content_pages')}, "
                f"text_pages={analysis.get('text_pages')}, unsafe={analysis.get('unsafe_page_count')}, "
                f"text_chars={analysis.get('total_text_chars')}, unsafe_pages={unsafe_summary}, file={pdf_path}",
                flush=True,
            )
            return None

        pdf_md5 = md5(pdf_path.encode()).hexdigest()
        output_dir = _pdf_image_storage_dir(pdf_md5, output_dir) if save_images else None
        existing_hashes = set()
        markdown_result = ""
        markdown_result_pages = []
        all_extracted_images = []
        file_remark = "attachment for " + pdf_path

        for page_index in range(len(pdf_document)):
            page = pdf_document.load_page(page_index)
            page_number = page_index + 1
            page_markdown, page_images = _native_pdf_page_markdown(
                pdf_document,
                page,
                pdf_md5,
                output_dir,
                page_number,
                existing_hashes,
                save_images=save_images,
                file_remark=file_remark,
            )
            if not page_markdown.strip():
                try:
                    page_markdown = (page.get_text("text") or "").strip()
                except Exception:
                    page_markdown = ""
            if page_markdown.strip():
                markdown_result += page_markdown.rstrip() + "\n\n"
            markdown_result_pages.append({
                "page_number": page_number,
                "markdown": page_markdown,
                "chunk_index": 0,
                "images": page_images,
            })
            all_extracted_images.extend(page_images)

        markdown_result = markdown_result.strip()
        if len(re.sub(r"\s+", "", markdown_result)) < _pdf_native_text_min_chars():
            print(f"PDF原生解析结果文本过少，回退OCR: file={pdf_path}", flush=True)
            return None

        print(
            f"PDF原生解析成功: pages={len(pdf_document)}, chars={len(markdown_result)}, "
            f"images={len(all_extracted_images)}, file={pdf_path}",
            flush=True,
        )
        if save_images:
            return markdown_result, markdown_result_pages, all_extracted_images
        return markdown_result, markdown_result_pages
    except SourceDocumentParseError:
        raise
    except Exception as e:
        if _is_invalid_pdf_error(e):
            raise SourceDocumentParseError(
                f"PDF源文件无法打开或已损坏，请检查文件格式是否正确: {pdf_path}"
            ) from e
        print(f"PDF原生解析失败，将回退OCR: {e}", flush=True)
        traceback.print_exc()
        return None
    finally:
        if pdf_document is not None:
            try:
                pdf_document.close()
            except Exception:
                pass
        gc.collect()


def _parse_pdf_with_native_fallback_to_ocr(file_path, *, save_images=True, output_dir=None, user_id=None):
    native_result = perform_pdf_native_markdown(
        file_path,
        save_images=save_images,
        output_dir=output_dir,
        user_id=user_id,
    )
    if native_result:
        return native_result
    return perform_pdf_ocr(file_path, save_images=save_images, output_dir=output_dir, user_id=user_id)


def _pdf_parse_result_text(result) -> str:
    if isinstance(result, tuple):
        return str(result[0] or "")
    return str(result or "")


def _try_local_pdf_ocr(pdf_path, save_images=False, output_dir=None):
    print(f"尝试使用本地 PaddleOCR-VL 解析PDF: {pdf_path}")
    try:
        import local_ocr_client

        return local_ocr_client.run_ppocr_vl_pdf_ocr(
            pdf_path,
            save_images=save_images,
            output_dir=output_dir,
        )
    except Exception as e:
        print(f"本地 PaddleOCR-VL 解析失败: {e}")
        traceback.print_exc()
        return None


def _try_siliconflow_pdf_ocr(pdf_path, save_images=False, output_dir=None):
    print(f"尝试使用硅基流动 PaddleOCR-VL 解析PDF: {pdf_path}")
    try:
        import siliconflow_ocr_client

        return siliconflow_ocr_client.run_siliconflow_pdf_ocr(
            pdf_path,
            save_images=save_images,
            output_dir=output_dir,
        )
    except Exception as e:
        print(f"硅基流动 PaddleOCR-VL 解析失败: {e}")
        traceback.print_exc()
        return None


def _get_deepseek_ocr_balancer():
    global deepseek_ocr_balancer
    if deepseek_ocr_balancer is None:
        import siliconflow_deepseek_ocr_balancer

        deepseek_ocr_balancer = siliconflow_deepseek_ocr_balancer.get_shared_balancer()
    return deepseek_ocr_balancer


def _get_paddleocrvl_balancer():
    global paddleocrvl_balancer
    if paddleocrvl_balancer is None:
        import siliconflow_deepseek_ocr_balancer

        paddleocrvl_balancer = siliconflow_deepseek_ocr_balancer.get_shared_paddleocrvl_balancer()
    return paddleocrvl_balancer


def _try_siliconflow_deepseek_pdf_ocr(pdf_path, save_images=False, output_dir=None):
    print(f"尝试使用硅基流动 DeepSeek-OCR 解析PDF: {pdf_path}")
    import siliconflow_deepseek_ocr_sdk

    attempt = 0
    retry_forever = _deepseek_ocr_retry_forever()
    while True:
        attempt += 1
        try:
            balancer = _get_deepseek_ocr_balancer()
            if not hasattr(balancer, "request_balanced_client"):
                raise RuntimeError("DeepSeek-OCR balancer does not support request-level key leasing")
            client = balancer.request_balanced_client()
            result = siliconflow_deepseek_ocr_sdk.run_siliconflow_deepseek_pdf_ocr(
                pdf_path,
                save_images=save_images,
                output_dir=output_dir,
                client=client,
            )
            if result:
                return result

            if not retry_forever:
                print(f"硅基流动 DeepSeek-OCR 解析返回空: {pdf_path}")
                return None

            wait_sec = _deepseek_retry_wait_seconds(attempt)
            print(
                f"硅基流动 DeepSeek-OCR 解析返回空，将继续重试 "
                f"(attempt={attempt}, wait={wait_sec:.1f}s): {pdf_path}",
                flush=True,
            )
            time.sleep(wait_sec)
        except Exception as e:
            if _is_deepseek_non_retryable_error(e):
                print(f"硅基流动 DeepSeek-OCR 遇到源文档不可重试错误: {e}")
                traceback.print_exc()
                return None
            if not retry_forever:
                print(f"硅基流动 DeepSeek-OCR 解析失败: {e}")
                traceback.print_exc()
                return None

            wait_sec = _deepseek_retry_wait_seconds(attempt)
            print(
                f"硅基流动 DeepSeek-OCR 通道错误，将继续重试 "
                f"(attempt={attempt}, wait={wait_sec:.1f}s): {e}",
                flush=True,
            )
            traceback.print_exc()
            time.sleep(wait_sec)


def _try_siliconflow_paddleocrvl_pdf_ocr(pdf_path, save_images=False, output_dir=None):
    print(f"尝试使用硅基流动 PaddleOCR-VL 1.5 解析PDF: {pdf_path}")
    import siliconflow_paddleocrvl_sdk

    if os.getenv("SILICONFLOW_PADDLEOCRVL_SERVICE_URL", "").strip() and _env_bool(
        "SILICONFLOW_PADDLEOCRVL_REMOTE_SERVICE_OWNS_KEYS",
        True,
    ):
        attempt = 0
        while True:
            attempt += 1
            try:
                return siliconflow_paddleocrvl_sdk.run_siliconflow_paddleocrvl_pdf_ocr(
                    pdf_path,
                    save_images=save_images,
                    output_dir=output_dir,
                    api_key=os.getenv("SILICONFLOW_PADDLEOCRVL_API_KEY") or os.getenv("SILICONFLOW_API_KEY") or "remote-service",
                    base_url=os.getenv("SILICONFLOW_PADDLEOCRVL_BASE_URL", "https://api.siliconflow.cn/v1"),
                )
            except Exception as e:
                if _is_paddleocrvl_non_retryable_error(e):
                    print(f"远程 PaddleOCR-VL 服务遇到不可重试错误: {e}")
                    traceback.print_exc()
                    return None
                if not _paddleocrvl_retry_forever():
                    print(f"远程 PaddleOCR-VL 服务解析失败: {e}")
                    traceback.print_exc()
                    return None
                wait_sec = _deepseek_retry_wait_seconds(attempt)
                print(
                    f"远程 PaddleOCR-VL 服务错误，将继续重试 "
                    f"(attempt={attempt}, wait={wait_sec:.1f}s): {e}",
                    flush=True,
                )
                traceback.print_exc()
                time.sleep(wait_sec)

    attempt = 0
    retry_forever = _paddleocrvl_retry_forever()
    while True:
        attempt += 1
        siliconflow_client = None
        try:
            balancer = _get_paddleocrvl_balancer()
            with balancer.lease(timeout=balancer.default_timeout) as siliconflow_client:
                result = siliconflow_paddleocrvl_sdk.run_siliconflow_paddleocrvl_pdf_ocr(
                    pdf_path,
                    save_images=save_images,
                    output_dir=output_dir,
                    api_key=getattr(siliconflow_client, "api_key", None),
                    base_url=os.getenv(
                        "SILICONFLOW_PADDLEOCRVL_BASE_URL",
                        getattr(siliconflow_client, "base_url", "https://api.siliconflow.cn/v1"),
                    ),
                )
            if result:
                return result

            if not retry_forever:
                print(f"硅基流动 PaddleOCR-VL 解析返回空: {pdf_path}")
                return None

            wait_sec = _deepseek_retry_wait_seconds(attempt)
            print(
                f"硅基流动 PaddleOCR-VL 解析返回空，将继续重试 "
                f"(attempt={attempt}, wait={wait_sec:.1f}s): {pdf_path}",
                flush=True,
            )
            time.sleep(wait_sec)
        except Exception as e:
            if _is_rate_limited_or_busy_error(e):
                _mark_ocr_client_rate_limited(siliconflow_client, e)
            if _is_paddleocrvl_non_retryable_error(e):
                print(f"硅基流动 PaddleOCR-VL 遇到不可重试错误: {e}")
                traceback.print_exc()
                return None
            if not retry_forever:
                print(f"硅基流动 PaddleOCR-VL 解析失败: {e}")
                traceback.print_exc()
                return None

            wait_sec = _deepseek_retry_wait_seconds(attempt)
            print(
                f"硅基流动 PaddleOCR-VL 通道错误，将继续重试 "
                f"(attempt={attempt}, wait={wait_sec:.1f}s): {e}",
                flush=True,
            )
            traceback.print_exc()
            time.sleep(wait_sec)


def _paddleocrvl_remote_queue_pressure() -> dict:
    service_url = os.getenv("SILICONFLOW_PADDLEOCRVL_SERVICE_URL", "").strip().rstrip("/")
    if not service_url:
        return {"ok": False, "reason": "no_service_url"}
    token = os.getenv("SILICONFLOW_PADDLEOCRVL_SERVICE_TOKEN", "").strip()
    headers = {"X-OCR-Token": token} if token else None
    timeout_sec = _env_float("SILICONFLOW_HYBRID_OCR_HEALTH_TIMEOUT_SEC", 1.0)
    try:
        with httpx.Client(timeout=httpx.Timeout(timeout_sec), trust_env=False) as client:
            response = client.get(f"{service_url}/health", headers=headers)
            response.raise_for_status()
            data = response.json()
    except Exception as exc:
        return {"ok": False, "reason": str(exc)[:200]}

    queue_info = data.get("queue") if isinstance(data, dict) else {}
    key_pool = data.get("siliconflow_paddleocrvl_key_pool") if isinstance(data, dict) else {}
    waiting = int((queue_info or {}).get("waiting") or (queue_info or {}).get("queued") or 0)
    running = int((queue_info or {}).get("running") or 0)
    total_slots = int((key_pool or {}).get("total_slots") or data.get("max_concurrency") or 1)
    return {
        "ok": True,
        "waiting": waiting,
        "running": running,
        "backlog": waiting + running,
        "total_slots": max(1, total_slots),
    }


def _select_hybrid_ocr_provider(pdf_path: str) -> str:
    forced = os.getenv("SILICONFLOW_HYBRID_OCR_FORCE", "").strip().lower()
    if forced in {"deepseek", "deepseek_ocr", "siliconflow_deepseek_ocr"}:
        return "deepseek"
    if forced in {"paddle", "paddleocrvl", "paddleocr_vl", "siliconflow_paddleocrvl"}:
        return "paddleocrvl"

    try:
        import siliconflow_paddleocrvl_sdk

        sample_pages = max(1, _env_int("SILICONFLOW_HYBRID_OCR_SAMPLE_PAGES", 5))
        analysis = siliconflow_paddleocrvl_sdk.analyze_pdf_layout_for_routing(
            pdf_path,
            max_pages=sample_pages,
        )
    except Exception as exc:
        print(f"Hybrid OCR 路由分析失败，默认使用 DeepSeek-OCR: {exc}", flush=True)
        analysis = {"ok": False, "route": "deepseek", "route_reason": "analysis_failed"}

    route = str(analysis.get("route") or "deepseek")
    max_paddle_pages = _env_int("SILICONFLOW_HYBRID_OCR_PADDLE_MAX_PAGES", 300)
    page_count = int(analysis.get("page_count") or 0)
    if route == "paddleocrvl" and max_paddle_pages > 0 and page_count > max_paddle_pages:
        route = "deepseek"
        analysis["route_reason"] = f"page_count>{max_paddle_pages}"

    if _env_bool("SILICONFLOW_HYBRID_OCR_BALANCE", True):
        score = _stable_route_score(f"{pdf_path}:{page_count}")
        min_paddle_share = _clamp_float(
            _env_float("SILICONFLOW_HYBRID_OCR_MIN_PADDLE_SHARE", 0.20),
            0.0,
            1.0,
        )
        preferred_paddle_share = _clamp_float(
            _env_float("SILICONFLOW_HYBRID_OCR_PADDLE_PREFERRED_SHARE", 0.45),
            min_paddle_share,
            1.0,
        )
        target_share = preferred_paddle_share if route == "paddleocrvl" else min_paddle_share
        balanced_route = "paddleocrvl" if score < target_share else "deepseek"
        analysis["content_route"] = route
        analysis["balance_score"] = round(score, 6)
        analysis["balance_target_paddle_share"] = round(target_share, 3)
        route = balanced_route

    if route == "paddleocrvl" and _env_bool("SILICONFLOW_HYBRID_OCR_AVOID_PADDLE_BACKLOG", True):
        pressure = _paddleocrvl_remote_queue_pressure()
        max_backlog = _env_int("SILICONFLOW_HYBRID_OCR_PADDLE_MAX_BACKLOG", 24)
        if pressure.get("ok"):
            analysis["paddle_pressure"] = pressure
            if int(pressure.get("backlog") or 0) >= max_backlog:
                route = "deepseek"
                analysis["route_reason"] = f"paddle_backlog>={max_backlog}"
        else:
            analysis["paddle_pressure"] = pressure
            if _env_bool("SILICONFLOW_HYBRID_OCR_REQUIRE_PADDLE_HEALTH", True):
                route = "deepseek"
                analysis["route_reason"] = "paddle_health_unavailable"

    print(
        f"Hybrid OCR 路由: route={route}, analysis={json.dumps(analysis, ensure_ascii=False)}",
        flush=True,
    )
    return "paddleocrvl" if route == "paddleocrvl" else "deepseek"


def _try_siliconflow_hybrid_pdf_ocr(pdf_path, save_images=False, output_dir=None):
    route = _select_hybrid_ocr_provider(pdf_path)
    if route == "paddleocrvl":
        primary = _try_siliconflow_paddleocrvl_pdf_ocr
        secondary = _try_siliconflow_deepseek_pdf_ocr
        secondary_name = "DeepSeek-OCR"
    else:
        primary = _try_siliconflow_deepseek_pdf_ocr
        secondary = _try_siliconflow_paddleocrvl_pdf_ocr
        secondary_name = "PaddleOCR-VL"

    result = primary(pdf_path, save_images=save_images, output_dir=output_dir)
    if result:
        return result

    print(f"Hybrid OCR 主路由失败，尝试备用通道: {secondary_name}", flush=True)
    return secondary(pdf_path, save_images=save_images, output_dir=output_dir)


def _try_pdf_ocr_provider(provider, pdf_path, save_images=False, output_dir=None):
    if _is_local_ocr_provider(provider):
        return _try_local_pdf_ocr(pdf_path, save_images=save_images, output_dir=output_dir)
    if _is_siliconflow_hybrid_ocr_provider(provider):
        return _try_siliconflow_hybrid_pdf_ocr(pdf_path, save_images=save_images, output_dir=output_dir)
    if _is_siliconflow_paddleocrvl_provider(provider):
        return _try_siliconflow_paddleocrvl_pdf_ocr(pdf_path, save_images=save_images, output_dir=output_dir)
    if _is_siliconflow_deepseek_ocr_provider(provider):
        return _try_siliconflow_deepseek_pdf_ocr(pdf_path, save_images=save_images, output_dir=output_dir)
    if _is_siliconflow_ocr_provider(provider):
        return _try_siliconflow_pdf_ocr(pdf_path, save_images=save_images, output_dir=output_dir)
    return None


def encode_pdf(pdf_path):
    """Encode the pdf to base64."""
    try:
        with open(pdf_path, "rb") as pdf_file:
            return base64.b64encode(pdf_file.read()).decode("utf-8")
    except FileNotFoundError:
        print(f"Error: The file {pdf_path} was not found.")
        return None
    except Exception as e:  # Added general exception handling
        print(f"Error: {e}")
        return None


def _is_invalid_pdf_error(error) -> bool:
    message = str(error).lower()
    invalid_markers = (
        "eof marker not found",
        "no objects found",
        "cannot open broken document",
        "failed to open file",
        "not a pdf",
        "invalid pdf",
        "startxref not found",
    )
    return any(marker in message for marker in invalid_markers)


def _is_ocr_dependency_error(error) -> bool:
    message = str(error).lower()
    markers = (
        "mistral key manager",
        "zero active keys",
        "no active mistral api keys",
        "mistral ocr key",
        "global mistral ocr key slot",
        "redis mistral ocr key-pool",
        "timed out waiting for a global mistral",
        "timed out waiting for an available mistral",
    )
    return any(marker in message for marker in markers)


def cut_pdf(pdf_file):
    output_dir = tempfile.gettempdir()
    filename = os.path.basename(pdf_file)
    name, ext = os.path.splitext(filename)

    try:
        # 首先尝试使用 PyPDF2 处理
        with open(pdf_file, "rb") as file:
            reader = PyPDF2.PdfReader(file)
            
            # 检查PDF是否加密，如果加密则尝试解密
            if reader.is_encrypted:
                # print(f"PDF文件已加密，尝试解密: {pdf_file}")
                try:
                    success = reader.decrypt("")
                    if not success:
                        # print(f"PDF需要密码，尝试使用替代方法: {pdf_file}")
                        return try_alternative_pdf_split(pdf_file)
                except Exception as decrypt_error:
                    # print(f"解密失败: {decrypt_error}，尝试使用替代方法")
                    return try_alternative_pdf_split(pdf_file)
                
            total_pages = len(reader.pages)
            # print(f"PDF总页数: {total_pages}")
            
            if total_pages == 0:
                # print(f"PDF文件无有效页面: {pdf_file}")
                return try_alternative_pdf_split(pdf_file)

            # 优化策略：批量处理页面
            successful_chunks = []
            current_chunk = 1
            
            # 预设每个块的页数范围，避免频繁计算大小
            pages_per_chunk = min(200, max(20, total_pages // 10))  # 动态调整页数
            max_pages_per_chunk = 300
            
            start_page = 0
            while start_page < total_pages:
                end_page = min(start_page + pages_per_chunk, total_pages)
                
                # 创建新的writer
                writer = PyPDF2.PdfWriter()
                actual_pages_added = 0
                
                # 批量添加页面
                for page_num in range(start_page, end_page):
                    try:
                        page = reader.pages[page_num]
                        writer.add_page(page)
                        actual_pages_added += 1
                    except Exception as page_error:
                        print(f"跳过损坏的第 {page_num + 1} 页: {page_error}")
                        continue
                
                # 如果有有效页面，保存块
                if actual_pages_added > 0:
                    output_path = os.path.join(output_dir, f"{name}_chunk{current_chunk}{ext}")
                    
                    try:
                        # 直接写入最终文件，减少IO操作
                        with open(output_path, "wb") as output_file:
                            writer.write(output_file)
                            output_file.flush()
                            os.fsync(output_file.fileno())
                        
                        # 验证文件
                        if os.path.exists(output_path) and os.path.getsize(output_path) > 0:
                            successful_chunks.append(output_path)
                            # print(f"成功创建PDF块: {output_path}, 包含 {actual_pages_added} 页")
                            
                            # 动态调整下一个块的页数
                            file_size_mb = os.path.getsize(output_path) / (1024 * 1024)
                            if file_size_mb > 8:  # 如果文件太大，减少页数
                                pages_per_chunk = max(20, int(pages_per_chunk * 0.8))
                            elif file_size_mb < 3:  # 如果文件太小，增加页数
                                pages_per_chunk = min(max_pages_per_chunk, int(pages_per_chunk * 1.2))
                        else:
                            print(f"PDF块创建失败: {output_path}")
                            
                    except Exception as save_error:
                        print(f"保存PDF块时出错: {save_error}")
                        continue
                
                current_chunk += 1
                start_page = end_page

            if successful_chunks:
                # print(f"成功创建 {len(successful_chunks)} 个PDF块")
                return successful_chunks
            else:
                print("PyPDF2未能创建任何有效的PDF块，尝试使用替代方法")
                return try_alternative_pdf_split(pdf_file)
                
    except Exception as e:
        # PyPDF2 对部分可修复 PDF 比较敏感，先走 PyMuPDF 兜底；若兜底也打不开，
        # 由 try_alternative_pdf_split 抛出源文件错误，避免坏 PDF 进入 OCR 重试风暴。
        return try_alternative_pdf_split(pdf_file, original_error=e)

def try_alternative_pdf_split(pdf_file, original_error=None):
    """
    使用替代方法分割PDF文件
    """
    doc = None
    new_doc = None
    
    try:
        import fitz  # PyMuPDF
        
        output_dir = tempfile.gettempdir()
        filename = os.path.basename(pdf_file)
        name, ext = os.path.splitext(filename)
        
        # PyMuPDF 对加密和有保护的PDF有更好的兼容性
        doc = fitz.open(pdf_file)
        
        # 检查是否需要密码
        if doc.needs_pass:
            if not doc.authenticate(""):
                doc.close()
                return None
        
        total_pages = len(doc)
        # print(f"PyMuPDF检测到PDF总页数: {total_pages}")
        
        if total_pages == 0:
            doc.close()
            return None
            
        successful_chunks = []
        pages_per_chunk = 50
        current_chunk = 1
        
        for start_page in range(0, total_pages, pages_per_chunk):
            end_page = min(start_page + pages_per_chunk, total_pages)
            new_doc = None  # 在循环内重置
            
            try:
                # 创建新的PDF文档
                new_doc = fitz.open()
                
                # 复制页面
                added_pages = 0
                for page_num in range(start_page, end_page):
                    try:
                        new_doc.insert_pdf(doc, from_page=page_num, to_page=page_num)
                        added_pages += 1
                    except Exception as page_error:
                        print(f"处理第 {page_num + 1} 页时出错，跳过此页: {page_error}")
                        continue
            
                # 如果成功添加了页面，保存文件
                if added_pages > 0:
                    output_path = os.path.join(output_dir, f"{name}_chunk{current_chunk}{ext}")
                    new_doc.save(output_path)
                    
                    # 立即关闭new_doc
                    new_doc.close()
                    new_doc = None
                    
                    # 强制文件系统同步
                    try:
                        os.sync()
                    except AttributeError:
                        pass
                    
                    # 文件验证逻辑...
                    max_retries = 5
                    retry_count = 0
                    file_ready = False
                    
                    while retry_count < max_retries and not file_ready:
                        if os.path.exists(output_path):
                            try:
                                with open(output_path, 'rb') as test_file:
                                    test_data = test_file.read(1024)
                                    if len(test_data) > 0:
                                        file_ready = True
                                        break
                            except (IOError, OSError) as e:
                                print(f"文件验证失败，重试 {retry_count + 1}/{max_retries}: {e}")
                        
                        retry_count += 1
                        time.sleep(0.1)
                    
                    if file_ready:
                        successful_chunks.append(output_path)
                        # print(f"PyMuPDF成功创建PDF块: {output_path}, 包含 {added_pages} 页")
                    else:
                        print(f"PyMuPDF: 文件创建验证失败: {output_path}")
                else:
                    # 如果没有添加页面，也要关闭new_doc。PyMuPDF 文档对象的
                    # truthiness 会读取页数，关闭后再判断会触发 document closed。
                    if new_doc is not None:
                        new_doc.close()
                        new_doc = None
                    print(f"PyMuPDF: 块 {current_chunk} 没有有效页面，跳过")
                
                current_chunk += 1
            
            except Exception as chunk_error:
                print(f"PyMuPDF: 创建块 {current_chunk} 时出错: {chunk_error}")
                if new_doc is not None:
                    try:
                        new_doc.close()
                    except Exception:
                        pass
                    new_doc = None
                current_chunk += 1
                continue
        
        return successful_chunks
            
    except ImportError:
        return None
    except Exception as e:
        import traceback
        traceback.print_exc()
        if _is_invalid_pdf_error(e) or (original_error is not None and _is_invalid_pdf_error(original_error)):
            raise SourceDocumentParseError(
                f"PDF源文件无法打开或已损坏，请检查文件格式是否正确: {pdf_file}"
            ) from e
        return None
    finally:
        # 确保所有资源都被释放
        if doc is not None:
            try:
                doc.close()
            except Exception:
                pass
        if new_doc is not None:
            try:
                new_doc.close()
            except Exception:
                pass
        
        # 强制垃圾回收
        gc.collect()

def _detect_pdf_error_type(error):
    """
    检测和分类PDF错误类型

    Args:
        error: Exception对象

    Returns:
        tuple: (error_type, is_recoverable, description)
    """
    error_str = str(error)

    # ExtGState资源错误
    if "cannot find ExtGState resource" in error_str or "ExtGState" in error_str:
        return ("ExtGState", True, "ExtGState资源缺失（图形状态）")

    # 字体资源错误
    if "cannot find Font resource" in error_str or "Font" in error_str:
        return ("Font", True, "字体资源缺失")

    # 颜色空间错误
    if "cannot find ColorSpace resource" in error_str or "ColorSpace" in error_str:
        return ("ColorSpace", True, "颜色空间资源缺失")

    # XObject资源错误
    if "cannot find XObject resource" in error_str:
        return ("XObject", True, "XObject资源缺失")

    # 语法错误
    if "syntax error" in error_str.lower():
        return ("Syntax", True, "PDF语法错误")

    # 内存错误
    if "memory" in error_str.lower() or "MemoryError" in error_str:
        return ("Memory", False, "内存不足")

    # 其他未知错误
    return ("Unknown", False, f"未知错误: {error_str[:100]}")

def _open_pdf_with_repair(pdf_path, max_retries=2):
    """
    尝试打开PDF文件，如果失败则尝试修复模式

    Args:
        pdf_path: PDF文件路径
        max_retries: 最大重试次数

    Returns:
        fitz.Document对象，如果失败返回None
    """
    pdf_document = None

    for attempt in range(max_retries):
        try:
            if attempt == 0:
                # 第一次尝试：正常打开
                pdf_document = fitz.open(pdf_path)
                print(f"成功打开PDF: {pdf_path}")
            else:
                # 后续尝试：使用修复模式
                print(f"尝试使用修复模式打开PDF (尝试 {attempt + 1}/{max_retries})")
                # PyMuPDF会自动尝试修复一些常见问题
                pdf_document = fitz.open(pdf_path)

            # 检查是否需要密码
            if pdf_document.needs_pass:
                if not pdf_document.authenticate(""):
                    print(f"PDF需要密码: {pdf_path}")
                    if pdf_document:
                        pdf_document.close()
                    return None

            # 验证文档是否有效
            if len(pdf_document) == 0:
                print(f"PDF文档无有效页面: {pdf_path}")
                if pdf_document:
                    pdf_document.close()
                return None

            return pdf_document

        except Exception as e:
            error_type, is_recoverable, description = _detect_pdf_error_type(e)
            print(f"打开PDF失败 (尝试 {attempt + 1}/{max_retries}): {description}")

            if pdf_document:
                try:
                    pdf_document.close()
                except:
                    pass
                pdf_document = None

            if not is_recoverable or attempt == max_retries - 1:
                print(f"无法打开PDF文件: {pdf_path}")
                return None

            time.sleep(0.5)

    return None

def _try_render_with_fallback(page, rect, output_path, target_dpi=300):
    """
    多级降级渲染策略

    Args:
        page: PyMuPDF页面对象
        rect: 要渲染的矩形区域
        output_path: 输出图片路径
        target_dpi: 目标DPI

    Returns:
        tuple: (success, method_used, error_message)
    """
    img = None
    pix = None

    try:
        # Level 1: 标准高质量渲染 (300 DPI, alpha=False)
        try:
            dpi_scale = target_dpi / 72
            matrix = fitz.Matrix(dpi_scale, dpi_scale)
            pix = page.get_pixmap(matrix=matrix, clip=rect, alpha=False)

            if pix.width > 0 and pix.height > 0:
                img_data = pix.tobytes("png")
                pix = None

                with io.BytesIO(img_data) as img_buffer:
                    img = Image.open(img_buffer)
                    img = img.copy()

                img_data = None

                if img.mode != 'RGB':
                    new_img = img.convert('RGB')
                    img.close()
                    img = new_img

                os.makedirs(os.path.dirname(output_path), exist_ok=True)
                dpi_value = int(round(max(72, target_dpi)))
                img.save(output_path, "JPEG", quality=95, optimize=True, dpi=(dpi_value, dpi_value))

                print(f"Level 1渲染成功: 标准高质量 ({target_dpi} DPI)")
                return (True, "Level1_Standard", None)

        except Exception as e1:
            error_type, is_recoverable, description = _detect_pdf_error_type(e1)
            print(f"Level 1渲染失败 ({description}), 尝试 Level 2")

            if not is_recoverable:
                return (False, "Level1_Failed", description)

        # Level 2: 降低DPI渲染 (不高于150 DPI)
        try:
            lower_dpi = max(72, min(target_dpi, 150))
            dpi_scale = lower_dpi / 72
            matrix = fitz.Matrix(dpi_scale, dpi_scale)
            pix = page.get_pixmap(matrix=matrix, clip=rect, alpha=False)

            if pix.width > 0 and pix.height > 0:
                img_data = pix.tobytes("png")
                pix = None

                with io.BytesIO(img_data) as img_buffer:
                    img = Image.open(img_buffer)
                    img = img.copy()

                img_data = None

                if img.mode != 'RGB':
                    new_img = img.convert('RGB')
                    img.close()
                    img = new_img

                os.makedirs(os.path.dirname(output_path), exist_ok=True)
                dpi_value = int(round(max(72, lower_dpi)))
                img.save(output_path, "JPEG", quality=90, optimize=True, dpi=(dpi_value, dpi_value))

                print(f"Level 2渲染成功: 降低DPI ({lower_dpi} DPI)")
                return (True, "Level2_LowerDPI", None)

        except Exception as e2:
            error_type, is_recoverable, description = _detect_pdf_error_type(e2)
            print(f"Level 2渲染失败 ({description}), 尝试 Level 3")

        # Level 3: 整页渲染后裁剪
        try:
            print("Level 3: 尝试整页渲染后裁剪")

            # 获取页面实际尺寸
            page_rect = page.rect

            # 使用较低的DPI渲染整个页面
            whole_page_dpi = max(72, min(target_dpi, 150))
            dpi_scale = whole_page_dpi / 72
            matrix = fitz.Matrix(dpi_scale, dpi_scale)

            # 渲染整个页面
            pix = page.get_pixmap(matrix=matrix, alpha=False)

            if pix.width > 0 and pix.height > 0:
                img_data = pix.tobytes("png")
                pix = None

                with io.BytesIO(img_data) as img_buffer:
                    whole_img = Image.open(img_buffer)
                    whole_img = whole_img.copy()

                img_data = None

                # 计算裁剪区域（按比例缩放坐标）
                scale_x = whole_img.width / page_rect.width
                scale_y = whole_img.height / page_rect.height

                crop_left = int(rect.x0 * scale_x)
                crop_top = int(rect.y0 * scale_y)
                crop_right = int(rect.x1 * scale_x)
                crop_bottom = int(rect.y1 * scale_y)

                # 确保裁剪区域在图像范围内
                crop_left = max(0, min(crop_left, whole_img.width))
                crop_top = max(0, min(crop_top, whole_img.height))
                crop_right = max(0, min(crop_right, whole_img.width))
                crop_bottom = max(0, min(crop_bottom, whole_img.height))

                # 裁剪图像
                if crop_right > crop_left and crop_bottom > crop_top:
                    img = whole_img.crop((crop_left, crop_top, crop_right, crop_bottom))
                    whole_img.close()

                    if img.mode != 'RGB':
                        new_img = img.convert('RGB')
                        img.close()
                        img = new_img

                    os.makedirs(os.path.dirname(output_path), exist_ok=True)
                    dpi_value = int(round(max(72, whole_page_dpi)))
                    img.save(output_path, "JPEG", quality=85, optimize=True, dpi=(dpi_value, dpi_value))

                    print(f"Level 3渲染成功: 整页渲染后裁剪 ({whole_page_dpi} DPI)")
                    return (True, "Level3_WholePage", None)
                else:
                    whole_img.close()
                    return (False, "Level3_Failed", "裁剪区域无效")

        except Exception as e3:
            error_type, is_recoverable, description = _detect_pdf_error_type(e3)
            print(f"Level 3渲染失败 ({description})")
            return (False, "Level3_Failed", description)

        return (False, "AllLevels_Failed", "所有渲染策略均失败")

    finally:
        # 清理资源
        try:
            if img:
                img.close()
        except:
            pass

        try:
            if pix:
                pix = None
        except:
            pass

        gc.collect()


def _rect_intersection_area(rect1, rect2):
    """Return the overlapping area between two rectangles."""
    x0 = max(rect1.x0, rect2.x0)
    y0 = max(rect1.y0, rect2.y0)
    x1 = min(rect1.x1, rect2.x1)
    y1 = min(rect1.y1, rect2.y1)

    if x1 <= x0 or y1 <= y0:
        return 0.0
    return (x1 - x0) * (y1 - y0)


def _estimate_target_dpi(page, target_rect, fallback_dpi=300):
    """
    Estimate a DPI that matches the original image resolution on the page.
    Falls back to the provided DPI if no matching image is found.
    """
    try:
        images = page.get_images(full=True)
    except Exception:
        return fallback_dpi

    best_dpi = None
    best_score = 0.0
    target_area = max(target_rect.width * target_rect.height, 1e-6)

    for image_entry in images:
        if not image_entry:
            continue

        xref = image_entry[0]
        width_px = image_entry[2] if len(image_entry) > 2 else None
        height_px = image_entry[3] if len(image_entry) > 3 else None

        if not width_px or not height_px:
            continue

        try:
            rects = page.get_image_rects(xref)
        except Exception:
            continue

        for image_rect in rects:
            overlap_area = _rect_intersection_area(target_rect, image_rect)
            if overlap_area <= 0:
                continue

            image_area = max(image_rect.width * image_rect.height, 1e-6)
            coverage_ratio = overlap_area / image_area
            target_coverage = overlap_area / target_area
            match_score = min(coverage_ratio, target_coverage)

            # Require the regions to align closely to avoid mismatching small overlaps.
            if match_score < 0.75:
                continue

            px_per_point_x = width_px / max(image_rect.width, 1e-6)
            px_per_point_y = height_px / max(image_rect.height, 1e-6)

            dpi_x = px_per_point_x * 72
            dpi_y = px_per_point_y * 72
            candidate_dpi = max(dpi_x, dpi_y)

            if candidate_dpi <= 0:
                continue

            if match_score > best_score:
                best_score = match_score
                best_dpi = candidate_dpi

    if best_dpi:
        return max(72, best_dpi)

    return fallback_dpi

def extract_image_from_pdf_page(pdf_path, page_number, coordinates, image_id, output_path, page_dimensions=None):
    pdf_document = None
    pix = None
    img = None
    img_data = None
    page = None  # 添加页面变量
    
    try:
        # 使用带修复功能的PDF打开
        pdf_document = _open_pdf_with_repair(pdf_path, max_retries=2)

        if pdf_document is None:
            print(f"无法打开PDF文件: {pdf_path}")
            return False

        if page_number >= len(pdf_document):
            print(f"页面编号超出范围: {page_number} >= {len(pdf_document)}")
            return False
            
        # 获取指定页面
        page = pdf_document[page_number]

        # 获取页面的实际尺寸（PyMuPDF坐标系统）
        page_rect = page.rect
        actual_width = page_rect.width
        actual_height = page_rect.height
        
        # 如果有页面尺寸信息，进行坐标转换
        if page_dimensions:
            ocr_width = page_dimensions.get('width', actual_width)
            ocr_height = page_dimensions.get('height', actual_height)
            ocr_dpi = page_dimensions.get('dpi', 72)
            
            # 计算缩放比例
            width_scale = actual_width / ocr_width
            height_scale = actual_height / ocr_height
            
            # 转换坐标
            scaled_coords = {
                "top_left_x": coordinates["top_left_x"] * width_scale,
                "top_left_y": coordinates["top_left_y"] * height_scale,
                "bottom_right_x": coordinates["bottom_right_x"] * width_scale,
                "bottom_right_y": coordinates["bottom_right_y"] * height_scale
            }
        else:
            # 如果没有尺寸信息，直接使用原坐标
            scaled_coords = coordinates
        
        # 确保坐标在页面范围内
        scaled_coords["top_left_x"] = max(0, min(scaled_coords["top_left_x"], actual_width))
        scaled_coords["top_left_y"] = max(0, min(scaled_coords["top_left_y"], actual_height))
        scaled_coords["bottom_right_x"] = max(0, min(scaled_coords["bottom_right_x"], actual_width))
        scaled_coords["bottom_right_y"] = max(0, min(scaled_coords["bottom_right_y"], actual_height))
        
        # 确保坐标顺序正确
        if scaled_coords["top_left_x"] > scaled_coords["bottom_right_x"]:
            scaled_coords["top_left_x"], scaled_coords["bottom_right_x"] = scaled_coords["bottom_right_x"], scaled_coords["top_left_x"]
        if scaled_coords["top_left_y"] > scaled_coords["bottom_right_y"]:
            scaled_coords["top_left_y"], scaled_coords["bottom_right_y"] = scaled_coords["bottom_right_y"], scaled_coords["top_left_y"]
        
        # 定义截取区域 (x0, y0, x1, y1)
        rect = fitz.Rect(
            scaled_coords["top_left_x"], 
            scaled_coords["top_left_y"],
            scaled_coords["bottom_right_x"], 
            scaled_coords["bottom_right_y"]
        )
        
        # 检查截取区域是否有效
        if rect.width <= 0 or rect.height <= 0:
            print(f"无效的截取区域: width={rect.width}, height={rect.height}")
            return False

        # 使用多级降级渲染策略
        target_dpi = _estimate_target_dpi(page, rect, fallback_dpi=300)
        success, method_used, error_message = _try_render_with_fallback(
            page, rect, output_path, target_dpi=target_dpi
        )

        if success:
            print(f"图片提取成功 (方法: {method_used}): {output_path}")
            return True
        else:
            print(f"图片提取失败 (方法: {method_used}): {error_message}")
            return False

    except Exception as e:
        error_type, is_recoverable, description = _detect_pdf_error_type(e)
        print(f"截取图片时出错 ({error_type}): {description}")
        traceback.print_exc()
        return False
    finally:
        # 确保所有资源都被释放 - 按正确顺序
        try:
            if img:
                img.close()
        except:
            pass
        img = None
        
        try:
            if pix:
                pix = None
        except:
            pass
            
        try:
            img_data = None
        except:
            pass
            
        try:
            if page:
                page = None
        except:
            pass
            
        try:
            if pdf_document:
                pdf_document.close()
        except:
            pass
        pdf_document = None
        
        # 安全地清理局部变量
        try:
            if 'page_rect' in locals():
                del page_rect
            if 'actual_width' in locals():
                del actual_width
            if 'actual_height' in locals():
                del actual_height
            if 'scaled_coords' in locals():
                del scaled_coords
            if 'rect' in locals():
                del rect
            if 'matrix' in locals():
                del matrix
        except:
            pass
        
        # 强制垃圾回收
        gc.collect()

def process_pdf_chunk(pdf_path, index, save_images=False, output_dir=None, user_id=None, file_remark=""):
    pdfMd5 = md5(pdf_path.encode()).hexdigest()
    output_dir = _pdf_image_storage_dir(pdfMd5, output_dir)
    
    # 添加资源管理 - 在函数开始就初始化所有变量
    pdf_document = None
    uploaded_pdf = None
    ocr_response = None
    
    try:
        # 检查文件是否存在，增加重试机制
        max_file_check_retries = 3
        file_exists = False
        
        for retry in range(max_file_check_retries):
            if os.path.exists(pdf_path):
                # 验证文件可读性
                try:
                    with open(pdf_path, 'rb') as test_file:
                        test_file.read(1024)
                    file_exists = True
                    break
                except Exception as e:
                    print(f"文件读取测试失败，重试 {retry + 1}/{max_file_check_retries}: {e}")
                    time.sleep(0.2)
            else:
                print(f"文件不存在，重试 {retry + 1}/{max_file_check_retries}: {pdf_path}")
                time.sleep(0.2)
        
        if not file_exists:
            print(f"错误: PDF文件验证失败: {pdf_path}")
            return None
            
        print(f"开始处理PDF块 {index}: {pdf_path}")
        
        # 使用with语句确保文件正确关闭
        # with open(pdf_path, "rb") as pdf_file:
        #     uploaded_pdf = clients[chosen_api_key % len(clients)].files.upload(
        #         file={
        #             "file_name": os.path.basename(pdf_path),
        #             "content": pdf_file,
        #         },
        #         purpose="ocr",
        #     )
        
        # print(f"文件上传成功，文件ID: {uploaded_pdf.id}")
        # 获取pdf_path tempfile.gettempdir()下的相对路径
        path_relative = os.path.relpath(pdf_path, start=tempfile.gettempdir())
        # retrieved_file = clients[chosen_api_key % len(clients)].files.retrieve(file_id=uploaded_pdf.id)
        # signed_url = clients[chosen_api_key % len(clients)].files.get_signed_url(file_id=uploaded_pdf.id)
        
        # Retry logic for OCR processing with robust handling of transient proxy/upstream errors.
        max_retries = max(1, _env_int("MISTRAL_OCR_MAX_RETRIES", 10))
        retry_forever = _mistral_ocr_retry_forever()
        retry_count = 0
        ocr_response = None

        while retry_forever or retry_count < max_retries:
            try:
                lease_timeout = _get_mistral_ocr_balancer().default_timeout
                with _get_mistral_ocr_balancer().lease(timeout=lease_timeout) as client:
                    ocr_response = _call_mistral_ocr(
                        client,
                        {
                            "type": "document_url",
                            "document_url": value.tempfileserverurl + path_relative,
                        },
                        include_image_base64=save_images,
                    )
                break  # Success
            except Exception as ocr_error:
                if _is_mistral_ocr_non_retryable_error(ocr_error):
                    print(f"OCR non-retryable error for chunk {index}: {ocr_error}")
                    return None

                if not _is_mistral_ocr_retryable_error(ocr_error):
                    raise ocr_error

                retry_count += 1
                wait_sec = _mistral_ocr_retry_wait_seconds(retry_count)
                retry_label = f"{retry_count}/forever" if retry_forever else f"{retry_count}/{max_retries}"
                print(f"OCR transient error, retry {retry_label} in {wait_sec:.1f}s: {ocr_error}")
                if retry_forever or retry_count < max_retries:
                    time.sleep(wait_sec)
                    continue
                print(f"OCR processing failed after {max_retries} retries, skipping chunk {index}")
                return None
        
        if ocr_response is None:
            print(f"Failed to get OCR response for chunk {index}, skipping")
            return None
        
        result_pages = []
        chunk_markdown = ""
        extracted_images = []
        total_add_space = 0

        # 图片提取统计
        image_stats = {
            "total": 0,
            "success": 0,
            "failed": 0,
            "base64_success": 0,
            "coordinate_success": 0,
            "embedded_success": 0,
        }
        
        for page in ocr_response.pages:
            images = getattr(page, 'images', None) or []
            if images and save_images and output_dir:
                os.makedirs(output_dir, exist_ok=True)
                for image in images:
                    image_stats["total"] += 1

                    # 构建图片文件名和路径
                    image_filename = f"page_{page.index}_chunk_{index}_img_{image.id}.jpeg"
                    image_path = os.path.join(output_dir, image_filename)

                    # 优先使用base64数据，如果没有则根据坐标截取
                    success = False
                    extraction_method = "unknown"

                    if hasattr(image, 'image_base64') and image.image_base64:
                        # 使用base64数据保存图片
                        try:
                            image_data = base64.b64decode(_strip_data_url_base64(image.image_base64))
                            with open(image_path, 'wb') as img_file:
                                img_file.write(image_data)
                            success = True
                            extraction_method = "base64"
                            image_stats["base64_success"] += 1

                            # 立即清理图片数据
                            image.image_base64 = None

                        except Exception as img_error:
                            print(f"Base64解码失败: {img_error}, 尝试使用坐标截取")

                    # 如果base64方式失败或不存在，使用坐标截取
                    if not success and hasattr(image, 'top_left_x'):
                        if not os.path.exists(pdf_path):
                            image_stats["failed"] += 1
                            continue

                        coordinates = {
                            "top_left_x": image.top_left_x,
                            "top_left_y": image.top_left_y,
                            "bottom_right_x": image.bottom_right_x,
                            "bottom_right_y": image.bottom_right_y
                        }

                        # 获取页面尺寸信息
                        page_dimensions = None
                        if hasattr(page, 'dimensions'):
                            page_dimensions = {
                                'width': page.dimensions.width,
                                'height': page.dimensions.height,
                                'dpi': page.dimensions.dpi
                            }

                        success = extract_image_from_pdf_page(
                            pdf_path, page.index, coordinates, image.id, image_path, page_dimensions
                        )

                        if success:
                            extraction_method = "coordinate"
                            image_stats["coordinate_success"] += 1

                    # 如果成功截取图片，添加到结果中
                    if success:
                        image_stats["success"] += 1
                        image_size = os.path.getsize(image_path)
                        total_add_space += image_size
                        imageMd5 = md5(open(image_path, 'rb').read()).hexdigest()
                        objKey = f"/pdf_images/{pdfMd5}/{image_filename}"
                        
                        exist_file = database.get_file_by_obj_key(objKey)
                        if exist_file is None:
                            try:
                                database.insert_file("storage/pdf_images/" + pdfMd5 + "/" + image_filename, objKey,image_filename, "jpeg", image_size, imageMd5, None, False,file_remark=file_remark)
                            except Exception as e:
                                print(f"插入文件记录失败: {e}")
                                traceback.print_exc()
                                pass
                        
                        extracted_images.append({
                            "image_id": image.id,
                            "page_number": page.index,
                            "chunk_index": index,
                            "image_path": image_path,
                            "url": value.storageUrl + objKey,
                            "obj_key": objKey,
                            "md5": imageMd5,
                            "coordinates": {
                                "top_left_x": getattr(image, 'top_left_x', 0),
                                "top_left_y": getattr(image, 'top_left_y', 0),
                                "bottom_right_x": getattr(image, 'bottom_right_x', 0),
                                "bottom_right_y": getattr(image, 'bottom_right_y', 0)
                            },
                            "dimensions": page.dimensions.__dict__ if hasattr(page, 'dimensions') else None
                        })
                        
                        # 替换markdown中的图片链接
                        page.markdown = _replace_pdf_image_markers(page.markdown, image.id, value.storageUrl + objKey)
                    else:
                        image_stats["failed"] += 1
                        print(f"图片截取失败 (方法: {extraction_method}): {image_path}")
            
            # 清理页面中的图片数据
            if hasattr(page, 'images') and page.images:
                for image in page.images:
                    if hasattr(image, 'image_base64'):
                        image.image_base64 = None

            if save_images and output_dir:
                _append_pdf_embedded_images(
                    pdf_path,
                    index,
                    pdfMd5,
                    output_dir,
                    [page],
                    extracted_images,
                    file_remark=file_remark,
                    image_stats=image_stats,
                )
            
            chunk_markdown += page.markdown
            result_pages.append(
                {
                    "page_number": page.index,
                    "markdown": page.markdown,
                    "chunk_index": index,
                    "images": [img for img in extracted_images if img["page_number"] == page.index]
                }
            )

        # 打印图片提取统计信息
        if image_stats["total"] > 0:
            success_rate = (image_stats["success"] / image_stats["total"]) * 100
            print(f"\n=== PDF块 {index} 图片提取统计 ===")
            print(f"总图片数: {image_stats['total']}")
            print(f"成功提取: {image_stats['success']} ({success_rate:.1f}%)")
            print(f"提取失败: {image_stats['failed']}")
            print(f"  - Base64方式成功: {image_stats['base64_success']}")
            print(f"  - 坐标截取成功: {image_stats['coordinate_success']}")
            print(f"  - PDF内嵌图片兜底成功: {image_stats['embedded_success']}")
            print(f"总存储空间: {total_add_space / 1024:.2f} KB")
            print(f"================================\n")

        return chunk_markdown, result_pages, extracted_images
        
    except Exception as e:
        print(f"Error processing chunk {index}: {e}")
        print(f"错误类型: {type(e)}")
        traceback.print_exc()
        if "Could not parse body - request body did not fit into client body buffer, consider raising 'client_body_buffer_size'" in str(e):
            print("Error: Request body too large, skipping this chunk.")
            return None
        if _is_ocr_dependency_error(e) or _is_mistral_ocr_retryable_error(e):
            raise TransientParseError(f"Mistral OCR块处理依赖暂不可用（可重试）: {e}") from e
        print("Skipping chunk after unexpected error without retry escalation.")
        return None
    finally:
        # 确保资源被释放
        # if uploaded_pdf is not None:
        #     try:
        #         clients[chosen_api_key % len(clients)].files.delete(file_id=uploaded_pdf.id)
        #     except:
        #         pass
        
        # 清理OCR响应对象
        if ocr_response is not None:
            try:
                # 清理OCR响应中的图片数据
                for page in ocr_response.pages:
                    if hasattr(page, 'images'):
                        for img in page.images:
                            if hasattr(img, 'image_base64'):
                                img.image_base64 = None
                del ocr_response
            except:
                pass
        
        # 强制垃圾回收
        gc.collect()

def perform_pdf_ocr(pdf_path, save_images=False, output_dir=None, user_id=None):
    """
    Perform OCR on the PDF file.
    """
    if not os.path.exists(pdf_path):
        print(f"Error: The file {pdf_path} was not found.")
        return None

    provider = _ocr_provider()
    provider_result = _try_pdf_ocr_provider(
        provider,
        pdf_path,
        save_images=save_images,
        output_dir=output_dir,
    )
    if provider_result:
        return provider_result
    if _is_modern_siliconflow_ocr_provider(provider):
        print(
            f"硅基流动现代 OCR 通道未返回结果，跳过旧OCR路径回退: provider={provider}, file={pdf_path}",
            flush=True,
        )
        return None
    
    cuted_pdfs = None
    results = []
    
    try:
        cuted_pdfs = cut_pdf(pdf_path)
        if cuted_pdfs is None:
            print(f"Error: Failed to cut the PDF file {pdf_path}.")
            return None
        
        markdown_result = ""
        markdown_result_pages = []
        all_extracted_images = []
        
        def process_pdf_chunk_wrapper(pdf_path, index, user_id=None, file_remark=""):
            return process_pdf_chunk(pdf_path, index, save_images, output_dir, user_id, file_remark)

        chunk_worker_limit = _env_int("MISTRAL_OCR_CHUNK_MAX_WORKERS", 0)
        if chunk_worker_limit <= 0:
            chunk_worker_limit = max(4, _get_mistral_ocr_balancer().total_slots)
        max_workers = max(1, min(len(cuted_pdfs), chunk_worker_limit))
        chunk_results = [None] * len(cuted_pdfs)
        with ThreadPoolExecutor(max_workers=max_workers) as executor:
            futures = [
                executor.submit(process_pdf_chunk_wrapper, pdf, idx, user_id=user_id, file_remark="attachment for " + pdf_path)
                for idx, pdf in enumerate(cuted_pdfs)
            ]

            for idx, future in enumerate(futures):
                try:
                    chunk_results[idx] = future.result()
                except TransientParseError:
                    raise
                except Exception as e:
                    print(f"处理future结果时出错 (块 {idx}): {e}")
                    chunk_results[idx] = None

        # 失败块串行补试一轮；仍失败则整篇判定失败，
        # 避免缺页文档被静默标记为解析成功
        failed_indexes = [i for i, r in enumerate(chunk_results) if r is None]
        for i in failed_indexes:
            print(f"PDF块 {i} 处理失败，串行补试一次")
            try:
                chunk_results[i] = process_pdf_chunk_wrapper(
                    cuted_pdfs[i], i, user_id=user_id, file_remark="attachment for " + pdf_path
                )
            except TransientParseError:
                raise
            except Exception as e:
                print(f"PDF块 {i} 补试仍失败: {e}")
                chunk_results[i] = None

        still_failed = [i for i, r in enumerate(chunk_results) if r is None]
        if still_failed:
            print(
                f"错误: {len(still_failed)}/{len(cuted_pdfs)} 个PDF块处理失败 "
                f"(块索引: {still_failed[:10]})，整篇判定解析失败"
            )
            fallback_provider = _ocr_fallback_provider()
            if fallback_provider:
                fallback_result = _try_pdf_ocr_provider(
                    fallback_provider,
                    pdf_path,
                    save_images=save_images,
                    output_dir=output_dir,
                )
                if fallback_result:
                    return fallback_result
            return None

        for result in chunk_results:
            chunk_markdown, chunk_pages, chunk_images = result
            markdown_result += chunk_markdown
            markdown_result_pages.extend(chunk_pages)
            all_extracted_images.extend(chunk_images)

            # 立即清理处理后的结果
            del chunk_markdown, chunk_pages, chunk_images
        del chunk_results
        gc.collect()

        # 如果没有任何成功的结果，返回None
        if not markdown_result:
            print("错误: 所有PDF块都处理失败")
            fallback_provider = _ocr_fallback_provider()
            if fallback_provider:
                fallback_result = _try_pdf_ocr_provider(
                    fallback_provider,
                    pdf_path,
                    save_images=save_images,
                    output_dir=output_dir,
                )
                if fallback_result:
                    return fallback_result
            return None

        # 重新编排页码
        for i in range(len(markdown_result_pages)):
            markdown_result_pages[i]["page_number"] = i + 1
            
        # 返回结果
        if save_images:
            return markdown_result, markdown_result_pages, all_extracted_images
        else:
            return markdown_result, markdown_result_pages
            
    finally:
        # 清理临时文件
        if cuted_pdfs:
            for pdf in cuted_pdfs:
                try:
                    if os.path.exists(pdf):
                        os.remove(pdf)
                except Exception as e:
                    print(f"Error removing file {pdf}: {e}")
        
        # 清理变量
        if 'results' in locals():
            results.clear()
        
        # 强制垃圾回收
        gc.collect()
def _find_markdown_elements(text):
    """
    增强版的 Markdown 元素查找函数
    """
    ranges = []

    # 原有的匹配规则
    # Match links: [text](url)
    for match in re.finditer(r"\[.*?\]\(.*?\)", text):
        ranges.append((match.start(), match.end()))

    # Match images: ![alt](url)  
    for match in re.finditer(r"!\[.*?\]\(.*?\)", text):
        ranges.append((match.start(), match.end()))

    # Match inline code: `code` or ``code``
    for match in re.finditer(r"(`+)(.*?)\1", text, re.DOTALL):
        ranges.append((match.start(), match.end()))

    # 新增的 Markdown 元素
    # 代码块 ```code```
    for match in re.finditer(r"```[\s\S]*?```", text):
        ranges.append((match.start(), match.end()))
    
    # HTML 标签
    for match in re.finditer(r"<[^>]+>", text):
        ranges.append((match.start(), match.end()))
    
    # 粗体 **text** 或 __text__
    for match in re.finditer(r"(\*\*|__)(.*?)\1", text):
        ranges.append((match.start(), match.end()))
    
    # 斜体 *text* 或 _text_ (但要避免与粗体冲突)
    for match in re.finditer(r"(?<!\*)\*(?!\*)([^*]+?)\*(?!\*)", text):
        ranges.append((match.start(), match.end()))
    for match in re.finditer(r"(?<!_)_(?!_)([^_]+?)_(?!_)", text):
        ranges.append((match.start(), match.end()))
    
    # 删除线 ~~text~~
    for match in re.finditer(r"~~(.*?)~~", text):
        ranges.append((match.start(), match.end()))
    
    # 引用参考链接 [text][ref]
    for match in re.finditer(r"\[.*?\]\[.*?\]", text):
        ranges.append((match.start(), match.end()))
    
    # 脚注 [^footnote]
    for match in re.finditer(r"\[\^[^\]]+\]", text):
        ranges.append((match.start(), match.end()))
    
    # 表格行（简单检测）
    for match in re.finditer(r"\|.*?\|", text):
        ranges.append((match.start(), match.end()))

    # Merge overlapping ranges
    if not ranges:
        return []

    ranges.sort()
    merged = [list(ranges[0])]
    for current_start, current_end in ranges[1:]:
        last_start, last_end = merged[-1]
        if current_start <= last_end:
            merged[-1][1] = max(last_end, current_end)
        else:
            merged.append([current_start, current_end])

    return merged


def _in_markdown_range(pos, ranges):
    """
    Checks if the given position is inside any of the markdown ranges.
    """
    for start, end in ranges:
        if start <= pos < end:
            return True
    return False

def split_text_by_length(text, max_chars=1000, overlap_chars=50):
    """
    Simple text splitting function that splits text based on character count only.
    
    Args:
        text (str): The text to split
        max_chars (int): Maximum characters per chunk
        overlap_chars (int): Number of characters to overlap between chunks
            
    Returns:
        list: List of text chunks
    """
    if not text:
        return []
        
    if max_chars <= 0:
        raise ValueError("max_chars must be greater than 0")
        
    if overlap_chars < 0:
        raise ValueError("overlap_chars cannot be negative")
        
    # Ensure overlap doesn't exceed half of max_chars to prevent infinite loops
    overlap_chars = min(overlap_chars, max_chars // 2)
        
    chunks = []
    start = 0
    text_len = len(text)
        
    while start < text_len:
        end = min(start + max_chars, text_len)
        chunk = text[start:end].strip()
            
        if chunk:
            chunks.append(chunk)
            
        # If we've reached the end, break
        if end >= text_len:
            break
                
        # Calculate next start position with overlap
        start = end - overlap_chars
            
        # Ensure we're making progress
        if start <= chunks.__len__() and len(chunks) > 0:
            start = start + 1
        
    return chunks

def split_paragraph(
    paragraph,
    max_chars=1000,
    overlap_chars=50,
    split_markers=None,
    min_split_chars=200,
    force_split=False,
    absolute_max_chars=None,  # 新增：绝对最大字符数
    force_max_chars=4000
    
):
    """
    Splits a paragraph into smaller chunks while preserving markdown elements and ensuring overlap.
    """
    if not paragraph:
        return []

    if max_chars <= 0:
        raise ValueError("max_chars must be greater than 0")

    if overlap_chars < 0:
        raise ValueError("overlap_chars cannot be negative")

    if min_split_chars < 0:
        raise ValueError("min_split_chars cannot be negative")

    # 设置绝对最大字符数，如果未指定则为max_chars的2倍
    if absolute_max_chars is None:
        absolute_max_chars = max_chars * 2
    
    # 预处理：去除首尾空白但保留内容
    paragraph = paragraph.strip()
    if not paragraph:
        return []
    
    # 确保重叠字符数不会超过最大字符数的一半，防止无限循环
    overlap_chars = min(overlap_chars, max_chars // 2)

    # 设置默认分割标记
    markers = split_markers if split_markers is not None else ["\n\n", "\n"]

    # 预处理查找 markdown 元素范围
    markdown_ranges = _find_markdown_elements(paragraph)

    chunks = []
    start = 0
    text_len = len(paragraph)
    iteration_count = 0
    max_iterations = text_len + 100

    while start < text_len and iteration_count < max_iterations:
        iteration_count += 1
        end = min(start + max_chars, text_len)

        # 如果已经到达文本末尾，直接添加剩余部分
        if end >= text_len:
            remaining_text = paragraph[start:].strip()
            if remaining_text:
                # 检查剩余文本是否超过绝对最大长度
                if len(remaining_text) > absolute_max_chars:
                    # 强制分割超长文本
                    while len(remaining_text) > absolute_max_chars:
                        chunks.append(remaining_text[:absolute_max_chars].strip())
                        remaining_text = remaining_text[absolute_max_chars - overlap_chars:].strip()
                    if remaining_text:
                        chunks.append(remaining_text)
            break

        # 寻找最佳分割点
        split_pos = end
        search_start = max(start + min_split_chars, start + max_chars // 4)
        search_end = min(end, text_len)

        # 首先检查预设的结束位置是否在 Markdown 元素内
        if _in_markdown_range(end - 1, markdown_ranges):
            # 如果在 Markdown 元素内，寻找元素结束位置
            for range_start, range_end in markdown_ranges:
                if range_start <= end <= range_end:
                    # 检查元素长度是否超过绝对最大限制
                    element_length = range_end - range_start
                    if element_length > absolute_max_chars:
                        # 如果Markdown元素过长，强制分割
                        split_pos = start + absolute_max_chars
                        break
                    elif range_end < text_len:
                        split_pos = range_end
                        break
                    else:
                        # 元素延伸到文本末尾，向前查找分割点
                        for i in range(search_end - 1, search_start - 1, -1):
                            if not _in_markdown_range(i, markdown_ranges):
                                for marker in markers:
                                    if (i + len(marker) <= text_len and 
                                        paragraph[i:i + len(marker)] == marker):
                                        split_pos = i + len(marker)
                                        break
                                if split_pos != end:
                                    break
                        break
        else:
            # 正常的反向查找分割点
            found = False
            for marker in markers:
                marker_len = len(marker)
                for i in range(search_end - marker_len, search_start - 1, -1):
                    if (i + marker_len <= text_len and 
                        paragraph[i:i + marker_len] == marker and
                        not _in_markdown_range(i, markdown_ranges)):
                        split_pos = i + marker_len
                        found = True
                        break
                if found:
                    break
            
            # 如果没找到合适的分割点
            if not found:
                if force_split:
                    # 向前查找，但避免 Markdown 元素
                    for marker in markers:
                        marker_len = len(marker)
                        for i in range(end, min(end + 100, text_len - marker_len + 1)):
                            if (paragraph[i:i + marker_len] == marker and
                                not _in_markdown_range(i, markdown_ranges)):
                                split_pos = i + marker_len
                                break
                        if split_pos > end:
                            break
                
                # 如果仍然没找到，确保至少不在 Markdown 元素中间分割
                if split_pos == end and _in_markdown_range(end - 1, markdown_ranges):
                    # 寻找最近的非 Markdown 区域
                    for i in range(end - 1, start, -1):
                        if not _in_markdown_range(i, markdown_ranges):
                            split_pos = i + 1
                            break

        # 检查当前块是否超过绝对最大长度
        current_chunk_length = split_pos - start
        if current_chunk_length > absolute_max_chars:
            # 强制分割，不考虑Markdown元素
            split_pos = start + absolute_max_chars
            # print(f"警告：检测到超长段落({current_chunk_length}字符)，强制分割到{absolute_max_chars}字符")

        # 添加当前块
        chunk_text = paragraph[start:split_pos].strip()
        if chunk_text:
            # 再次检查块长度
            if len(chunk_text) > absolute_max_chars:
                # print(f"警告：即将添加的块仍然过长({len(chunk_text)}字符)，进行递归分割")
                # 递归分割超长块
                sub_chunks = split_paragraph(
                    chunk_text, 
                    max_chars=max_chars, 
                    overlap_chars=overlap_chars,
                    split_markers=markers,
                    min_split_chars=min_split_chars,
                    force_split=True,
                    absolute_max_chars=absolute_max_chars
                )
                chunks.extend(sub_chunks)
            else:
                chunks.append(chunk_text)

        # 计算下一个块的起始位置
        if split_pos >= text_len:
            break
            
        overlap_start = max(split_pos - overlap_chars, start + 1)
        
        next_start = overlap_start
        if _in_markdown_range(next_start, markdown_ranges):
            for range_start, range_end in markdown_ranges:
                if range_start <= next_start <= range_end:
                    if range_start > start:
                        next_start = range_start
                    else:
                        next_start = min(range_end + 1, split_pos)
                    break
        
        if next_start <= start:
            next_start = start + 1
        
        if next_start >= text_len:
            break
            
        start = next_start

    # 如果达到最大迭代次数，添加警告
    if iteration_count >= max_iterations:
        print(f"警告：split_paragraph 达到最大迭代次数 {max_iterations}，可能存在无限循环")
        if start < text_len:
            remaining_text = paragraph[start:].strip()
            if remaining_text:
                chunks.append(remaining_text)

    # 新增保护机制：如果没有分出任何块，使用简单分段
    if not chunks:
        original_stripped = paragraph.strip()
        if original_stripped:
            print(f"警告：split_paragraph 未能分出任何块，使用 split_text_by_length 进行简单分段")
            chunks = split_text_by_length(
                original_stripped, 
                max_chars=force_max_chars, 
                overlap_chars=overlap_chars
            )
    
    return chunks


def excel_to_html(file_path):
    """
    将Excel文件（.xlsx 或 .xls）转换为HTML表格代码列表，每个工作表对应一个表格。

    参数:
        file_path (str): Excel文件的路径

    返回:
        list: 包含每个工作表HTML表格代码的列表，顺序与Excel中的工作表顺序一致
    """
    # 使用pandas读取Excel文件
    xls = pd.ExcelFile(file_path)
    html_tables = []
    text_contents = []

    # 遍历所有工作表
    for sheet_name in xls.sheet_names:
        # 读取工作表数据并处理空值
        df = pd.read_excel(xls, sheet_name=sheet_name).fillna("")
        # 生成HTML表格代码（不包含索引）
        html_table = df.to_html(index=False, classes="excel-table")
        html_tables.append(html_table)
        # 提取工作表的所有文本内容
        sheet_text = ""
        for col in df.columns:
            sheet_text += " ".join(df[col].astype(str).tolist()) + " "
        text_contents.append(sheet_text.strip())

    return html_tables, text_contents


class _HTMLTextExtractor(HTMLParser):
    def __init__(self):
        super().__init__()
        self._chunks = []
        self._skip_depth = 0

    def handle_starttag(self, tag, attrs):
        tag = (tag or "").lower()
        if tag in {"script", "style", "noscript"}:
            self._skip_depth += 1
            return
        if tag in {"p", "div", "br", "tr", "li", "table", "h1", "h2", "h3", "h4", "h5", "h6"}:
            self._chunks.append("\n")

    def handle_endtag(self, tag):
        tag = (tag or "").lower()
        if tag in {"script", "style", "noscript"} and self._skip_depth > 0:
            self._skip_depth -= 1
            return
        if tag in {"p", "div", "tr", "li", "table", "h1", "h2", "h3", "h4", "h5", "h6"}:
            self._chunks.append("\n")

    def handle_data(self, data):
        if self._skip_depth > 0:
            return
        if data:
            self._chunks.append(data)

    def get_text(self):
        text = "".join(self._chunks)
        text = re.sub(r"[ \t\r\f\v]+", " ", text)
        text = re.sub(r"\n\s*\n\s*\n+", "\n\n", text)
        return text.strip()


def _read_text_file_with_encoding(file_path):
    with open(file_path, "rb") as f:
        raw_data = f.read()
    if not raw_data:
        return ""

    declared_encoding = None
    if raw_data.startswith(b"\xef\xbb\xbf"):
        declared_encoding = "utf-8-sig"
    elif raw_data.startswith(b"\xff\xfe") or raw_data.startswith(b"\xfe\xff"):
        declared_encoding = "utf-16"
    else:
        head = raw_data[:4096].decode("ascii", errors="ignore")
        charset_match = re.search(
            r"charset\s*=\s*['\"]?([a-zA-Z0-9._:-]+)",
            head,
            flags=re.IGNORECASE,
        )
        if charset_match:
            declared_encoding = charset_match.group(1)

    detected = chardet.detect(raw_data) or {}
    encodings = [
        declared_encoding,
        detected.get("encoding"),
        "utf-8",
        "utf-8-sig",
        "gb18030",
        "gbk",
        "gb2312",
        "big5",
        "latin-1",
    ]
    for encoding in [enc for enc in encodings if enc]:
        try:
            return raw_data.decode(encoding, errors="ignore")
        except Exception:
            continue
    return raw_data.decode("utf-8", errors="ignore")


def _html_to_plain_text(html):
    parser = _HTMLTextExtractor()
    try:
        parser.feed(html or "")
        parser.close()
        return parser.get_text()
    except Exception:
        return re.sub(r"<[^>]+>", " ", html or "").strip()


def _looks_like_html_file(file_path):
    try:
        with open(file_path, "rb") as f:
            head = f.read(4096).lstrip().lower()
        return (
            head.startswith(b"<!doctype html")
            or head.startswith(b"<html")
            or b"<html" in head[:1024]
            or b"<body" in head[:2048]
        )
    except Exception:
        return False


def _extract_doc_like_html_text(file_path):
    html = _read_text_file_with_encoding(file_path)
    return _html_to_plain_text(html)


def _extract_html_text(file_path):
    return _html_to_plain_text(_read_text_file_with_encoding(file_path))


def _extract_delimited_text(file_path, delimiter):
    text = _read_text_file_with_encoding(file_path)
    sample = text[:4096]
    if delimiter is None:
        try:
            dialect = csv.Sniffer().sniff(sample)
            delimiter = dialect.delimiter
        except Exception:
            delimiter = ","

    rows = []
    reader = csv.reader(io.StringIO(text), delimiter=delimiter)
    for row in reader:
        cleaned = [cell.strip() for cell in row if cell and cell.strip()]
        if cleaned:
            rows.append(" | ".join(cleaned))
    return "\n".join(rows)


def _extract_textract_text(file_path, extension=None):
    kwargs = {"extension": extension} if extension else {}
    return textract.process(file_path, **kwargs).decode("utf-8", errors="ignore")


def _convert_office_file(file_path, output_ext, export_filter=None, max_retries=3):
    tmp_dir = tempfile.mkdtemp()
    try:
        success, converted_path, error = convert_with_retry(
            libreoffice_pool,
            file_path,
            tmp_dir,
            output_ext,
            max_retries=max_retries,
            base_delay=1.0,
            export_filter=export_filter,
        )
        if not success or not converted_path or not os.path.exists(converted_path):
            raise Exception(f"LibreOffice转换失败: {error}")
        return converted_path, tmp_dir
    except Exception:
        if os.path.exists(tmp_dir):
            shutil.rmtree(tmp_dir)
        raise


def _extract_rtf_text(file_path):
    try:
        return _extract_textract_text(file_path, extension="rtf")
    except Exception as e:
        print(f"RTF textract解析失败，尝试LibreOffice转换: {e}")
        converted_path = None
        tmp_dir = None
        try:
            converted_path, tmp_dir = _convert_office_file(file_path, "txt", max_retries=1)
            return _read_text_file_with_encoding(converted_path)
        finally:
            if tmp_dir and os.path.exists(tmp_dir):
                shutil.rmtree(tmp_dir)


def _extract_odt_text(file_path):
    try:
        return _extract_textract_text(file_path)
    except Exception as e:
        print(f"ODT textract解析失败，尝试LibreOffice转换: {e}")
        converted_path = None
        tmp_dir = None
        try:
            converted_path, tmp_dir = _convert_office_file(file_path, "docx")
            return perform_doc_text_extraction(converted_path)
        finally:
            if tmp_dir and os.path.exists(tmp_dir):
                shutil.rmtree(tmp_dir)


def _extract_ods_text(file_path):
    try:
        _, text_contents = excel_to_html(file_path)
        if text_contents:
            return "\n\n".join(text_contents)
    except Exception as e:
        print(f"ODS直接解析失败，尝试LibreOffice转换: {e}")

    converted_path = None
    tmp_dir = None
    try:
        converted_path, tmp_dir = _convert_office_file(file_path, "xlsx")
        _, text_contents = excel_to_html(converted_path)
        return "\n\n".join(text_contents)
    finally:
        if tmp_dir and os.path.exists(tmp_dir):
            shutil.rmtree(tmp_dir)


def _extract_odp_text(file_path):
    converted_path = None
    tmp_dir = None
    try:
        converted_path, tmp_dir = _convert_office_file(file_path, "pptx")
        text = extract_text_from_pptx(converted_path)
        if text:
            return text
    except Exception as e:
        print(f"ODP转PPTX解析失败，尝试PDF OCR: {e}")
    finally:
        if tmp_dir and os.path.exists(tmp_dir):
            shutil.rmtree(tmp_dir)

    converted_path = None
    tmp_dir = None
    try:
        converted_path, tmp_dir = _convert_office_file(file_path, "pdf")
        result = _parse_pdf_with_native_fallback_to_ocr(converted_path, save_images=True)
        if isinstance(result, tuple):
            return _describe_markdown_images_for_direct_ocr(result[0])
        return result or ""
    finally:
        if tmp_dir and os.path.exists(tmp_dir):
            shutil.rmtree(tmp_dir)


def _extract_text_from_lo_txt(txt_path):
    text = _read_text_file_with_encoding(txt_path)
    return text.strip()


def _convert_doc_to_docx(file_path, tmpdir, max_retries=3):
    filters = [
        "Office Open XML Text",
        "MS Word 2007 XML",
    ]
    last_error = ""
    for export_filter in filters:
        success, converted_docx_path, error = convert_with_retry(
            libreoffice_pool,
            file_path,
            tmpdir,
            "docx",
            max_retries=max_retries,
            base_delay=1.0,
            export_filter=export_filter,
        )
        if success and converted_docx_path and os.path.exists(converted_docx_path):
            print(f"LibreOffice转换成功: {converted_docx_path} filter={export_filter}")
            return converted_docx_path, ""
        last_error = error

    success, converted_docx_path, error = convert_with_retry(
        libreoffice_pool,
        file_path,
        tmpdir,
        "docx",
        max_retries=max_retries,
        base_delay=1.0,
    )
    if success and converted_docx_path and os.path.exists(converted_docx_path):
        print(f"LibreOffice转换成功: {converted_docx_path}")
        return converted_docx_path, ""
    return "", error or last_error


def _convert_doc_to_txt_text(file_path, tmpdir):
    for export_filter in ("Text", "Text (encoded)", None):
        success, converted_txt_path, error = convert_with_retry(
            libreoffice_pool,
            file_path,
            tmpdir,
            "txt",
            max_retries=1,
            base_delay=0.5,
            export_filter=export_filter,
        )
        if success and converted_txt_path and os.path.exists(converted_txt_path):
            text = _extract_text_from_lo_txt(converted_txt_path)
            if text:
                print(f"LibreOffice TXT提取成功: {converted_txt_path} filter={export_filter or 'default'}")
                return text
        else:
            print(f"LibreOffice TXT提取失败: {error}")
    return ""


def _extract_doc_text_fallback(file_path, tmpdir):
    if _looks_like_html_file(file_path):
        text = _extract_doc_like_html_text(file_path)
        if text:
            print(f"HTML伪DOC文本提取成功: {file_path}")
            return text

    text = _convert_doc_to_txt_text(file_path, tmpdir)
    if text:
        return text

    for kwargs in ({"extension": "doc"}, {}):
        try:
            text = textract.process(file_path, **kwargs).decode("utf-8", errors="ignore")
            if text.strip():
                return text
        except Exception as e:
            print(f"DOC textract fallback失败 kwargs={kwargs}: {e}")

    if _looks_like_html_file(file_path):
        text = _extract_doc_like_html_text(file_path)
        if text:
            return text

    return ""


def perform_doc_text_extraction(file_path):
    # textract 处理 docx 和 doc 文件
    if not os.path.exists(file_path):
        raise FileNotFoundError(f"文件不存在: {file_path}")
    # 如果是doc，则使用 LibreOffice 转换为 docx
    _, ext = os.path.splitext(file_path)
    filename = os.path.basename(file_path)
    ext = ext.lower()

    if ext == ".doc":
        tmpdir = tempfile.mkdtemp(prefix="doc_convert_")
        try:
            converted_docx_path, error = _convert_doc_to_docx(file_path, tmpdir)

            if converted_docx_path and os.path.exists(converted_docx_path):
                text = (
                    textract.process(converted_docx_path)
                    .decode("utf-8", errors="ignore")
                )
            else:
                print(f"LibreOffice DOCX转换失败，进入DOC文本兜底链路: {error}")
                text = _extract_doc_text_fallback(file_path, tmpdir)

            if not text or not text.strip():
                print(f"DOC文本提取结果为空: {file_path}")
                text = "文档解析失败，请检查文件格式是否正确。1"
        except Exception as e:
            print(f"处理DOC文件时出错: {e}")
            traceback.print_exc()
            try:
                text = _extract_doc_text_fallback(file_path, tmpdir)
            except Exception:
                traceback.print_exc()
                text = ""
            if not text or not text.strip():
                text = "文档解析失败，请检查文件格式是否正确。1"
        finally:
            if os.path.exists(tmpdir):
                shutil.rmtree(tmpdir, ignore_errors=True)
    else:
        # 直接使用 textract 处理 docx
        try:
            text = (
                textract.process(file_path)
                .decode("utf-8", errors="ignore")
            )
        except Exception as e:
            traceback.print_exc()
            print(f"Error processing DOCX file: {e}")
            text = "文档解析失败，请检查文件格式是否正确。2"
    return text


def extract_images_from_docx(docx_path, output_dir=None, file_remark=""):
    """
    从DOCX文件中提取图片
    
    Args:
        docx_path (str): DOCX文件路径
        output_dir (str): 图片保存目录
        
    Returns:
        dict: 图片ID到文件路径的映射
    """
    if output_dir is None:
        docx_md5 = md5(docx_path.encode()).hexdigest()
        output_dir = f"../../storage/docx_images/{docx_md5}"
    
    if not os.path.exists(output_dir):
        os.makedirs(output_dir, exist_ok=True)
    
    image_mapping = {}
    
    try:
        with zipfile.ZipFile(docx_path, 'r') as docx_zip:
            # 获取所有图片文件
            image_files = [f for f in docx_zip.namelist() if f.startswith('word/media/')]
            
            for image_file in image_files:
                image_name = os.path.basename(image_file)
                image_id = os.path.splitext(image_name)[0]
                image_ext = os.path.splitext(image_name)[1].lower()
                
                output_filename = f"docx_img_{image_id}{image_ext}"
                output_path = os.path.join(output_dir, output_filename)
                
                # 提取并保存图片
                with docx_zip.open(image_file) as img_file:
                    with open(output_path, 'wb') as output_file:
                        shutil.copyfileobj(img_file, output_file)  # 使用copyfileobj避免一次性读取大文件
                
                # 计算文件大小和MD5
                image_size = os.path.getsize(output_path)
                with open(output_path, 'rb') as f:
                    imageMd5 = md5(f.read()).hexdigest()
                
                # 生成对象键
                docx_md5 = md5(docx_path.encode()).hexdigest()
                objKey = f"/docx_images/{docx_md5}/{output_filename}"
                exist_file = database.get_file_by_obj_key(objKey)
                if exist_file is None:
                    try:
                        # 插入数据库记录
                        database.insert_file(
                            f"storage/docx_images/{docx_md5}/{output_filename}", 
                            objKey,
                            output_filename, 
                            image_ext[1:], 
                            image_size, 
                            imageMd5, 
                            None, 
                            False,
                            file_remark=file_remark
                        )
                    except:
                        pass
                # 保存映射关系，使用原始文件名作为key
                image_mapping[image_id] = {
                    'file_path': output_path,
                    'url': value.storageUrl + objKey,
                    'filename': output_filename
                }
                
                # print(f"成功提取图片: {output_filename}, ID: {image_id}")
        
        return image_mapping
        
    except Exception as e:
        print(f"提取DOCX图片时出错: {e}")
        return {}
    finally:
        # 强制垃圾回收
        gc.collect()
def perform_doc_text_extraction_with_images(file_path, extract_images=False, output_dir=None):
    """
    增强版的DOCX文本提取函数，支持图片提取和markdown链接替换
    """
    if not os.path.exists(file_path):
        raise FileNotFoundError(f"文件不存在: {file_path}")

    _, ext = os.path.splitext(file_path)
    filename = os.path.basename(file_path)
    ext = ext.lower()

    extracted_images = []

    if ext == ".doc":
        tmpdir = tempfile.mkdtemp(prefix="doc_convert_")
        try:
            converted_docx_path, error = _convert_doc_to_docx(file_path, tmpdir)

            if converted_docx_path and os.path.exists(converted_docx_path):
                if extract_images:
                    image_mapping = extract_images_from_docx(converted_docx_path, output_dir, "attachment for " + file_path)
                    text = process_docx_with_images(converted_docx_path, image_mapping)
                    extracted_images = list(image_mapping.values())
                else:
                    text = (
                        textract.process(converted_docx_path)
                        .decode("utf-8", errors="ignore")
                    )
            else:
                print(f"LibreOffice DOCX转换失败，进入DOC文本兜底链路: {error}")
                text = _extract_doc_text_fallback(file_path, tmpdir)

            if not text or not text.strip():
                print(f"DOC文本提取结果为空: {file_path}")
                text = "文档解析失败，请检查文件格式是否正确。3"
        except Exception as e:
            print(f"处理DOC文件时出错: {e}")
            traceback.print_exc()
            try:
                text = _extract_doc_text_fallback(file_path, tmpdir)
            except Exception:
                traceback.print_exc()
                text = ""
            if not text or not text.strip():
                text = "文档解析失败，请检查文件格式是否正确。3"
        finally:
            if os.path.exists(tmpdir):
                shutil.rmtree(tmpdir, ignore_errors=True)
    elif ext == ".docx":
        # DOCX文件处理
        try:
            if extract_images:
                image_mapping = extract_images_from_docx(file_path, output_dir,"attachment for " + file_path)
                text = process_docx_with_images(file_path, image_mapping)
                extracted_images = list(image_mapping.values())
            else:
                text = (
                    textract.process(file_path)
                    .decode("utf-8", errors="ignore")
                )
        except Exception as e:
            traceback.print_exc()
            print(f"Error processing DOCX file: {e}")
            text = "文档解析失败，请检查文件格式是否正确。4"
    else:
        raise ValueError(f"不支持的文件格式: {ext}")
    
    if extract_images:
        return text, extracted_images
    else:
        return text

def process_docx_with_images(docx_path, image_mapping):
    """
    处理DOCX文件，将图片位置替换为markdown格式的图片链接
    """
    doc = None
    
    try:
        doc = Document(docx_path)
        text_parts = []
        
        # 建立关系ID到图片文件的映射
        rel_mapping = {}
        try:
            with zipfile.ZipFile(docx_path, 'r') as docx_zip:
                if 'word/_rels/document.xml.rels' in docx_zip.namelist():
                    rels_content = docx_zip.read('word/_rels/document.xml.rels')
                    rels_root = ET.fromstring(rels_content)
                    
                    for rel in rels_root.findall('.//{http://schemas.openxmlformats.org/package/2006/relationships}Relationship'):
                        rel_id = rel.get('Id')
                        target = rel.get('Target')
                        if target and target.startswith('media/'):
                            image_filename = os.path.splitext(os.path.basename(target))[0]
                            rel_mapping[rel_id] = image_filename
                            # print(f"关系映射: {rel_id} -> {image_filename}")
        except Exception as e:
            print(f"解析关系文件时出错: {e}")
        
        # 处理段落
        for paragraph in doc.paragraphs:
            paragraph_text = paragraph.text
            has_image = False
            image_links = []
            
            # 检查段落中的所有run
            for run in paragraph.runs:
                # 检查run的XML中是否包含图片
                run_xml = run.element.xml
                
                # 查找图片元素
                for drawing in run.element.iter(qn('w:drawing')):
                    has_image = True
                    # 查找嵌入的图片引用
                    for embed in drawing.iter():
                        if embed.get(qn('r:embed')):
                            rel_id = embed.get(qn('r:embed'))
                            # print(f"找到图片引用: {rel_id}")
                            
                            # 根据关系映射查找对应的图片
                            if rel_id in rel_mapping:
                                image_id = rel_mapping[rel_id]
                                # print(f"映射到图片ID: {image_id}")
                                
                                if image_id in image_mapping:
                                    image_url = image_mapping[image_id]['url']
                                    image_filename = image_mapping[image_id]['filename']
                                    markdown_image = f"![{image_filename}]({image_url})"
                                    image_links.append(markdown_image)
                                    # print(f"添加图片链接: {markdown_image}")
                
                # 也检查旧式的图片格式
                for pict in run.element.iter(qn('w:pict')):
                    has_image = True
                    # 处理旧式图片格式的逻辑
                    
            # 如果段落包含图片，在文本中插入图片链接
            if has_image and image_links:
                # 如果段落有文字，在文字后面添加图片
                if paragraph_text.strip():
                    final_text = paragraph_text + " " + " ".join(image_links)
                else:
                    # 如果段落只有图片，直接使用图片链接
                    final_text = " ".join(image_links)
                text_parts.append(final_text)
            elif paragraph_text.strip():
                # 普通文本段落
                text_parts.append(paragraph_text)
        
        # 处理表格
        for table in doc.tables:
            table_rows = []
            for row in table.rows:
                row_cells = []
                for cell in row.cells:
                    cell_text_parts = []
                    for paragraph in cell.paragraphs:
                        paragraph_text = paragraph.text
                        cell_image_links = []
                        
                        for run in paragraph.runs:
                            for drawing in run.element.iter(qn('w:drawing')):
                                for embed in drawing.iter():
                                    if embed.get(qn('r:embed')):
                                        rel_id = embed.get(qn('r:embed'))
                                        if rel_id in rel_mapping:
                                            image_id = rel_mapping[rel_id]
                                            if image_id in image_mapping:
                                                image_url = image_mapping[image_id]['url']
                                                image_filename = image_mapping[image_id]['filename']
                                                markdown_image = f"![{image_filename}]({image_url})"
                                                cell_image_links.append(markdown_image)
                        
                        if paragraph_text.strip() or cell_image_links:
                            if paragraph_text.strip() and cell_image_links:
                                cell_text_parts.append(paragraph_text + " " + " ".join(cell_image_links))
                            elif cell_image_links:
                                cell_text_parts.append(" ".join(cell_image_links))
                            elif paragraph_text.strip():
                                cell_text_parts.append(paragraph_text)
                    
                    row_cells.append(" ".join(cell_text_parts))
                table_rows.append(" | ".join(row_cells))
            
            if table_rows:
                text_parts.append("| " + " |\n| ".join(table_rows) + " |")
        
        final_text = "\n".join(text_parts)
        # print(f"最终处理的文本长度: {len(final_text)}")
        return final_text
        
    except Exception as e:
        print(f"处理DOCX图片链接时出错: {e}")
        traceback.print_exc()
        # 回退到基本文本提取
        return textract.process(docx_path).decode("utf-8", errors="ignore")
    finally:
        # 清理Document对象
        if doc:
            del doc
        # 清理映射表
        if 'rel_mapping' in locals():
            del rel_mapping
        gc.collect()
def extract_image_id_from_element(element):
    """
    从XML元素中提取图片ID，改进匹配逻辑
    """
    try:
        # 查找图片引用的多种方式
        for desc in element.iter():
            # 检查 embed 属性（drawing 元素）
            if 'embed' in desc.attrib:
                rid = desc.attrib['embed']
                # 提取数字部分，例如 rId1 -> 1
                image_num = rid.replace('rId', '')
                return f"image{image_num}"
            
            # 检查其他可能的 ID 属性
            elif 'id' in desc.attrib:
                return f"image{desc.attrib['id']}"
                
        # 如果没有找到明确的引用，尝试其他方法
        return None
    except Exception as e:
        print(f"提取图片ID时出错: {e}")
        return None

def extract_text_from_pptx(pptx_path):
    """
    直接从PPTX文件中提取文本
    
    参数:
        pptx_path (str): PPTX文件路径
        
    返回:
        str: 提取的文本内容
    """
    try:
        prs = Presentation(pptx_path)
        text_content = []
        
        # 遍历所有幻灯片
        for i, slide in enumerate(prs.slides):
            slide_text = []
            slide_text.append(f"# 第 {i+1} 页")
            
            # 提取幻灯片标题
            title_shape = None
            if hasattr(slide.shapes, "title") and slide.shapes.title:
                title_shape = slide.shapes.title
                if hasattr(title_shape, "text") and title_shape.text:
                    slide_text.append(f"## {title_shape.text}")
                
            # 提取所有文本框的内容
            for shape in slide.shapes:
                # 检查是否是标题框，如果是则跳过（已经处理过了）
                if title_shape and shape == title_shape:
                    continue
                    
                # 提取文本框内容
                if hasattr(shape, "text") and shape.text and isinstance(shape.text, str):
                    text = shape.text.strip()
                    if text:
                        slide_text.append(text)
                    
            # 提取表格内容
            for shape in slide.shapes:
                if hasattr(shape, "has_table") and shape.has_table:
                    try:
                        table = shape.table
                        for row in table.rows:
                            for cell in row.cells:
                                if hasattr(cell, "text") and cell.text:
                                    slide_text.append(cell.text.strip())
                    except Exception as table_error:
                        print(f"处理表格时出错: {table_error}")
                        continue
            
            # 添加当前幻灯片的文本到内容列表
            if len(slide_text) > 1:  # 如果只有标题行，说明没有实际内容
                text_content.append("\n\n".join(slide_text))
        
        # 如果没有提取到内容，返回空字符串
        if not text_content:
            return ""
            
        return "\n\n" + "\n\n---\n\n".join(text_content)
    except Exception as e:
        print(f"提取PPTX文本时出错: {e}")
        print(f"错误类型: {type(e)}")
        traceback.print_exc()
        return None


def ppt_to_pptx(ppt_path):
    """
    将PPT文件转换为PPTX文件

    参数:
        ppt_path (str): PPT文件路径

    返回:
        str: 转换后的PPTX文件路径
    """
    if not os.path.exists(ppt_path):
        raise FileNotFoundError(f"文件不存在: {ppt_path}")

    # 创建临时目录存放转换后的PPTX
    tmp_dir = tempfile.mkdtemp()
    base_name = os.path.splitext(os.path.basename(ppt_path))[0]
    output_pptx = os.path.join(tmp_dir, f"{base_name}.pptx")

    try:
        # 使用进程池管理器转换PPT
        success, converted_path, error = convert_with_retry(
            libreoffice_pool,
            ppt_path,
            tmp_dir,
            "pptx",
            max_retries=3,
            base_delay=1.0
        )

        if not success or not converted_path or not os.path.exists(converted_path):
            raise Exception(f"PPT转换失败: {error}")

        return converted_path

    except Exception as e:
        print(f"PPT转换过程中发生错误: {e}")
        if os.path.exists(tmp_dir):
            shutil.rmtree(tmp_dir)
        raise


def pptx_to_pdf(pptx_path):
    """
    将PPTX文件转换为PDF文件

    参数:
        pptx_path (str): PPTX文件路径

    返回:
        str: 转换后的PDF文件路径
    """
    if not os.path.exists(pptx_path):
        raise FileNotFoundError(f"文件不存在: {pptx_path}")

    # 创建临时目录存放转换后的PDF
    tmp_dir = tempfile.mkdtemp()
    base_name = os.path.splitext(os.path.basename(pptx_path))[0]
    output_pdf = os.path.join(tmp_dir, f"{base_name}.pdf")

    try:
        # 使用进程池管理器转换PPTX到PDF
        success, converted_path, error = convert_with_retry(
            libreoffice_pool,
            pptx_path,
            tmp_dir,
            "pdf",
            max_retries=3,
            base_delay=1.0
        )

        if not success or not converted_path or not os.path.exists(converted_path):
            raise Exception(f"PPTX转换失败: {error}")

        return converted_path

    except Exception as e:
        print(f"PPTX转换过程中发生错误: {e}")
        if os.path.exists(tmp_dir):
            shutil.rmtree(tmp_dir)
        raise


def _cleanup_temp_file(file_path):
    if not file_path:
        return
    tmp_root = os.path.abspath(tempfile.gettempdir())
    parent_dir = os.path.abspath(os.path.dirname(file_path))
    try:
        if os.path.exists(file_path):
            os.remove(file_path)
    except Exception as e:
        print(f"删除临时文件失败 {file_path}: {e}")
    try:
        if parent_dir.startswith(tmp_root) and os.path.exists(parent_dir):
            shutil.rmtree(parent_dir, ignore_errors=True)
    except Exception as e:
        print(f"删除临时目录失败 {parent_dir}: {e}")


def extract_pdf_images(pdf_path, output_dir=None):
    """
    专门用于提取PDF中所有图片的函数
    
    Args:
        pdf_path (str): PDF文件路径
        output_dir (str): 图片保存目录，如果为None则在PDF同目录下创建images文件夹
        
    Returns:
        list: 提取的图片信息列表
    """
    if output_dir is None:
        output_dir = os.path.join(os.path.dirname(pdf_path), "extracted_images")
    
    # 优先走原生PDF图片提取，扫描件或失败时回退OCR。
    result = _parse_pdf_with_native_fallback_to_ocr(pdf_path, save_images=True, output_dir=output_dir)
    
    if result and len(result) == 3:
        markdown_result, pages, images = result
        # print(f"成功提取 {len(images)} 张图片到目录: {output_dir}")
        return images
    else:
        print("图片提取失败")
        return []


def recognize_file(file_path):
    """
    Direct-OCR entry point used by chat attachments.
    Returns extracted plain/markdown text and keeps transient OCR failures retryable.
    """
    if not os.path.exists(file_path):
        raise FileNotFoundError(f"文件不存在: {file_path}")

    _, ext = os.path.splitext(file_path)
    ext = ext.lower()

    try:
        if ext == ".pdf":
            result = _parse_pdf_with_native_fallback_to_ocr(file_path, save_images=True)
            if not result:
                raise TransientParseError(f"PDF OCR失败（可重试）: {file_path}")
            if isinstance(result, tuple):
                return _describe_markdown_images_for_direct_ocr(result[0] or "")
            return str(result or "")

        if ext in [".docx", ".doc"]:
            result = perform_doc_text_extraction_with_images(file_path, extract_images=True)
            if isinstance(result, tuple):
                return _describe_markdown_images_for_direct_ocr(result[0] or "")
            return str(result or "")

        if ext in [".xlsx", ".xls", ".ods"]:
            if ext == ".ods":
                return _extract_ods_text(file_path)
            _, text_contents = excel_to_html(file_path)
            return "\n\n".join(text_contents or [])

        if ext in [".csv", ".tsv"]:
            delimiter = "\t" if ext == ".tsv" else ","
            return _extract_delimited_text(file_path, delimiter)

        if ext in [".txt", ".md"]:
            return _read_text_file_with_encoding(file_path)

        if ext in [".html", ".htm", ".xhtml"]:
            return _extract_html_text(file_path)

        if ext == ".rtf":
            return _extract_rtf_text(file_path)

        if ext == ".odt":
            return _extract_odt_text(file_path)

        if ext == ".odp":
            return _extract_odp_text(file_path)

        if ext in IMAGE_PARSE_EXTENSIONS:
            paragraphs, _ = _extract_image_paragraphs(file_path)
            return "\n\n".join(item for group in paragraphs for item in group)

        if ext == ".pptx":
            pdf_path = None
            try:
                pdf_path = pptx_to_pdf(file_path)
                result = _parse_pdf_with_native_fallback_to_ocr(pdf_path, save_images=True)
                if not result:
                    raise TransientParseError(f"PPTX转PDF后OCR失败（可重试）: {file_path}")
                if isinstance(result, tuple):
                    return _describe_markdown_images_for_direct_ocr(result[0] or "")
                return str(result or "")
            finally:
                _cleanup_temp_file(pdf_path)

        if ext == ".ppt":
            pptx_path = None
            pdf_path = None
            try:
                pptx_path = ppt_to_pptx(file_path)
                pdf_path = pptx_to_pdf(pptx_path)
                result = _parse_pdf_with_native_fallback_to_ocr(pdf_path, save_images=True)
                if not result:
                    raise TransientParseError(f"PPT转PDF后OCR失败（可重试）: {file_path}")
                if isinstance(result, tuple):
                    return _describe_markdown_images_for_direct_ocr(result[0] or "")
                return str(result or "")
            finally:
                _cleanup_temp_file(pdf_path)
                _cleanup_temp_file(pptx_path)

        raise ValueError(f"不支持的文件格式: {ext}")
    except TransientParseError:
        raise
    except Exception as e:
        print(f"文件识别处理失败 {file_path}: {e}")
        traceback.print_exc()
        return f"文件解析失败，请检查文件格式是否正确。{e}"


def get_paragraph_router(file_path, extract_images=True, image_output_dir=None,user_id=None):
    """
    获取文件的段落路由
    :param file_path: 文件路径
    :param extract_images: 是否提取PDF中的图片
    :param image_output_dir: 图片输出目录
    :return: 段落列表、其他信息、图片信息（如果提取）
    """
    _, ext = os.path.splitext(file_path)
    ext = ext.lower()
    text = ""
    extra_info = {}
    originalText = []
    html_tables = []
    text_contents = []
    extracted_images = []
    
    # 添加内存管理
    temp_files_to_cleanup = []
    
    try:
        if _is_image_file(file_path):
            return _extract_image_paragraphs(file_path)
        if ext == ".pdf":
            try:
                # PDF 默认保留图片标记和图片文件，与 OCR 路径保持同等能力。
                result = _parse_pdf_with_native_fallback_to_ocr(
                    file_path,
                    save_images=True,
                    output_dir=image_output_dir,
                    user_id=user_id,
                )
            except SourceDocumentParseError as e:
                text = f"文档解析失败，请检查文件格式是否正确。{e}"
                result = None
            except Exception as e:
                if _is_ocr_dependency_error(e):
                    raise TransientParseError(f"PDF OCR依赖暂不可用（可重试）: {e}") from e
                raise
            if result and len(result) == 3:
                text, _, extracted_images = result
                # 清理临时结果
                del result
            elif result and len(result) == 2:
                text, _ = result
            elif not text:
                # OCR 失败属于瞬时错误（限流/网络），走失败-重试链路，
                # 不能落入"请检查文件格式"分支导致任务被当作源文档错误删除
                raise TransientParseError(f"PDF OCR失败（可重试）: {file_path}")
        elif ext in [".docx", ".doc"]:
            if extract_images:
                result = perform_doc_text_extraction_with_images(file_path, extract_images=True, output_dir=image_output_dir)
                if isinstance(result, tuple) and len(result) == 2:
                    text, extracted_images = result
                else:
                    text = result
                    extracted_images = []
            else:
                text = perform_doc_text_extraction(file_path)
        elif ext in [".xlsx", ".xls", ".ods"]:
            if ext == ".ods":
                text = _extract_ods_text(file_path)
                text_contents = [text]
                html_tables = [text]
            else:
                html_tables, text_contents = excel_to_html(file_path)
                # print(f"Excel文件 {file_path} 包含 {len(html_tables)} 个工作表")
                # print(f"文本内容: {text_contents}")
                # print(f"HTML表格: {html_tables}")
                text = "\n".join(html_tables)
        elif ext in [".csv", ".tsv"]:
            delimiter = "\t" if ext == ".tsv" else ","
            text = _extract_delimited_text(file_path, delimiter)
        elif ext in [".html", ".htm", ".xhtml"]:
            text = _extract_html_text(file_path)
        elif ext == ".rtf":
            text = _extract_rtf_text(file_path)
        elif ext == ".odt":
            text = _extract_odt_text(file_path)
        elif ext == ".odp":
            text = _extract_odp_text(file_path)
        elif ext == ".md":
            with open(file_path, "r", encoding="utf-8") as f:
                text = f.read()
        elif ext == ".txt":
            # 尝试多种编码方式读取txt文件
            text = ""
            encodings_to_try = ['utf-8', 'gbk', 'gb2312', 'gb18030', 'ascii', 'latin-1']

            # 初始化raw_data
            raw_data = b""
            # 首先尝试检测文件编码
            try:
                with open(file_path, 'rb') as f:
                    raw_data = f.read()
                    # 处理空文件
                    if len(raw_data) == 0:
                        text = ""
                        print(f"txt文件为空: {file_path}")
                    else:
                        detected = chardet.detect(raw_data)
                        if detected and detected['confidence'] > 0.7:
                            # 将检测到的编码放在列表最前面
                            if detected['encoding'] and detected['encoding'].lower() not in [e.lower() for e in encodings_to_try]:
                                encodings_to_try.insert(0, detected['encoding'])
                            else:
                                # 如果检测到的编码已在列表中，将其移到最前面
                                detected_encoding = detected['encoding']
                                for i, enc in enumerate(encodings_to_try):
                                    if enc.lower() == detected_encoding.lower():
                                        encodings_to_try.pop(i)
                                        encodings_to_try.insert(0, enc)
                                        break
            except Exception as e:
                print(f"检测文件编码时出错: {e}")

            # 只有在文件不为空时才尝试读取
            if len(raw_data) > 0:  # 使用raw_data而不是text来判断
                # 尝试各种编码
                for encoding in encodings_to_try:
                    try:
                        with open(file_path, "r", encoding=encoding) as f:
                            text = f.read()
                        print(f"成功使用编码 {encoding} 读取txt文件: {file_path}")
                        break
                    except UnicodeDecodeError:
                        continue
                    except Exception as e:
                        print(f"使用编码 {encoding} 读取文件时出错: {e}")
                        continue

                if not text:
                    # 如果所有编码都失败，抛出更明确的错误
                    raise ValueError(f"无法读取txt文件，尝试了多种编码: {', '.join(encodings_to_try)}。可能是文件已损坏或使用了不支持的编码。")
        elif ext == ".pptx":
            # 直接从PPTX提取文本
            text = extract_text_from_pptx(file_path)
            if text is None or True:
                # 如果直接提取失败，回退到PDF转换方案
                try:
                    pdf_path = pptx_to_pdf(file_path)
                    ocr_result = _parse_pdf_with_native_fallback_to_ocr(pdf_path, save_images=True)
                    # 清理临时文件
                    tmp_dir = os.path.dirname(pdf_path)
                    if os.path.exists(pdf_path):
                        os.remove(pdf_path)
                    if os.path.exists(tmp_dir):
                        shutil.rmtree(tmp_dir)
                    if not ocr_result:
                        raise TransientParseError(f"PPTX转PDF后OCR失败（可重试）: {file_path}")
                    text = _pdf_parse_result_text(ocr_result)
                except TransientParseError:
                    raise
                except Exception as e:
                    print(f"处理PPTX文件时出错: {e}")
                    text = "PowerPoint文件解析失败，请检查文件格式是否正确。"
        elif ext == ".ppt":
            # PPT文件需要先转换为PPTX
            try:
                pptx_path = ppt_to_pptx(file_path)
                text = extract_text_from_pptx(pptx_path)
                # 清理临时文件
                tmp_dir = os.path.dirname(pptx_path)
                if os.path.exists(pptx_path):
                    os.remove(pptx_path)
                if os.path.exists(tmp_dir):
                    shutil.rmtree(tmp_dir)
                
                if text is None:
                    # 如果直接提取失败，回退到PDF转换方案
                    pdf_path = pptx_to_pdf(file_path)
                    ocr_result = _parse_pdf_with_native_fallback_to_ocr(pdf_path, save_images=True)
                    # 清理临时文件
                    tmp_dir = os.path.dirname(pdf_path)
                    if os.path.exists(pdf_path):
                        os.remove(pdf_path)
                    if os.path.exists(tmp_dir):
                        shutil.rmtree(tmp_dir)
                    if not ocr_result:
                        raise TransientParseError(f"PPT转PDF后OCR失败（可重试）: {file_path}")
                    text = _pdf_parse_result_text(ocr_result)
            except TransientParseError:
                raise
            except Exception as e:
                print(f"处理PPT文件时出错: {e}")
                text = "PowerPoint文件解析失败，请检查文件格式是否正确。"

        else:
            raise ValueError(f"不支持的文件格式: {ext}")

    except TransientParseError:
        # 瞬时失败直接上抛，由 doc_processor 标记 failed 并走重试
        raise
    except Exception as e:
        traceback.print_exc()
        print(f"处理文件 {file_path} 时出错: {e}")
        text = "文件解析失败，请检查文件格式是否正确。" + str(e)
    
        
    # 处理文本时的内存优化
    if ext not in [".xlsx", ".xls", ".ods"]:
        # 分批处理大文本
        if len(text) > 10000:  # 对于超过10000字符的文本
            # 先进行粗分割
            rough_chunks = [text[i:i+10000] for i in range(0, len(text), 8000)]  # 有重叠
            
            paragraphs = []
            for chunk in rough_chunks:
                chunk_paragraphs = split_paragraph(
                    chunk, max_chars=1024, min_split_chars=50, 
                    split_markers=["\n\n"], absolute_max_chars=4000
                )
                paragraphs.extend(chunk_paragraphs)
                
                # 清理临时数据
                del chunk_paragraphs
                gc.collect()
            
            paragraphs = [paragraphs]
            del rough_chunks
        else:
            paragraphs = [
                split_paragraph(
                    text, max_chars=512, min_split_chars=50, 
                    split_markers=["\n\n"], absolute_max_chars=2048
                )
            ]
        
        # 检查并处理超过3000字的段落
        final_paragraphs = []
        final_original_text = []
        
        for paragraph in paragraphs[0]:
            if len(paragraph) > 4000:
                # print(f"检测到超长段落({len(paragraph)}字符)，进行3000字强制分段")
                # 以3000字为单位强制分段
                chunk_size = 4000
                paragraph_chunks = []
                for i in range(0, len(paragraph), chunk_size):
                    chunk = paragraph[i:i + chunk_size]
                    if chunk.strip():  # 确保非空
                        paragraph_chunks.append(chunk.strip())
            
                final_paragraphs.extend(paragraph_chunks)
                # 同步修改originalText，每个分段都对应原始段落
                final_original_text.extend([paragraph] * len(paragraph_chunks))
            else:
                final_paragraphs.append(paragraph)
                final_original_text.append(paragraph)
        
        paragraphs = [final_paragraphs]
        originalText = [final_original_text]
        print(f"最终段落数量: {len(paragraphs[0])}")
    else:
        paragraphs = []
        originalText = []

        if not text_contents:
            # 无法提取表格文本时直接返回空结果，避免 NoneType 异常
            return paragraphs, originalText

        for i, text_content in enumerate(text_contents):
            if len(text_content) > 0:
                temp_paragraphs = split_text_by_length(
                    text_content, max_chars=1024, overlap_chars=50
               
                )
                # print(f"temp_paragraphs:{temp_paragraphs}")
                
                # 检查并处理超过3000字的段落
                final_temp_paragraphs = []
                final_temp_original = []
                
                for paragraph in temp_paragraphs:
                    if len(paragraph) > 3000:
                        print(f"Excel段落检测到超长段落({len(paragraph)}字符)，进行3000字强制分段")
                        # 以3000字为单位强制分段
                        chunk_size = 3000

                        for j in range(0, len(paragraph), chunk_size):
                            chunk = paragraph[j:j + chunk_size]
                            if chunk.strip():  # 确保非空
                                final_temp_paragraphs.append(chunk.strip())
                                # Excel 场景下 originalText 需与分段一一对应，避免整表 HTML 被重复写入向量库
                                final_temp_original.append(chunk.strip())
                    else:
                        final_temp_paragraphs.append(paragraph)
                        final_temp_original.append(paragraph)
                
                paragraphs.append(final_temp_paragraphs)
                originalText.append(final_temp_original)

    # 如果提取了图片，返回图片信息
    return paragraphs, originalText