cert_parser.py - main - Paff/declarant

cert_parser.py
3 123 строки131 KB
1pash1985-gif
feat: populate certificate invoice number field from master invoice mapping
10 авг 2026, 09:14
10 авг 2026, 09:14d8339de
# -*- coding: utf-8 -*-
"""
Парсер PDF-сертификатов экспорта (Sertif Ec/Co/Ke.pdf).

Поддерживаемые форматы:
  • **Ecuador** — «FRESH SOLUTIONS CARGO / MASTER INVOICE #NNNNNNNN» + построчные позиции
    формата `<flower> <farm> <marking> <inv#> <kg> <boxes> <HS_code> <stems> <price> <total>`.
    Один PDF содержит несколько master invoice блоков (по 3 стр. каждый).
  • **Colombia** — EUR1-формат DIAN, шапка + агрегированная строка на весь груз.
  • **Kenya** — обычно EUR1/COMESA, аналогично колумбийскому — шапка + агрегат.

Возвращает:
  CertificateData(
      country: str,
      cert_number: str,             # EUR1-... или DIAN-...
      master_invoice: str,           # 11322702 (Ecuador master invoice)
      awb: str,                      # 369-9957 0623
      exporter: str,                 # FRESH SOLUTIONS CARGO CIA LTDA
      consignee: str,                # FLORUNNER B.V. …
      items: list[CertItem],         # построчные позиции (Ecuador)
      total_boxes: float,            # сумма из items или из агрегированной строки
      total_stems: int,              # то же
  )

Использование:
    from cert_parser import parse_certificate
    cd = parse_certificate('Sertif Ec 209.pdf')
    print(cd.exporter, cd.total_boxes, len(cd.items))
"""
from __future__ import annotations
import logging
import os
import re
from dataclasses import dataclass, field
from typing import Optional

logger = logging.getLogger(__name__)

try:
    import pdfplumber
except ImportError:
    pdfplumber = None

try:
    from ocr_eur1 import (
        ocr_eur1_numbers, ocr_dian_numbers, is_ocr_available, _get_paddle,
    )
except ImportError:                          # ocr_eur1 отсутствует — работаем без OCR
    ocr_eur1_numbers = None
    ocr_dian_numbers = None
    is_ocr_available = lambda: False
    _get_paddle = None

# По умолчанию используем Tesseract вместо PaddleOCR — он намного быстрее
# на сервере и не уходит в многоминутный застой. Paddle можно вернуть,
# задав переменную окружения OCR_ENGINE=paddle.
if os.environ.get('OCR_ENGINE') is None:
    os.environ['OCR_ENGINE'] = 'tesseract'

# --- Fast full-page Tesseract fallback (Kenya scan pages) ---
try:
    import pytesseract
    from PIL import Image
except ImportError:
    pytesseract = None
    Image = None

try:
    import fitz  # PyMuPDF
except ImportError:
    fitz = None

# --- Yandex Vision OCR fallback ---
try:
    from web import yandex_vision
except Exception:
    yandex_vision = None


# === Модели ===

@dataclass
class CertItem:
    """Одна позиция сертификата (Ecuador — из построчной таблицы)."""
    flower: str = ""       # ROSA, ALSTROEMERIA, GYPSOPHILA, MATHIOLA, CLAVEL, ...
    farm: str = ""         # FLORICOLA LATINAFARMS CIA LTDA / ростовка (Kenya)
    marking: str = ""      # ALEX KAZAN / B-BOSS 2 FR / IZUM102 / TARV116
    invoice_nr: str = ""   # 5736 (sub-invoice — от фермы)
    cert_number: str = ""  # номер сертификата конкретной позиции (Kenya/Ecuador)
    weight_kg: float = 0.0
    boxes: float = 0.0
    hs_code: str = ""      # 0603.11.0050
    stems: int = 0
    price_unit: float = 0.0
    total_price: float = 0.0
    master_invoice: str = "" # 11322702 (мастер-инвойс блока, к которому относится позиция)
    is_face_page: bool = False  # True если позиция извлечена с лицевой страницы EUR.1


@dataclass
class CertificateData:
    country: str = ""             # Ecuador / Colombia / Kenya
    cert_number: str = ""         # EUR1-… / DIAN-…  (первый/основной)
    eur1_numbers: list[str] = field(default_factory=list)  # все EUR1-номера (OCR + text)
    dian_numbers: list[str] = field(default_factory=list)  # все DIAN-номера (OCR + text)
    eur1_by_master: dict[str, str] = field(default_factory=dict)  # {master_inv: eur1/dian} — карта пар
    eur1_by_cert: dict[str, str] = field(default_factory=dict)    # {cert_number: eur1_nr} — Kenya позиционная связка
    master_invoices: list[str] = field(default_factory=list)  # для Ecuador — несколько
    awb: str = ""
    all_awbs: list[str] = field(default_factory=list)   # все AWB-ссылки из текста (для AWB→country map)
    exporter: str = ""
    consignee: str = ""
    date: str = ""
    items: list[CertItem] = field(default_factory=list)
    total_boxes: float = 0.0
    total_stems: int = 0
    raw_pages: int = 0
    source: str = ""
    # Проверка согласованности AWB: список несовпадений {cert/inv: {eur1_awb, invoice_awb}}
    awb_mismatches: list[dict] = field(default_factory=list)
    # Иерархическая структура: AWB → сертификаты → позиции
    awb_groups: list[dict] = field(default_factory=list)
    # Плоский список сертификатов: одна строка = один сертификат
    certificates: list[dict] = field(default_factory=list)


@dataclass
class CertPageMeta:
    idx: int = 0                      # 0-based page index
    text: str = ''
    awb: str = ''                     # normalized AWB
    awb_candidates: list[str] = field(default_factory=list)
    eur1_numbers: list[str] = field(default_factory=list)
    dian_numbers: list[str] = field(default_factory=list)
    phyto_cert_number: str = ''       # Kenya Milele bare number (526829)
    form_a_ref_number: str = ''       # Kenya ULTRA FLO Form A Reference No. (533392)
    invoice_numbers: list[str] = field(default_factory=list)
    proforma_numbers: list[str] = field(default_factory=list)
    etham_invoice: str = ''           # EFE-980
    has_etham_table: bool = False
    has_milele_table: bool = False
    # Ecuador tables extracted by PyMuPDF (cleaner farm/client split)
    ecu_table_rows: list[list[str]] = field(default_factory=list)


# === Классификация страны по имени файла ===

_COUNTRY_BY_FNAME = [
    (re.compile(r'\bEc\b|\bECU\b', re.IGNORECASE),   'Ecuador'),
    (re.compile(r'\bCo\b|\bCOL\b', re.IGNORECASE),   'Colombia'),
    (re.compile(r'\bKe\b|\bKEN\b', re.IGNORECASE),   'Kenya'),
]

def _detect_country(fname: str) -> str:
    for pat, c in _COUNTRY_BY_FNAME:
        if pat.search(fname):
            return c
    return ""


# Фолбэк: страна по содержимому PDF — когда имя файла не подсказывает
# (напр. веб-загрузка переименовывает в certificate_444_1.pdf).
_COUNTRY_BY_TEXT = [
    ('Ecuador',  re.compile(r'\bECUADOR\b|\bQUITO\b|\bTABABELA\b', re.IGNORECASE)),
    ('Colombia', re.compile(r'\bCOLOMBIA\b|\bBOGOTA\b|\bDIAN\b', re.IGNORECASE)),
    ('Kenya',    re.compile(r'\bKENYA\b|\bNAIROBI\b', re.IGNORECASE)),
]

def _detect_country_from_text(text: str) -> str:
    """Страна с максимумом упоминаний в тексте (пусто — если нет ни одного)."""
    best, best_n = '', 0
    for country, pat in _COUNTRY_BY_TEXT:
        n = len(pat.findall(text))
        if n > best_n:
            best, best_n = country, n
    return best


# === Регекспы для Ecuador-строк ===

# ROSA FLORICOLA LATINAFARMS CIA LTDA ALEX KAZAN 5736 0.500 1 0603.11.0050 450 0.289 130.00
# Стратегия: обратный разбор — конец строки известнее (числа), потом маркировка, ферма, цветок.
_ECU_LINE = re.compile(
    r'^(?P<flower>[A-Z][A-Z\-\s]+?)\s+'                         # flower
    r'(?P<middle>.+?)\s+'                                        # farm + marking (жадно, разберём отдельно)
    r'(?P<inv>\d{3,7})\s+'                                       # invoice#
    r'(?P<kg>\d+(?:\.\d+)?)\s+'                                  # weight_kg
    r'(?P<boxes>\d+(?:\.\d+)?)\s+'                               # boxes
    r'(?P<hs>\d{4}\.\d{2}(?:\.\d{4}|\.\d{2}\.\d{2}))\s+'         # HS-code (0603.11.0050 or 0603.11.00.00)
    r'(?P<stems>\d+)\s+'                                         # stems
    r'(?P<price>\d+(?:\.\d+)?)\s+'                               # unit price
    r'(?P<total>[\d,]+(?:\.\d+)?)\s*$'                           # total
)

# Известные типы цветов — для точности определения границы `flower` / `farm`
_FLOWERS = (
    'ROSA', 'ROSAS', 'ROSES', 'SPRAY ROSES', 'CLAVEL', 'CLAVELINA', 'ALSTROEMERIA',
    'GYPSOPHILA', 'MATHIOLA', 'CHRYSANTHEMUM', 'DIANTHUS',
    'HYPERICUM', 'CALLA', 'LILIUM', 'LILY', 'ANTHURIUM',
    'ALHELI', 'ALELI', 'STATICE', 'LIMONIUM', 'PROTEA',
    'MOLUCELLA', 'BUPLEURUM', 'ASTER', 'AMMI', 'CRASPEDIA',
    'DELPHINIUM', 'EUCALYPTUS', 'GERBERA', 'GYPSO',
    'HYDRANGEA', 'MUM', 'PEONY', 'PAEONIA', 'PONI', 'PONY',
    'RANUNCULUS', 'SOLIDAGO', 'STOCK', 'TULIP', 'VERONICA',
    'HELICONIA', 'ERINGIUM', 'ORNITOGALO',
)

# Явные синонимы названий цветов (варианты OCR/написания → каноническая форма)
_FLOWER_ALIASES = {
    'ROSES': 'ROSA',
    'CLAVELES': 'CLAVEL',
    'ALSTROEMERIAS': 'ALSTROEMERIA',
    'GYPSOPHILAS': 'GYPSOPHILA',
    'MATHIOLAS': 'MATHIOLA',
    'CHRYSANTHEMUMS': 'CHRYSANTHEMUM',
    'DIANTHUSES': 'DIANTHUS',
    'LILIES': 'LILIUM',
    'LILYS': 'LILIUM',
    'ANTHURIUMS': 'ANTHURIUM',
    'STATICES': 'STATICE',
    'LIMONIUMS': 'LIMONIUM',
    'PROTEAS': 'PROTEA',
    'ASTERS': 'ASTER',
    'GERBERAS': 'GERBERA',
    'HYDRANGEAS': 'HYDRANGEA',
    'PEONIES': 'PEONY',
    'PAEONIAS': 'PAEONIA',
    'TULIPS': 'TULIP',
    'VERONICAS': 'VERONICA',
}


def _normalize_flower_name(name: str) -> str:
    """Привести название цветка к канонической форме (единственное число).

    Убирает распространённые множественные окончания и OCR-варианты:
    ROSAS → ROSA, ROSES → ROSA, ALSTROEMERIAS → ALSTROEMERIA и т.д.
    """
    if not name:
        return name
    upper = name.strip().upper()
    # Явные синонимы
    if upper in _FLOWER_ALIASES:
        return _FLOWER_ALIASES[upper]
    # Множественное число: если отбрасывание 'S'/'ES' даёт известный цветок — берём единственное число
    if upper.endswith('ES') and len(upper) > 2:
        singular = upper[:-2]
        if singular in _FLOWERS:
            return singular
    if upper.endswith('S') and len(upper) > 1:
        singular = upper[:-1]
        if singular in _FLOWERS:
            return singular
    return upper


# Известные маркировки (расширяется динамически; можно тоже собрать из справочника)
_MARKINGS_HINTS = re.compile(
    r'\b(ALEX\s*KAZAN|IZUM\d*|TARV\d+|KAZSTORE|RANGER|B-\w+|'
    r'BUTA|SUN\s*FLORA|VALIEV|BOSS|NEFT|NERO|OKTY|RONC|ELA|'
    r'HARDIN|DONN\d+|MARGO|ECU\s*SOL|BOOM\d*|PRADO|TJ\s*PRADO|'
    r'HASAN\w+|TOPALO|FOREL|SAMARA|POKEDOVA|MAX\w*|KIRO|BORT|'
    r'SAMPLE|SKLADULIANOVSK|MZURRIE|IMPEX)\b', re.IGNORECASE
)


def _split_farm_marking(middle: str) -> tuple[str, str]:
    """
    Разделить `<FARM> <MARKING>` в куске между flower и invoice#.
    Возвращает (farm, marking). Использует эвристику: последний известный marking
    в строке — маркировка; всё до него — ферма.
    """
    if not middle:
        return "", ""
    matches = list(_MARKINGS_HINTS.finditer(middle))
    if matches:
        m = matches[-1]
        farm = middle[:m.start()].strip()
        marking = middle[m.start():].strip()
        # Уберём случайные хвостовые части типа " 2 FR" — оставим ровно матч + optional 2-3 короткие токена
        # Пока просто вернём как есть.
        return farm, marking
    # Fallback: последнее слово (или 2 слова) — маркировка
    parts = middle.rsplit(maxsplit=2)
    if len(parts) >= 2:
        return ' '.join(parts[:-1]), parts[-1]
    return middle, ""


# === EBF CARGO / PNT master-invoice variants ===

# EBF CARGO inline row head (trailing farm/RUC/address/invoice/date is parsed separately):
# 0.50 1 23.00 0603.11.00.00 ROSAS 200 $0.25 $50.00 EBF100349469 ...
# 0.25 1 13.00 0603.11.00.00 SPRAY ROSES 100 $0.25 $25.00 EBF100350082 ...
_EBF_ROW_HEAD = re.compile(
    r'^(?P<boxes>\d+(?:\.\d+)?)\s+'
    r'(?P<pcs>\d+)\s+'
    r'(?P<weight>\d+(?:\.\d+)?)\s+'
    r'(?P<hs>\d{4}\.\d{2}(?:\.\d{4}|\.\d{2}\.\d{2}))\s+'
    r'(?P<flower>.+?)\s+'
    r'(?P<stems>\d+)\s+'
    r'\$(?P<price>\d+(?:\.\d+)?)\s+'
    r'\$(?P<total>[\d,]+(?:\.\d+)?)\s+'
    r'(?P<hawb>EBF\d+)'
)

# PNT inline row:
# ROSA ROSA SP. 0603.11.00.00 1 450 US$126.75
_PNT_ROW = re.compile(
    r'^(?P<concept>[A-Z][A-Za-z]+)\s+'
    r'(?P<scientific>[A-Z]+(?:\s+SP\.?)?)\s+'
    r'(?P<hs>\d{4}\.\d{2}(?:\.\d{4}|\.\d{2}\.\d{2}))\s+'
    r'(?P<boxes>\d+)\s+'
    r'(?P<stems>\d+)\s+'
    r'US\$(?P<total>[\d,]+(?:\.\d+)?)'
)


def _extract_ebf_farm(prev_line: str) -> str:
    """Extract farm name from the EBF CARGO preceding line."""
    if not prev_line:
        return ''
    # Skip if previous line is another data row or table header
    if re.search(r'EBF\d+|^BXS|^US DOLLARS|^PRICE|^MASTER REPORT|Total:|TOTAL', prev_line, re.IGNORECASE):
        return ''
    # 'FLORISOL CIA. LTDA PICHINCHA / QUITO / ...' -> farm is before the slash
    parts = prev_line.split(' / ')
    farm = parts[0].strip() if parts else prev_line
    # Remove trailing logistic keywords if any
    farm = re.sub(r'\s+(?:GROSS|CHARGE|RUC#|PHONE|AWB|HAWB|MARCAS).*', '', farm, flags=re.IGNORECASE).strip()
    return farm


def _extract_pnt_farm(recent_lines: list[str]) -> str:
    """Extract farm name from a recent PNT block line containing HAWB."""
    for line in reversed(recent_lines):
        if 'HAWB' in line.upper():
            farm = line.split('HAWB')[0].strip()
            # Strip leading counter digits if line starts with a number
            farm = re.sub(r'^\d+\s+', '', farm).strip()
            return farm
    return ''


def _parse_ecu_ebf_inline_line(line: str, prev_line: str, current_master: str = '') -> Optional[CertItem]:
    """Parse an EBF CARGO inline space-separated row."""
    m = _EBF_ROW_HEAD.match(line)
    if not m:
        return None
    flower_raw = m.group('flower').strip()
    flower_parts = flower_raw.split()
    flower_token = flower_parts[-1].upper() if flower_parts else ''
    if flower_token not in _FLOWERS:
        return None
    hawb = m.group('hawb')
    invoice_nr = hawb[3:]  # EBF... -> numeric invoice fallback
    tail = line[m.end():].strip()
    farm = ''
    # Try farm from row tail (between HAWB and RUC)
    ruc_match = re.search(r'\b(\d{10,13})\b', tail)
    if ruc_match and ruc_match.start() > 0:
        farm = tail[:ruc_match.start()].strip()
    if not farm:
        # Fallback: leading alphabetic prefix before the first digit
        fm = re.match(r'^([A-Z][A-Z\s\.\,&]*?)\s*\d', tail)
        if fm:
            farm = fm.group(1).strip()
    if not farm:
        farm = _extract_ebf_farm(prev_line)
    # Look for a cleaner explicit invoice between RUC and date
    date_match = re.search(r'\b(\d{4}-\d{2}-\d{2})\b', tail)
    if ruc_match and date_match:
        middle = tail[ruc_match.end():date_match.start()]
        inv_match = re.search(r'\b(\d{6,10})\b', middle)
        if inv_match:
            invoice_nr = inv_match.group(1)
    try:
        return CertItem(
            flower=flower_token,
            farm=farm,
            marking='',
            invoice_nr=invoice_nr,
            weight_kg=float(m.group('weight')),
            boxes=float(m.group('boxes')),
            hs_code=m.group('hs'),
            stems=int(m.group('stems')),
            price_unit=float(m.group('price')),
            total_price=float(m.group('total').replace(',', '')),
            master_invoice=current_master,
        )
    except (ValueError, TypeError):
        return None


def _parse_ecu_pnt_inline_line(line: str, recent_lines: list[str], current_master: str = '') -> Optional[CertItem]:
    """Parse a PNT inline space-separated row."""
    if line.startswith('Total:') or line.upper().startswith('TOTAL'):
        return None
    m = _PNT_ROW.match(line)
    if not m:
        return None
    flower_token = m.group('concept').upper()
    if flower_token not in _FLOWERS:
        return None
    farm = _extract_pnt_farm(recent_lines)
    try:
        return CertItem(
            flower=flower_token,
            farm=farm,
            marking='',
            invoice_nr='',
            weight_kg=0.0,
            boxes=float(m.group('boxes')),
            hs_code=m.group('hs'),
            stems=int(m.group('stems')),
            price_unit=0.0,
            total_price=float(m.group('total').replace(',', '')),
            master_invoice=current_master,
        )
    except (ValueError, TypeError):
        return None


# === Ecuador pipe-delimited [ТАБЛИЦА] rows ===
# Concept | Farm | Client | Invoice | F. BXS | Pieces | Tariff # | Stems | P. Unit | Total Price
_ECU_PIPE_HEADER = re.compile(
    r'^\s*Concept\s*\|.*Farm\s*\|.*Client\s*\|.*Invoice',
    re.IGNORECASE,
)


def _parse_ecu_standard_row(row: list[str], current_master: str = '') -> Optional[CertItem]:
    """Parse a standard Ecuador table row (Concept/Farm/Client/Invoice...)."""
    if len(row) < 10:
        return None
    flower_cell = str(row[0] or '').strip()
    flower_token = flower_cell.split()[0].upper() if flower_cell else ''
    if flower_token not in _FLOWERS:
        return None
    invoice = str(row[3] or '').strip()
    if not invoice or not re.fullmatch(r'\d+', invoice.replace(' ', '')):
        return None
    try:
        weight_kg = float(str(row[4] or '').replace(',', ''))
        boxes = float(str(row[5] or '').replace(',', ''))
        hs_code = str(row[6] or '').replace(' ', '')
        stems = int(str(row[7] or '').replace(',', ''))
        price_unit = float(str(row[8] or '').replace(',', ''))
        total_price = float(str(row[9] or '').replace(',', ''))
    except (ValueError, TypeError):
        return None
    if not re.match(r'\d{4}\.\d{2}(?:\.\d{4}|\.\d{2}\.\d{2})$', hs_code):
        return None
    return CertItem(
        flower=flower_token,
        farm=str(row[1] or '').strip(),
        marking=str(row[2] or '').strip().upper(),
        invoice_nr=invoice,
        weight_kg=weight_kg,
        boxes=boxes,
        hs_code=hs_code,
        stems=stems,
        price_unit=price_unit,
        total_price=total_price,
        master_invoice=current_master,
    )


def _parse_ecu_air_canada_row(row: list[str], current_master: str = '') -> Optional[CertItem]:
    """Parse an Air Canada Ecuador commercial master-invoice row.

    Columns: BXS, PCS, WEIGHT, HTS, FLOWER, STEMS, PRICE UNITARY, PRICE TOTAL,
    HAWB, FARMS.
    """
    if len(row) < 10:
        return None
    flower_cell = str(row[4] or '').strip()
    flower_token = flower_cell.split()[0].upper() if flower_cell else ''
    if flower_token not in _FLOWERS:
        return None
    invoice = str(row[8] or '').strip()
    if not invoice or not re.fullmatch(r'\d+', invoice.replace(' ', '')):
        return None
    try:
        # Air Canada / EBF CARGO: BXS = full boxes, PCS = total pieces/boxes
        # on the certificate.  Use PCS as the authoritative box count.
        boxes = float(str(row[1] or '').replace(',', ''))
        weight_kg = float(str(row[2] or '').replace(',', ''))
        stems = int(str(row[5] or '').replace(',', ''))
        price_unit = float(str(row[6] or '').replace(',', ''))
        total_price = float(str(row[7] or '').replace(',', ''))
    except (ValueError, TypeError):
        return None
    return CertItem(
        flower=flower_token,
        farm=str(row[9] or '').strip(),
        marking='',
        invoice_nr=invoice,
        weight_kg=weight_kg,
        boxes=boxes,
        hs_code='',
        stems=stems,
        price_unit=price_unit,
        total_price=total_price,
        master_invoice=current_master,
    )


def _parse_ecu_table_row(row: list[str], current_master: str = '') -> Optional[CertItem]:
    """Parse a Ecuador table row from a list of cell strings (PyMuPDF table extraction)."""
    return _parse_ecu_standard_row(row, current_master)


def _parse_ecu_pipe_line(line: str, current_master: str = '') -> Optional[CertItem]:
    """Parse a pipe-delimited Ecuador table row (the cleaner [ТАБЛИЦА] view)."""
    if '|' not in line:
        return None
    parts = [p.strip() for p in line.split('|')]
    return _parse_ecu_table_row(parts, current_master)


def _parse_ecu_inline_line(line: str, current_master: str = '') -> Optional[CertItem]:
    """Parse the original space-separated Ecuador item row."""
    m = _ECU_LINE.match(line)
    if not m:
        return None
    flower_token = m.group('flower').strip()
    flower_first = flower_token.split()[0].upper()
    if flower_first not in _FLOWERS:
        return None
    farm, marking = _split_farm_marking(m.group('middle').strip())
    try:
        return CertItem(
            flower=flower_first,
            farm=farm.strip(),
            marking=marking.strip().upper(),
            invoice_nr=m.group('inv'),
            weight_kg=float(m.group('kg')),
            boxes=float(m.group('boxes')),
            hs_code=m.group('hs'),
            stems=int(m.group('stems')),
            price_unit=float(m.group('price')),
            total_price=float(m.group('total').replace(',', '')),
            master_invoice=current_master,
        )
    except (ValueError, TypeError):
        return None


def _extract_ecu_master_invoices(text: str) -> list[str]:
    """Найти все Ecuador master-invoice номера в тексте.

    Обрабатывает случаи:
      • MASTER INVOICE #11534802  (номер на той же строке)
      • 11534802                  (номер на строке выше)
        FRESH ... MASTER INVOICE #
      • Air Canada: Invoice No. EC 421820
    """
    masters: list[str] = []
    # Air Canada commercial master-invoice (same line)
    for m in _ECU_AIR_CANADA_INV.finditer(text):
        nr = m.group(1)
        if nr and nr not in masters:
            masters.append(nr)
    # Air Canada multi-line: "Invoice No." then "... EC 421820"
    lines = [l.strip() for l in text.split('\n')]
    for i, line in enumerate(lines):
        if re.search(r'Invoice\s*No\.?', line, re.IGNORECASE):
            # search current and next 2 lines for EC + number
            window = ' '.join(lines[i:i + 3])
            m2 = re.search(r'\bEC\s+(\d{5,10})\b', window, re.IGNORECASE)
            if m2:
                nr = m2.group(1)
                if nr and nr not in masters:
                    masters.append(nr)
    i = 0
    while i < len(lines):
        line = lines[i]
        if not line:
            i += 1
            continue
        # Случай 1: номер на той же строке с меткой
        mi = _ECU_MASTER_INV.search(line)
        if mi:
            nr = mi.group(1)
            if nr not in masters:
                masters.append(nr)
            i += 1
            continue
        # Случай 2: номер на предшествующей строке, метка — на следующей
        if i + 1 < len(lines) and re.fullmatch(r'[A-Z0-9]{6,12}', line):
            next_line = lines[i + 1]
            if 'MASTER INVOICE' in next_line.upper() and '#' in next_line:
                if line not in masters:
                    masters.append(line)
                i += 2
                continue
        i += 1
    return masters


def _parse_ecu_items(text: str) -> list[CertItem]:
    """Parse Ecuador item rows from a text block.

    Handles inline space-separated rows and pipe-delimited [ТАБЛИЦА] rows,
    plus EBF CARGO and PNT master-invoice variants.
    """
    pipe_items: list[CertItem] = []
    inline_items: list[CertItem] = []
    ebf_items: list[CertItem] = []
    pnt_items: list[CertItem] = []
    current_master = ''
    lines = [l.strip() for l in text.split('\n')]
    recent_lines: list[str] = []
    i = 0
    while i < len(lines):
        line = lines[i]
        if not line:
            i += 1
            continue
        # Случай 1: номер на той же строке с меткой
        mi = _ECU_MASTER_INV.search(line)
        if mi:
            current_master = mi.group(1)
            recent_lines.clear()
            i += 1
            continue
        # Случай 2: номер на предшествующей строке, метка — на следующей
        if i + 1 < len(lines) and re.fullmatch(r'[A-Z0-9]{6,12}', line):
            next_line = lines[i + 1]
            if 'MASTER INVOICE' in next_line.upper() and '#' in next_line:
                current_master = line
                recent_lines.clear()
                i += 2
                continue
        # Случай 3: Air Canada / EBF CARGO 'Invoice No.' then 'EC NNNNNNNN'
        if re.search(r'Invoice\s*No\.?', line, re.IGNORECASE):
            window = ' '.join(lines[i:i + 3])
            m2 = re.search(r'\bEC\s+(\d{5,10})\b', window, re.IGNORECASE)
            if m2:
                current_master = m2.group(1)
                recent_lines.clear()
                i += 1
                continue
        if line.startswith('[ТАБЛИЦА]'):
            recent_lines.clear()
            i += 1
            continue
        # PNT: detect header and parse following data rows with farm from recent lines
        if 'CONCEPT' in line.upper() and 'SCIENTIFIC' in line.upper() and 'TARIFOFIC' in line.upper():
            i += 1
            while i < len(lines):
                data_line = lines[i]
                if not data_line or data_line.startswith('[') or data_line.upper().startswith('TOTAL') or 'SUMMARY' in data_line.upper():
                    break
                pnt_item = _parse_ecu_pnt_inline_line(data_line, recent_lines, current_master)
                if pnt_item:
                    pnt_items.append(pnt_item)
                i += 1
            recent_lines.clear()
            continue
        pipe_item = _parse_ecu_pipe_line(line, current_master)
        if pipe_item:
            pipe_items.append(pipe_item)
            recent_lines.append(line)
            if len(recent_lines) > 8:
                recent_lines.pop(0)
            i += 1
            continue
        ebf_item = _parse_ecu_ebf_inline_line(line, recent_lines[-1] if recent_lines else '', current_master)
        if ebf_item:
            ebf_items.append(ebf_item)
            recent_lines.append(line)
            if len(recent_lines) > 8:
                recent_lines.pop(0)
            i += 1
            continue
        inline_item = _parse_ecu_inline_line(line, current_master)
        if inline_item:
            inline_items.append(inline_item)
        recent_lines.append(line)
        if len(recent_lines) > 8:
            recent_lines.pop(0)
        i += 1

    items: list[CertItem] = []
    seen: set[tuple] = set()
    for it in pipe_items + inline_items + ebf_items + pnt_items:
        key = (it.master_invoice, it.invoice_nr, it.flower, it.boxes, it.stems)
        if key in seen:
            continue
        seen.add(key)
        items.append(it)

    # If the page looks like an EUR.1 face page, use face-page aggregates
    # (authoritative totals per flower) instead of invoice rows.
    face_items = _extract_ecu_face_page_items(text, current_master)
    if face_items:
        return face_items

    return items


# === PyMuPDF table extraction for Ecuador (cleaner farm/client split) ===

_EC_TABLE_HEADER_SET = {
    'concept', 'farm', 'client', 'invoice', 'f. bxs', 'pieces',
    'tariff #', 'stems', 'p. unit', 'total price',
}
# Air Canada commercial master-invoice variant
_EC_AIR_CANADA_HEADER_SET = {
    'bxs', 'pcs', 'weight', 'hts', 'flower', 'stems',
    'price', 'unitary', 'us dollars', 'price total', 'hawb', 'farms',
}


def _is_ecu_table(rows: list[list[str]]) -> bool:
    """Check whether a PyMuPDF table looks like an Ecuador master-invoice table."""
    for row in rows:
        cells = [str(c or '').strip().lower() for c in row[:10]]
        if len(cells) < 10:
            continue
        if set(cells) >= _EC_TABLE_HEADER_SET:
            return True
        # Also accept when header cells are spread one per cell
        if all(h in cells for h in _EC_TABLE_HEADER_SET):
            return True
        # Air Canada variant: first cell BXS and last cell FARMS
        if cells and 'bxs' in cells[0] and 'farms' in cells[-1]:
            return True
    return False


def _ecu_master_from_page_text(text: str) -> str:
    """Return the first Ecuador master-invoice number found in page text."""
    masters = _extract_ecu_master_invoices(text)
    return masters[0] if masters else ''


def _parse_ecu_page(meta: CertPageMeta, ocr_master: str = '') -> list[CertItem]:
    """Parse Ecuador item rows from a single page.

    Prefers PyMuPDF table rows when available; otherwise falls back to
    inline/pipe text parsing.  Associates rows with the page's master invoice.
    If the text layer does not contain a master invoice, uses the OCR-provided
    value for this page.
    """
    current_master = _ecu_master_from_page_text(meta.text) or ocr_master
    items: list[CertItem] = []
    if meta.ecu_table_rows:
        current_format = 'standard'
        for row in meta.ecu_table_rows:
            cells = [str(c or '').strip().lower() for c in row]
            # Detect Air Canada commercial master-invoice header
            if cells and 'bxs' in cells[0] and len(cells) >= 10 and 'farms' in cells[-1]:
                current_format = 'air_canada'
                continue
            # Detect standard header
            if set(cells) >= _EC_TABLE_HEADER_SET or all(h in cells for h in _EC_TABLE_HEADER_SET):
                current_format = 'standard'
                continue
            item: Optional[CertItem] = None
            if current_format == 'air_canada':
                item = _parse_ecu_air_canada_row(row, current_master)
            else:
                item = _parse_ecu_standard_row(row, current_master)
            if item:
                items.append(item)
    if not items:
        for item in _parse_ecu_items(meta.text):
            if not item.master_invoice:
                item.master_invoice = current_master
            items.append(item)
    return items


def _extract_ecu_tables_fitz(pdf_path: str) -> dict[int, list[list[list[str]]]]:
    """Extract Ecuador-style item tables from all pages using PyMuPDF.

    Returns {page_index: [table_rows, ...]} where table_rows is a list of 10-cell rows.
    """
    if fitz is None:
        return {}
    result: dict[int, list[list[list[str]]]] = {}
    try:
        doc = fitz.open(pdf_path)
    except Exception:
        return result
    try:
        for idx, page in enumerate(doc):
            try:
                tabs = page.find_tables()
            except Exception:
                continue
            page_tables: list[list[list[str]]] = []
            for tab in tabs.tables:
                try:
                    rows = tab.extract()
                except Exception:
                    continue
                if not rows:
                    continue
                # Normalize to strings and skip empty rows
                clean_rows = []
                for row in rows:
                    if not row:
                        continue
                    clean = [str(c or '').replace('\n', ' ').strip() for c in row]
                    if not any(clean):
                        continue
                    clean_rows.append(clean)
                if _is_ecu_table(clean_rows):
                    page_tables.append(clean_rows)
            if page_tables:
                result[idx] = page_tables
    finally:
        doc.close()
    return result


# === Ecuador scan-page summary extraction (text layer empty, image only) ===
# EUR.1 face-page format: "6 PCS / ALSTROEMERIA 060319 / 1000 TALLOS"
_ECU_SCAN_CERT_LINE = re.compile(
    r'(?P<boxes>\d+(?:\.\d+)?)\s*PCS\s*/\s*(?P<flower>[A-Z]+)\s+'
    r'(?P<hs>\d{4}\.\d{2}(?:\.\d{4}|\.\d{2}\.\d{2})|\d{6})\s*/\s*'
    r'(?P<stems>[\d,]+)\s*(?:TALLOS|STEMS)',
    re.IGNORECASE,
)
# Separate summary table: "CLAVEL 750 2 0603.12.10.00 124.50"
# Noise tokens between stems and HS are allowed (e.g. "ALSTROEMERIA 1,000 Zz 0603.19.30.00 po 306.40")
_ECU_SCAN_SUMMARY_LINE = re.compile(
    r'(?P<flower>[A-Z]{3,})\s+(?P<stems>[\d,]+)(?:\s+(?P<boxes>\d+(?:\.\d+)?))?[^\n]*?'
    r'(?P<hs>\d{4}\.\d{2}(?:\.\d{4}|\.\d{2}\.\d{2}))[^\n]*?'
    r'(?P<total>\d[\d,]*\.\d{2})\b',
    re.IGNORECASE,
)
# Commercial master invoice scan: "ROSES 0603.11.00.50 |AGRINAG SAS. 600 0.46 277.50"
# flower comes first, then HS code, then grower, then stems and price.
_ECU_SCAN_INVOICE_LINE = re.compile(
    r'(?P<flower>ROSAS?|ROSES?|ALSTROEMERIA|CLAVEL|CLAVELINA|GYPSOPHILA|MATHIOLA|'
    r'CHRYSANTHEMUM|DIANTHUS|HYPERICUM|CALLA|LILIUM|LILY|ANTHURIUM|STOCK)'
    r'[^\n]*?(?P<hs>\d{4}\.\d{2}(?:\.\d{4}|\.\d{2}\.\d{2}))[^\n]*?'
    r'\b(?P<stems>\d{2,6})\s+(?P<price>\d+\.\d{2})\s+(?P<total>\d+\.\d{2})\b',
    re.IGNORECASE,
)
# Aggregate totals at the bottom of a commercial master invoice scan:
# e.g. "17.25 10105 4160.40"  (boxes stems total_value)
_ECU_SCAN_TOTAL_LINE = re.compile(
    r'(?P<boxes>\d+(?:\.\d+)?)\s+(?P<stems>[\d,]+)\s+(?P<total>[\d,]+(?:\.\d+)?)\s*$',
    re.MULTILINE,
)
# Total boxes on EUR.1 face page: "31 PCS FRESH CUT FLOWERS"
_ECU_SCAN_TOTAL_BOXES = re.compile(
    r'(?P<boxes>\d+(?:\.\d+)?)\s*PCS\s+(?:FRESH\s+CUT\s+)?FLOWERS',
    re.IGNORECASE,
)


def _parse_ecu_scan_summary(text: str, master_invoice: str) -> list[CertItem]:
    """Extract per-flower summary rows from OCR'd Ecuador scan pages.

    Combines EUR.1 face-page aggregates (authoritative stems) with any
    commercial-invoice rows found on the same scan page so downstream
    merging can keep stems from the face and boxes/farm/invoice from
    the invoice.
    """
    items: list[CertItem] = []
    if not text or not master_invoice:
        return items

    # 1. EUR.1 face-page formats (priority: authoritative certificate totals).
    items.extend(_extract_ecu_face_page_items(text, master_invoice))

    # 2. Separate summary table: flower + stems + optional boxes + hs + total.
    # Also matches noisy OCR like "ALSTROEMERIA 1,000 Zz 0603.19.30.00 po 306.40".
    for m in _ECU_SCAN_SUMMARY_LINE.finditer(text):
        try:
            stems = int(m.group('stems').replace(',', '').replace('.', ''))
            boxes_str = m.group('boxes')
            boxes = float(boxes_str.replace(',', '.')) if boxes_str else 0.0
            flower = m.group('flower').upper()
            hs = m.group('hs')
            if stems <= 0 or flower in ('TOTAL', 'VARIETY', 'STEMS', 'VALUE'):
                continue
            # Avoid exact duplicates already captured from the face page.
            if any(it.flower == flower and it.stems == stems and it.is_face_page for it in items):
                continue
            items.append(CertItem(
                flower=flower,
                boxes=boxes,
                stems=stems,
                hs_code=hs,
                master_invoice=master_invoice,
            ))
        except Exception:
            continue

    # 3. Commercial master invoice scan format:
    # "ROSES 0603.11.00.50 |AGRINAG SAS. 600 0.46 277.50"
    seen_inv: dict[str, int] = {}
    for m in _ECU_SCAN_INVOICE_LINE.finditer(text):
        try:
            flower = m.group('flower').upper()
            hs = m.group('hs')
            stems = int(m.group('stems').replace(',', ''))
            if stems <= 0:
                continue
            key = f'{flower}|{hs}'
            seen_inv[key] = seen_inv.get(key, 0) + stems
        except Exception:
            continue
    for key, total_stems in seen_inv.items():
        flower, hs = key.split('|', 1)
        # Avoid exact duplicates already captured from the face page.
        if any(it.flower == flower and it.stems == total_stems and it.is_face_page for it in items):
            continue
        items.append(CertItem(
            flower=flower,
            boxes=0.0,
            stems=total_stems,
            hs_code=hs,
            master_invoice=master_invoice,
        ))

    return items


def _extract_ecu_scan_totals(text: str) -> tuple[float, int] | None:
    """Extract aggregate boxes/stems from an OCR'd Ecuador scan page.

    Searches the entire text (not just the tail) because totals may appear
    mid-page on commercial master invoices.
    """
    if not text:
        return None
    lines = text.strip().split('\n')
    non_empty = [ln.strip() for ln in lines if ln.strip()]
    # Search from bottom up across the whole page
    for ln in reversed(non_empty):
        m = _ECU_SCAN_TOTAL_LINE.search(ln)
        if m:
            try:
                boxes = float(m.group('boxes').replace(',', '.'))
                stems = int(m.group('stems').replace(',', '').replace('.', ''))
                if 0 < boxes < 10000 and 0 < stems < 1000000:
                    return boxes, stems
            except Exception:
                continue
    return None


# EUR.1 face-page compact format: "ALSTROEMERIA 180 STEMS", "ROSAS 9375 STEMS"
# Also covers lines like "6 PCS / ALSTROEMERIA / 1000 TALLOS" on EUR.1 scans.
_ECU_SCAN_FLOWER_STEM_LINE = re.compile(
    r'(?:^|\b)(?P<flower>ROSAS?|ROSES?|ALSTROEMERIA|CLAVEL|CLAVELINA|GYPSOPHILA|MATHIOLA|'
    r'CHRYSANTHEMUM|DIANTHUS|HYPERICUM|CALLA|LILIUM|LILY|ANTHURIUM|STOCK)'
    r'\s+(?P<stems>[\d,]+)\s*(?:STEMS?|TALLOS?)',
    re.IGNORECASE | re.MULTILINE,
)


def _extract_ecu_face_page_items(text: str, master_invoice: str = '') -> list[CertItem]:
    """Извлечь позиции с лицевой страницы EUR.1 (все цвета + общее PCS).

    Поддерживаемые форматы:
      • "6 PCS / ALSTROEMERIA 060319 / 1000 TALLOS"
      • "ALSTROEMERIA 180 STEMS"
      • "31 PCS FRESH CUT FLOWERS" (общее количество коробок)
    """
    items: list[CertItem] = []
    if not text:
        return items

    # 1) PCS + flower + HS + stems
    for m in _ECU_SCAN_CERT_LINE.finditer(text):
        try:
            boxes = float(m.group('boxes').replace(',', '.'))
            stems = int(m.group('stems').replace(',', '').replace('.', ''))
            flower = m.group('flower').upper()
            hs = m.group('hs')
            items.append(CertItem(
                flower=flower,
                boxes=boxes,
                stems=stems,
                hs_code=hs,
                master_invoice=master_invoice,
                is_face_page=True,
            ))
        except Exception:
            continue

    # 2) Compact flower + stems (no per-flower boxes)
    if not items:
        for m in _ECU_SCAN_FLOWER_STEM_LINE.finditer(text):
            try:
                stems = int(m.group('stems').replace(',', '').replace('.', ''))
                flower = m.group('flower').upper()
                items.append(CertItem(
                    flower=flower,
                    boxes=0.0,
                    stems=stems,
                    master_invoice=master_invoice,
                    is_face_page=True,
                ))
            except Exception:
                continue

    # 3) Общее количество коробок на сертификате — распределяем пропорционально
    #    только если у нас есть compact flower/stems без boxes.
    if items and all(it.boxes == 0.0 for it in items):
        total_boxes = 0.0
        for m in _ECU_SCAN_TOTAL_BOXES.finditer(text):
            try:
                total_boxes = float(m.group('boxes').replace(',', '.'))
                break
            except Exception:
                continue
        if total_boxes > 0:
            total_stems = sum(it.stems for it in items)
            if total_stems > 0:
                for it in items:
                    it.boxes = round(total_boxes * it.stems / total_stems, 2)
                # Round to match total_boxes exactly on the last item
                diff = total_boxes - sum(it.boxes for it in items)
                if items and diff != 0:
                    items[-1].boxes = round(items[-1].boxes + diff, 2)

    return items


def _merge_ecu_face_and_invoice(items: list[CertItem]) -> list[CertItem]:
    """Нормализовать названия цветов и убрать дублирование face/invoice.

    Для каждого master invoice:
      • названия цветов приводятся к канонической форме (ROSAS/ROSES → ROSA);
      • если есть детальные строки commercial invoice — используем их,
        а face-агрегаты по тем же цветкам отбрасываем, чтобы не терять
        реальное количество стеблей и не дублировать итоги;
      • если invoice содержит только общий FLOWERS — заменяем его
        разбивкой по цветам с face-страницы, распределяя коробки
        пропорционально стеблям;
      • если есть только face-позиции — оставляем их.
    """
    if not items:
        return items

    # 1) Нормализуем названия цветов во всех позициях.
    normalized: list[CertItem] = []
    for it in items:
        if it.flower:
            it.flower = _normalize_flower_name(it.flower)
        normalized.append(it)

    by_master: dict[str, dict[str, list[CertItem]]] = {}
    for it in normalized:
        bucket = by_master.setdefault(it.master_invoice, {'face': [], 'inv': []})
        bucket['face' if it.is_face_page else 'inv'].append(it)

    out: list[CertItem] = []
    for master, groups in by_master.items():
        face = groups['face']
        inv = groups['inv']

        if not face:
            out.extend(inv)
            continue
        if not inv:
            out.extend(face)
            continue

        inv_flowers = {it.flower for it in inv}
        # Если commercial invoice даёт только общий агрегат — разбиваем
        # по цветам с лицевой страницы и прикрепляем коробки.
        if inv_flowers == {'FLOWERS'}:
            generic = inv[0]
            face_stems = sum(it.stems for it in face)
            for it in face:
                if face_stems > 0 and generic.boxes > 0:
                    it.boxes = round(generic.boxes * it.stems / face_stems, 2)
                it.invoice_nr = master
                out.append(it)
            if face:
                diff = round(generic.boxes - sum(it.boxes for it in face), 2)
                if diff:
                    face[-1].boxes = round(face[-1].boxes + diff, 2)
            continue

        # Есть детальные invoice-строки — используем их как основу,
        # а face-агрегаты отбрасываем, чтобы не затирать точные стебли.
        out.extend(inv)

    return out


# === Регекспы для шапки Ecuador ===

_ECU_MASTER_INV  = re.compile(r'MASTER\s+INVOICE\s*#\s*([A-Z0-9]{6,12})', re.IGNORECASE)
# Air Canada commercial master-invoice variant: Invoice No. EC 421820
_ECU_AIR_CANADA_INV = re.compile(r'Invoice\s*No\.?\s*EC\s*(\d{5,10})', re.IGNORECASE)
_ECU_AWB         = re.compile(r'AWB\s*[:#]?\s*(\d{3}[- ]\d{4}\s?\d{4})', re.IGNORECASE)
_ECU_DATE        = re.compile(r'Date:\s*([A-Za-z]+\s+\d{1,2},\s*\d{4})')
_ECU_EXPORTER    = re.compile(r'^([A-Z][A-Z\.\s&,]+?)\s+\d{10,}\s+MASTER', re.MULTILINE)

# EUR1 / DIAN certificate numbers
# EUR1 / EUR.1 — несколько форматов в сканированных сертификатах
def _normalize_eur1(nr: str) -> str:
    """Normalize EUR.1 number by removing whitespace and converting to uppercase."""
    return re.sub(r'\s+', '', nr.upper())


_EUR1_PATTERNS = [
    re.compile(r'\bEUR1[-\s]?([A-Z]?\d{5,20}[A-Z]?)\b'),                           # EUR1-12345678, EUR1-B1069139
    re.compile(r'\bEUR\.?\s*1\b[^a-z\n]{0,30}?([A-Z]\s+\d{6,20})\b'),              # EUR. 1 NO B 1059064
    re.compile(r'\bEUR\.?\s*1\b[^a-z\n]{0,25}?\b((?:[A-Z]{1,3}\s*)?\d{6,20}[A-Z]?)\b'),         # EUR.1 No A 0320033, EUR. 1 NA 09908859202600041645P
    # OCR-noisy Kenya scans: "EUR. 4 no B 1059064" / "EUR 4 . no B 1059075" /
    # "EU R 4 No) B 4 059064" / "EU R 4 ac B 4059090".  Spaces inside the
    # digit block and stray punctuation between EUR/NO/B are allowed.
    re.compile(r'\bE\s*U\s*R\.?\s*[14]\b[^\n]{0,40}?NO\.?[^\w\s]*\s*([A-Z]\s*\d(?:\s*\d){5,19})\b', re.IGNORECASE),
    re.compile(r'\bE\s*U\s*R\.?\s*[14]\b[^\n]{0,40}?\b([A-Z]\s*\d(?:\s*\d){5,19})\b', re.IGNORECASE),
]
_DIAN_PATTERN    = re.compile(r'\bDIAN[-\s]?(\d{9,20})\b', re.IGNORECASE)

# === Регекспы Colombia (DIAN) ===
#   AWB:006-4539-6912  или "AWB No\n006-4539-6912"
# Общий паттерн: "NNN-NNNN NNNN" (3-4space4) или "NNN-NNNNNNNN" (3-8)
# Space, hyphen, en dash and em dash are all treated as AWB separators.
_DASH_LIKE = r'[\s\-\u2013\u2014]'
_GENERAL_AWB     = re.compile(r'\b(\d{3}' + _DASH_LIKE + r'+\d{4}' + _DASH_LIKE + r'+\d{4})\b')
_COL_AWB         = re.compile(r'\b(\d{3}-\d{4}-?\d{4})\b')
#   BOGOTA, July 01, 2026
_COL_DATE_EN     = re.compile(r'\b([A-Z][a-z]+\s+\d{1,2},\s*\d{4})\b')
#   2026/06/30
_COL_DATE_ISO   = re.compile(r'\b(\d{4}/\d{2}/\d{2})\b')
#   FLOWERS FOR THE WORLD   (первая строка блока Exporter — capital letters + space + digits)
_COL_EXPORTER   = re.compile(
    r'^1\.\s*Exporter[^\n]*\n([A-Z][A-Z\.\s&,\-\d]+?)\s*\n', re.MULTILINE)

# Colombia proforma table (PIECES FULLES DESCRIPTION ... UNITS PRICE TOTAL FOB)
_COL_PROFORMA_NR  = re.compile(r'PROFORMA\s+(IFC-?\d+)', re.IGNORECASE)
_COL_DATE_RE      = re.compile(r'R\.E\.\s*(\d{1,2}/\d{1,2}/\d{4})')
_COL_EXPORTER_2   = re.compile(
    r'NIT\s+[\d.\-]+\s+PROFORMA[^\n]*\n([^\n]+?)(?:\s+AWB)', re.IGNORECASE)
# 26 6,5 CARNATIONS T C.O 10400 0,10 1040,00  — европейский формат десятичных
_COL_TABLE_LINE   = re.compile(
    r'^(\d+)\s+([\d,]+)\s+(?P<flower>[A-Z]+)\s+\w+\s+[\w.]+\s+(?P<stems>\d+)\s+[\d,]+\s+[\d,]+$'
)
_COL_TOTAL_BOXES  = re.compile(r'TOTAL\s+BOXES\s+PIEZAS\s+([\d,]+)', re.IGNORECASE)

# Kenya patterns
_KE_INVOICE_NR    = re.compile(
    r'INVOICE\s*(?:NO\.?)?\s*[:#]?\s*(\d{4,}(?:[-]\d+)?|[A-Z]{2,3}-\d+)',
    re.IGNORECASE,
)
# Kenya ULTRA FLO Form A (Certificate of Origin / FORMA): Reference No. 5 3 3.3 9 2
_KE_FORM_A_HEADER = re.compile(
    r'GENERALISED\s+SYSTEM\s+OF\s+PREFERENCES|CERTIFICATE\s+OF\s+ORIGIN|\bFORMA\b',
    re.IGNORECASE,
)
_KE_FORM_A_REF    = re.compile(r'Reference\s+No\.?\s*([\d\s\.]+)', re.IGNORECASE)
_KE_FLOWER_LINE   = re.compile(
    r'^(?P<flower>ROSES?|CARNATIONS?|ALSTROEMERIA|HYPERICUM|CHRYSANTHEMUM)\s+(?P<stems>\d+)\s+STEMS?',
    re.IGNORECASE
)
# Kenya Master Invoice table (Milele-format):
# «Roses 35-80 2 195 390 $0,10 $39,00»
# columns: VARIETY  BOXES  P.RATE  TT_STEMS  ...  (TT STEMS = 4th number)
_KE_INV_TABLE_LINE = re.compile(
    r'^(?P<flower>ROSES?|CARNATIONS?|ALSTROEMERIA|HYPERICUM|CHRYSANTHEMUM)'
    r'(?:\s+(?P<grade>[\d]+-[\d]+|[\d]+\s*(?:cm|CM)))?'   # необязательная ростовка: 35-80 или 80 cm
    r'\s+(?P<boxes>\d+)'          # # BOXES
    r'\s+(?P<rate>\d+)'           # P. RATE (стеблей на коробку)
    r'\s+(?P<stems>\d+)',          # TT STEMS
    re.IGNORECASE
)
# Кенийский фито-сертификат: отдельная строка-блок начинается с 5-6-значного номера бланка
_KE_CERT_NR_LINE  = re.compile(r'^(\d{5,7})$')

# AWB formats: "176-2825 7471", "157 5629 3742", "157-5629-3742", "006-45396912"
_AWB_PARTS = re.compile(r'\b(\d{3})' + _DASH_LIKE + r'*(\d{4})' + _DASH_LIKE + r'*(\d{4})\b')


def _normalize_awb(raw: str) -> str:
    m = _AWB_PARTS.search(raw)
    if not m:
        return ''
    return f'{m.group(1)}-{m.group(2)}-{m.group(3)}'


# ETHAM Fresh Exports invoice table
# FARM | BOXES | VARIETY | STEMS | PRICE | TOTAL | WEIGHT
# BARAKA 3 roses 1260 0.04 50.40
_KE_ETHAM_TABLE_HEADER = re.compile(
    r'FARM\s+BOXES\s+VARIETY\s+STEMS',
    re.IGNORECASE,
)
_KE_ETHAM_TABLE_LINE = re.compile(
    r'^(?P<farm>[A-Z]{3,10})\s+'
    r'(?P<boxes>\d{1,3})\s+'
    r'(?P<flower>[A-Za-z]+(?:\s+[A-Za-z]+)?)\s+'
    r'(?P<stems>\d{3,6})',
    re.IGNORECASE,
)

# ETHAM invoice number: "EFE-980"
_KE_ETHAM_INV = re.compile(r'EFE[-\s]?(\d+)', re.IGNORECASE)

# ETHAM aggregate on the EUR.1 face page: "23 Boxes of Fresh cut roses (11300 Stems)"
# OCR may append garbage such as "4.1.0KGS" to the same line; allow arbitrary text
# between the flower name and the parenthesised stem count.
_KE_ETHAM_AGG = re.compile(
    r'(\d+)\s+Boxes\s+of\s+Fresh\s+[Cc]ut\s+(?:roses?|flowers?)\b[\s\S]*?\(?(\d+)\s*Stems\)?',
    re.IGNORECASE,
)

# Noise-tolerant ETHAM row parser.
# OCR output may contain pipes '|', underscores '_', stray dots/commas and
# merged/split tokens.  We first strip non-alphanumeric noise, then try to
# match the canonical column order: FARM BOXES VARIETY STEMS ...
_KE_ETHAM_TABLE_LINE_LOOSE = re.compile(
    r'^(?P<farm>[A-Z]{3,10})\s+'
    r'(?P<boxes>\d{1,3})\s+'
    r'(?P<flower>[A-Za-z]+(?:\s+[A-Za-z]+)*)\s+'
    r'(?P<stems>\d{3,5})\b',
    re.IGNORECASE,
)


_ETHAM_FLOWER_NAMES = re.compile(
    r'^(?:ROSES?|SROSES?|FOSES?|ROSA|CARNATIONS?|ALSTROEMERIA|HYPERICUM|CHRYSANTHEMUM)$',
    re.IGNORECASE,
)


def _normalize_etham_flower(raw: str) -> str:
    """Normalize an ETHAM variety name, forgiving common OCR misreadings."""
    if not raw:
        return ''
    token = re.sub(r'[^A-Za-z]', '', raw).upper()
    if token in _KE_FLOWER_MAP:
        return _KE_FLOWER_MAP[token]
    if _ETHAM_FLOWER_NAMES.match(token):
        return 'ROSA' if 'ROSE' in token or 'ROSA' in token or token in ('FOSE', 'FOSES', 'SROSE', 'SROSES') else token
    if 'ROSE' in token or 'ROSA' in token or token in ('FOSE', 'FOSES', 'SROSE', 'SROSES'):
        return 'ROSA'
    if 'CARNATION' in token:
        return 'DIANTHUS'
    # Unknown but looks like a real word — return it as-is in upper case.
    return raw.strip().upper()


def _clean_ocr_line(line: str) -> str:
    """Remove OCR artifacts (pipes, underscores, currency marks, etc.) from a line."""
    # Keep letters, digits and whitespace; collapse multiple spaces.
    cleaned = re.sub(r'[^A-Za-z0-9\s]', ' ', line)
    # Split tokens like "1sroses" or "11roses" into separate numbers/words.
    cleaned = re.sub(r'(?<=\d)(?=[A-Za-z])|(?<=[A-Za-z])(?=\d)', ' ', cleaned)
    cleaned = re.sub(r'\s+', ' ', cleaned).strip()
    return cleaned


def _parse_etham_row(line: str) -> Optional[re.Match]:
    """Try to parse an ETHAM farm row, tolerating common OCR noise."""
    cleaned = _clean_ocr_line(line)
    if not cleaned or cleaned.upper().startswith('TOTAL'):
        return None

    # Helper to wrap parsed values in a match-compatible object.
    def _make_match(groups: dict) -> re.Match:
        class _PseudoMatch:
            def __init__(self, groups: dict):
                self._groups = groups
            def group(self, name: str) -> str:
                return self._groups[name]
        return _PseudoMatch(groups)  # type: ignore[return-value]

    # 1) Try the strict regex on cleaned text.
    m = _KE_ETHAM_TABLE_LINE.match(cleaned)
    if m:
        flower = _normalize_etham_flower(m.group('flower'))
        if flower:
            return _make_match({
                'farm': m.group('farm'),
                'boxes': m.group('boxes'),
                'flower': flower,
                'stems': m.group('stems'),
            })

    # 2) Try the loose regex on cleaned text.
    m = _KE_ETHAM_TABLE_LINE_LOOSE.match(cleaned)
    if m:
        flower = _normalize_etham_flower(m.group('flower'))
        if flower:
            return _make_match({
                'farm': m.group('farm'),
                'boxes': m.group('boxes'),
                'flower': flower,
                'stems': m.group('stems'),
            })

    # 3) Token-based fallback for heavily broken rows.
    tokens = cleaned.split()
    if len(tokens) < 4:
        return None

    num_indices = [i for i, t in enumerate(tokens) if t.isdigit()]
    if len(num_indices) < 2:
        return None

    # Find boxes: a small positive integer (1-99) after an all-alpha farm name.
    boxes_idx = None
    farm_end = None
    for i in num_indices:
        if i == 0:
            continue
        if all(t.isalpha() and len(t) >= 2 for t in tokens[:i]):
            val = int(tokens[i])
            if 1 <= val <= 99:
                boxes_idx = i
                farm_end = i
                break
    if boxes_idx is None:
        return None

    farm = ' '.join(tokens[:farm_end])
    if not re.match(r'^[A-Z]{2,10}$', farm, re.IGNORECASE):
        return None

    # Find stems: the first number >= 100 that appears after at least one
    # alphabetic token (the flower) following the boxes.
    stems_idx = None
    flower_tokens: list[str] = []
    for i in num_indices:
        if i <= boxes_idx:
            continue
        between = tokens[boxes_idx + 1:i]
        alpha_between = [t for t in between if t.isalpha()]
        if alpha_between and int(tokens[i]) >= 100:
            stems_idx = i
            flower_tokens = alpha_between
            break
    if stems_idx is None:
        return None

    flower = _normalize_etham_flower(' '.join(flower_tokens))
    if not flower:
        return None

    return _make_match({
        'farm': farm,
        'boxes': tokens[boxes_idx],
        'flower': flower,
        'stems': tokens[stems_idx],
    })

# Explicit AWB / MAWB label (used in _extract_page_meta)
_EXPLICIT_AWB = re.compile(
    r'(?:AWB|MAWB|MAWEB)\s*(?:NO\.?|NUMBER)?\s*[.:]?\s*([\d\s\-\u2013\u2014]{10,20})',
    re.IGNORECASE,
)
# Kenya exporter "FLOWERS LIMITED / LTD" (phyto header check)
_KE_FLOWERS_LTD = re.compile(r'FLOWERS\s+(?:LIMITED|LTD)', re.IGNORECASE)
# Kenya Milele master-invoice table header
_KE_MILELE_TABLE = re.compile(
    r'VARIETY.*BOXES.*STEMS|P\.?\s*RATE.*TT\s*STEMS',
    re.IGNORECASE,
)

# Module-level Kenya flower name normalization map (used in _items_from_block and parse_certificate)
_KE_FLOWER_MAP = {
    'ROSES': 'ROSA', 'ROSE': 'ROSA',
    'CARNATIONS': 'DIANTHUS', 'CARNATION': 'DIANTHUS',
}


def _ocr_pages_tesseract(
    pdf_path: str,
    page_indices: list[int],
    dpi: int = 300,
    psm: int = 4,
) -> dict[int, str]:
    """OCR a list of PDF pages with Tesseract and return {0-based index: text}.

    Uses PyMuPDF to render pages and pytesseract for full-page OCR.
    ``psm`` controls the Tesseract page segmentation mode (default 4):
    - psm=4 preserves row order for invoice tables
    - psm=6 treats the page as a uniform block; better for totals extraction
    """
    result: dict[int, str] = {}
    if not page_indices or fitz is None or Image is None or pytesseract is None:
        return result
    try:
        doc = fitz.open(pdf_path)
    except Exception:
        return result
    try:
        scale = dpi / 72.0
        for idx in page_indices:
            if idx < 0 or idx >= len(doc):
                continue
            page = doc.load_page(idx)
            pix = page.get_pixmap(matrix=fitz.Matrix(scale, scale), alpha=False)
            img = Image.frombytes('RGB', (pix.width, pix.height), pix.samples)
            text = pytesseract.image_to_string(img, lang='eng', config=f'--psm {psm}')
            result[idx] = text
    finally:
        doc.close()
    return result


def _ocr_pages(
    pdf_path: str,
    page_indices: list[int],
    dpi: int = 300,
    psm: int = 4,
) -> dict[int, str]:
    """OCR pages using Yandex Vision if configured, otherwise Tesseract.

    Yandex is tried first for better quality on scanned certificates.
    Tesseract remains the local fallback on API errors or empty responses.
    """
    if yandex_vision is not None and yandex_vision.is_configured():
        try:
            yandex_result = yandex_vision.ocr_pages_yandex(pdf_path, page_indices, dpi=dpi)
            if yandex_result and any(v.strip() for v in yandex_result.values()):
                return yandex_result
        except Exception as exc:
            # Fallback to Tesseract on any unexpected problem.
            logger.debug("Yandex OCR failed, falling back to Tesseract: %s", exc)
    return _ocr_pages_tesseract(pdf_path, page_indices, dpi=dpi, psm=psm)


def _extract_page_meta(page, idx: int, ocr: dict, text: str = '') -> CertPageMeta:
    if not text and page is not None:
        text = page.extract_text() or ''
    meta = CertPageMeta(idx=idx, text=text)

    # AWB candidates
    for m in _GENERAL_AWB.finditer(text):
        norm = _normalize_awb(m.group(1))
        if norm and norm not in meta.awb_candidates:
            meta.awb_candidates.append(norm)

    # Prefer explicit "AWB NUMBER: ..." or "MAWB NO: ..." if present
    explicit = _EXPLICIT_AWB.search(text)
    if explicit:
        norm = _normalize_awb(explicit.group(1))
        if norm:
            meta.awb = norm
            if norm not in meta.awb_candidates:
                meta.awb_candidates.insert(0, norm)
    elif meta.awb_candidates:
        meta.awb = meta.awb_candidates[0]

    # Certificate markers from text
    _is_kenya_text = 'KENYA' in text.upper()
    for pat in _EUR1_PATTERNS:
        for m in pat.finditer(text):
            nr = _normalize_eur1(m.group(1).strip())
            # Tesseract on Kenya scans often reads the leading "1" of EUR.1
            # and of the B1xxxxxx number as "4".  Fix the obvious case.
            if nr and _is_kenya_text and len(nr) == 8 and nr.startswith('B4') and nr[1:].isdigit():
                nr = 'B1' + nr[2:]
            if nr and nr not in meta.eur1_numbers:
                meta.eur1_numbers.append(nr)
    for m in _DIAN_PATTERN.finditer(text):
        nr = m.group(1).strip()
        if nr and nr not in meta.dian_numbers:
            meta.dian_numbers.append(nr)

    # Kenya phyto header: bare number + FLOWERS LIMITED (within first 5 lines only)
    first_line = text.strip().split('\n')[0].strip() if text.strip() else ''
    header_lines = text.strip().split('\n')[:5]
    header = '\n'.join(header_lines)
    if _KE_CERT_NR_LINE.fullmatch(first_line) and _KE_FLOWERS_LTD.search(header):
        meta.phyto_cert_number = first_line

    # Kenya ULTRA FLO Form A (Certificate of Origin): extract Reference No.
    if _KE_FORM_A_HEADER.search(text):
        ref_m = _KE_FORM_A_REF.search(text)
        if ref_m:
            ref = re.sub(r'[^\d]', '', ref_m.group(1))
            if ref and len(ref) >= 5:
                meta.form_a_ref_number = ref

    # Invoice numbers
    for m in _KE_INVOICE_NR.finditer(text):
        inv = m.group(1)
        if inv and inv not in meta.invoice_numbers:
            meta.invoice_numbers.append(inv)
    eth = _KE_ETHAM_INV.search(text)
    if eth:
        meta.etham_invoice = f'EFE-{eth.group(1)}'
        if meta.etham_invoice not in meta.invoice_numbers:
            meta.invoice_numbers.append(meta.etham_invoice)

    # Proforma (Colombia)
    prof = _COL_PROFORMA_NR.search(text)
    if prof:
        p = prof.group(1).upper().replace(' ', '')
        if p not in meta.proforma_numbers:
            meta.proforma_numbers.append(p)

    # Table detection (use noise-cleaned text so OCR pipes/underscores don't
    # hide the ETHAM/Milele headers).
    clean_text = re.sub(r'[^A-Za-z0-9\s]', ' ', text)
    meta.has_etham_table = bool(_KE_ETHAM_TABLE_HEADER.search(clean_text))
    meta.has_milele_table = bool(_KE_MILELE_TABLE.search(clean_text))

    # OCR sometimes drops the ETHAM header line but leaves the farm rows.
    # Recognize the table by at least two valid farm rows when the page also
    # looks like an ETHAM invoice.
    if not meta.has_etham_table and (meta.etham_invoice or 'EURO 1' in text.upper()):
        etham_rows = 0
        for line in text.split('\n'):
            if _parse_etham_row(line):
                etham_rows += 1
                if etham_rows >= 2:
                    meta.has_etham_table = True
                    break

    # OCR enrichment (per-page lookup)
    ocr_page = ocr.get(idx + 1, {})
    for nr in ocr_page.get('eur1', []):
        nr_norm = _normalize_eur1(nr)
        if nr_norm and nr_norm not in meta.eur1_numbers:
            meta.eur1_numbers.append(nr_norm)
    for nr in ocr_page.get('dian', []):
        if nr not in meta.dian_numbers:
            meta.dian_numbers.append(nr)
    if ocr_page.get('awb'):
        norm = _normalize_awb(ocr_page['awb'])
        if norm and norm not in meta.awb_candidates:
            meta.awb_candidates.insert(0, norm)
            # Only let OCR set the page AWB if text layer didn't already provide one.
            if not meta.awb:
                meta.awb = norm

    return meta


# === AWB grouping and certificate boundary detection ===

def _group_pages_by_awb(page_metas: list[CertPageMeta]) -> dict[str, list[CertPageMeta]]:
    """Group pages by normalized AWB. Pages without AWB inherit nearest neighbor's AWB."""
    groups: dict[str, list[CertPageMeta]] = {}
    last_awb = ''
    for meta in page_metas:
        awb = meta.awb or last_awb
        if not awb:
            # look ahead within next 5 pages for first AWB
            for m in page_metas[meta.idx:meta.idx + 6]:
                if m.awb:
                    awb = m.awb
                    break
        if awb:
            last_awb = awb
        if awb:
            groups.setdefault(awb, []).append(meta)
        else:
            groups.setdefault('unknown', []).append(meta)
    return groups


@dataclass
class CertBlock:
    cert_type: str = ''      # 'EUR1' | 'DIAN' | 'PHYTO' | 'FORM_A' | 'ECU_INV'
    cert_number: str = ''    # EUR1-B1059064 | DIAN-... | 526829 | ECU-11534802
    pages: list[CertPageMeta] = field(default_factory=list)
    invoice_pages: list[CertPageMeta] = field(default_factory=list)


def _detect_cert_blocks(pages: list[CertPageMeta]) -> list[CertBlock]:
    blocks: list[CertBlock] = []
    current: CertBlock | None = None
    for meta in pages:
        new_cert: str | None = None
        new_type: str = ''
        if meta.eur1_numbers:
            new_type = 'EUR1'
            new_cert = f'EUR1-{meta.eur1_numbers[0]}'
        elif meta.dian_numbers:
            new_type = 'DIAN'
            new_cert = f'DIAN-{meta.dian_numbers[0]}'
        elif meta.phyto_cert_number:
            new_type = 'PHYTO'
            new_cert = meta.phyto_cert_number
        elif meta.form_a_ref_number:
            new_type = 'FORM_A'
            new_cert = meta.form_a_ref_number
        else:
            # Ecuador commercial master-invoice pages have no EUR1/DIAN header;
            # treat each distinct master-invoice number as a certificate block.
            ecu_masters = _extract_ecu_master_invoices(meta.text)
            if ecu_masters:
                new_type = 'ECU_INV'
                new_cert = f'ECU-{ecu_masters[0]}'

        if new_cert and (not current or current.cert_number != new_cert):
            current = CertBlock(cert_type=new_type, cert_number=new_cert)
            blocks.append(current)

        if current is not None:
            current.pages.append(meta)
        # else: orphan pages before first cert — handled below

    # Attach orphan leading pages to first block if any
    if blocks:
        assigned_ids = {id(p) for b in blocks for p in b.pages}
        leading = [p for p in pages if id(p) not in assigned_ids]
        if leading:
            blocks[0].pages = leading + blocks[0].pages

    return blocks


def _attach_invoices(blocks: list[CertBlock], all_pages: list[CertPageMeta]) -> list[CertBlock]:
    """Attach invoice/proforma pages to the preceding certificate block.

    Strategy:
      1. If a cert page contains an invoice number, look for an invoice page
         with the same number within the same AWB group.
      2. Otherwise attach pages between this cert start and next cert start.
    """
    invoice_pages = [p for p in all_pages if p.invoice_numbers or p.proforma_numbers]

    for i, block in enumerate(blocks):
        cert_start = block.pages[0].idx if block.pages else 0
        next_start = blocks[i + 1].pages[0].idx if i + 1 < len(blocks) else 10**9
        cert_page_ids = {p.idx for p in block.pages}

        # 1. invoice number match
        cert_invoices: set[str] = set()
        for p in block.pages:
            cert_invoices.update(p.invoice_numbers)
            cert_invoices.update(p.proforma_numbers)

        matched = []
        for ip in invoice_pages:
            if cert_start <= ip.idx < next_start and ip.idx not in cert_page_ids:
                if any(n in cert_invoices for n in (ip.invoice_numbers + ip.proforma_numbers)):
                    matched.append(ip)

        # 2. positional fallback: invoice pages strictly between cert boundaries
        if not matched:
            matched = [ip for ip in invoice_pages if cert_start <= ip.idx < next_start and ip.idx not in cert_page_ids]

        block.invoice_pages = matched
    return blocks


def _is_page_effectively_empty(meta: CertPageMeta) -> bool:
    """A page is effectively empty if its text layer has fewer than 80 chars (scan-only)."""
    return len(meta.text.strip()) < 80


def _merge_empty_eur1_with_next(blocks: list[CertBlock]) -> list[CertBlock]:
    """Merge EUR1/DIAN blocks that have no invoice content with the following ECU_INV block.

    Ecuador PDF structure: [EUR.1 scan face page] → [commercial invoice pages].
    The EUR.1 scan creates an EUR1 block (possibly with some OCR text but no
    invoice data); the invoices create an ECU_INV block.
    This function absorbs the scan pages into the invoice block and promotes it to EUR1.
    """
    merged: list[CertBlock] = []
    i = 0
    while i < len(blocks):
        b = blocks[i]
        if b.cert_type in ('EUR1', 'DIAN') and i + 1 < len(blocks):
            # Check if this EUR1 block has no invoice-like content
            block_text = '\n'.join(p.text for p in b.pages)
            has_invoice_content = bool(re.search(
                r'MASTER\s+INVOICE|\bConcept\b|F\.\s*BXS|Invoice\s*No\.?\s*EC',
                block_text, re.IGNORECASE
            ))
            no_invoice = not b.invoice_pages
            next_block = blocks[i + 1]
            if not has_invoice_content and no_invoice and next_block.cert_type == 'ECU_INV':
                original_mi = next_block.cert_number.replace('ECU-', '', 1)
                next_block.pages = b.pages + next_block.pages
                next_block.cert_type = b.cert_type
                next_block.cert_number = b.cert_number
                next_block._ecu_master = original_mi  # preserve master invoice for items
                merged.append(next_block)
                i += 2
                continue
        merged.append(b)
        i += 1
    return merged


def _apply_eur1_by_master(blocks: list[CertBlock], eur1_by_master: dict[str, str]) -> list[CertBlock]:
    """Promote ECU_INV blocks to EUR1 when eur1_by_master has a mapping.

    After OCR pairs EUR1 numbers with master invoices, this replaces
    synthetic ECU-{master} cert_numbers with real EUR1 numbers.
    """
    if not eur1_by_master:
        return blocks
    for block in blocks:
        if block.cert_type == 'ECU_INV':
            mi = block.cert_number.replace('ECU-', '', 1)
            if mi in eur1_by_master:
                block.cert_type = 'EUR1'
                block.cert_number = eur1_by_master[mi]
                block._ecu_master = mi  # preserve master invoice for items
    return blocks


def _remove_empty_ocr_only_blocks(blocks: list[CertBlock]) -> list[CertBlock]:
    """Remove EUR.1/DIAN blocks that are based only on OCR hallucinations.

    A block is considered OCR-only garbage if:
      - cert_type is EUR1 or DIAN
      - all pages are effectively empty (scanned, no text layer)
      - no invoice/proforma pages attached
      - there is at least one other block in the list that has real content
    """
    has_content_block = any(
        not all(_is_page_effectively_empty(p) for p in b.pages)
        for b in blocks
    )
    kept: list[CertBlock] = []
    for b in blocks:
        if b.cert_type in ('EUR1', 'DIAN'):
            all_empty = all(_is_page_effectively_empty(p) for p in b.pages)
            no_invoice = not b.invoice_pages
            if all_empty and no_invoice and has_content_block:
                # discard — likely OCR hallucination on scan-only pages
                continue
        kept.append(b)
    return kept


def _redistribute_orphan_pages(blocks: list[CertBlock], all_pages: list[CertPageMeta]) -> list[CertBlock]:
    """Reattach pages that became orphaned after block removal to the nearest remaining block."""
    non_empty_blocks = [b for b in blocks if b.pages]
    if not non_empty_blocks:
        return blocks
    assigned_ids = {id(p) for b in blocks for p in b.pages + b.invoice_pages}
    orphans = [p for p in all_pages if id(p) not in assigned_ids]
    for p in orphans:
        # find nearest block by cert start page index
        nearest = min(non_empty_blocks, key=lambda b: abs(b.pages[0].idx - p.idx))
        if p.idx < nearest.pages[0].idx:
            nearest.pages.insert(0, p)
        else:
            nearest.pages.append(p)
    return blocks


def _items_from_block(block: CertBlock) -> list[CertItem]:
    """Parse invoice table pages inside a certificate block."""
    items: list[CertItem] = []
    for page in block.pages + block.invoice_pages:
        # ETHAM table
        if page.has_etham_table:
            for line in page.text.split('\n'):
                m = _parse_etham_row(line)
                if not m:
                    continue
                flower_raw = m.group('flower').upper()
                flower = _KE_FLOWER_MAP.get(flower_raw, flower_raw)
                try:
                    items.append(CertItem(
                        flower=flower,
                        farm=m.group('farm').upper(),
                        boxes=float(m.group('boxes')),
                        stems=int(m.group('stems')),
                        cert_number=block.cert_number,
                        invoice_nr=page.etham_invoice or (page.invoice_numbers[0] if page.invoice_numbers else ''),
                    ))
                except (ValueError, TypeError):
                    pass
        # Milele table
        elif page.has_milele_table:
            for line in page.text.split('\n'):
                m = _KE_INV_TABLE_LINE.match(line.strip())
                if not m:
                    continue
                flower_raw = m.group('flower').upper()
                flower = _KE_FLOWER_MAP.get(flower_raw, flower_raw)
                grade = (m.group('grade') or '').strip()
                try:
                    items.append(CertItem(
                        flower=flower,
                        farm=grade,
                        boxes=float(m.group('boxes')),
                        stems=int(m.group('stems')),
                        cert_number=block.cert_number,
                        invoice_nr=page.invoice_numbers[0] if page.invoice_numbers else '',
                    ))
                except (ValueError, TypeError):
                    pass

    # Ecuador master-invoice tables (PyMuPDF tables preferred, then inline/pipe)
    block_text = '\n'.join(p.text for p in block.pages + block.invoice_pages)
    is_ecu_block = block.cert_type == 'ECU_INV' or getattr(block, '_ecu_master', '') or (
        'MASTER INVOICE' in block_text and ('Concept' in block_text or 'F. BXS' in block_text)
    )
    if is_ecu_block:
        ecu_items: list[CertItem] = []
        for page in block.pages + block.invoice_pages:
            ecu_items.extend(_parse_ecu_page(page))
        _ecu_master = getattr(block, '_ecu_master', '')
        # Наследуем номер сертификата/мастер-инвойса блока, если позиция его не заполнила
        for it in ecu_items:
            if not it.cert_number:
                it.cert_number = block.cert_number
            if (block.cert_type == 'ECU_INV' or _ecu_master) and not it.master_invoice:
                it.master_invoice = _ecu_master or block.cert_number.replace('ECU-', '', 1)
        if block.cert_type == 'ECU_INV' or _ecu_master:
            items = ecu_items
        else:
            # Avoid duplicating items already found in ETHAM/Milele tables
            seen_keys = {(it.master_invoice, it.invoice_nr, it.flower, it.boxes, it.stems) for it in items}
            for it in ecu_items:
                key = (it.master_invoice, it.invoice_nr, it.flower, it.boxes, it.stems)
                if key not in seen_keys:
                    seen_keys.add(key)
                    items.append(it)

    return items


# Kenya ULTRA FLO Form A aggregate on cert face page:
# "3 BOXES ROSES 2400 STEMS HS CODE 060311"
_KE_FORM_A_AGG = re.compile(
    r'(?P<boxes>\d+)\s+BOXES\s+(?P<flower>ROSES?|CARNATIONS?|ALSTROEMERIA|HYPERICUM|CHRYSANTHEMUM)\s+(?P<stems>\d+)\s+STEMS',
    re.IGNORECASE,
)
# Separate "N BOXES" pattern (when boxes and stems are in different columns)
_KE_FORM_A_BOXES = re.compile(
    r'(?P<boxes>\d+)\s+BOXES',
    re.IGNORECASE,
)


def _form_a_cert_aggregate(block: CertBlock) -> tuple[float, int, str]:
    """Extract boxes, stems and flower from a Kenya ULTRA FLO Form A cert page.

    Form A face page contains a line like '3 BOXES ROSES 2400 STEMS'.
    In table layout, '3 BOXES' and 'ROSES 2400 STEMS' may be on separate
    lines (different columns).  We search for them independently.
    Returns (boxes, stems, flower), defaulting to (0, 0, 'ROSA').
    """
    cert_text = '\n'.join(p.text for p in block.pages)
    # Prefer the combined boxes+stems line if present.
    for line in cert_text.split('\n'):
        m = _KE_FORM_A_AGG.search(line.strip())
        if m:
            try:
                flower = _KE_FLOWER_MAP.get(
                    m.group('flower').upper(), m.group('flower').upper()
                )
                return float(m.group('boxes')), int(m.group('stems')), flower
            except (ValueError, TypeError):
                pass
    # Separate search: look for "N BOXES" and "FLOWER N STEMS" on different lines
    boxes_val = 0.0
    stems_val = 0
    flower_val = 'ROSA'
    for line in cert_text.split('\n'):
        if not boxes_val:
            m = _KE_FORM_A_BOXES.search(line.strip())
            if m:
                try:
                    boxes_val = float(m.group('boxes'))
                except (ValueError, TypeError):
                    pass
        if not stems_val:
            m = _KE_FLOWER_LINE.match(line.strip())
            if m:
                try:
                    flower_val = _KE_FLOWER_MAP.get(
                        m.group('flower').upper(), m.group('flower').upper()
                    )
                    stems_val = int(m.group('stems'))
                except (ValueError, TypeError):
                    pass
        if boxes_val and stems_val:
            break
    if boxes_val or stems_val:
        return boxes_val, stems_val, flower_val
    return 0.0, 0, 'ROSA'


def _render_page_image(pdf_path: str, page_idx: int, dpi: int = 300) -> Optional["Image.Image"]:
    """Render a single PDF page to a PIL RGB image."""
    if fitz is None or Image is None:
        return None
    try:
        doc = fitz.open(pdf_path)
        page = doc.load_page(page_idx)
        scale = dpi / 72.0
        pix = page.get_pixmap(matrix=fitz.Matrix(scale, scale), alpha=False)
        img = Image.frombytes('RGB', (pix.width, pix.height), pix.samples)
        doc.close()
        return img
    except Exception:
        return None


def _paddle_text_boxes(res) -> list[tuple[str, int, int]]:
    """Normalize PaddleOCR result to ``[(text, x_center, y_center), ...]``."""
    if isinstance(res, list):
        if not res:
            return []
        # ``ocr.predict`` returns a list of dicts; ``ocr.ocr`` returns a list of lines.
        if isinstance(res[0], dict):
            res = res[0]
        else:
            out: list[tuple[str, int, int]] = []
            for line in res:
                if not line:
                    continue
                for item in line:
                    try:
                        box, (txt, _score) = item
                        xs = [p[0] for p in box]
                        ys = [p[1] for p in box]
                        out.append((str(txt), int(sum(xs) / len(xs)), int(sum(ys) / len(ys))))
                    except Exception:
                        continue
            return out
    if not isinstance(res, dict):
        return []
    texts = res.get('rec_texts') or []
    boxes = res.get('rec_boxes')
    if boxes is None:
        return []
    out: list[tuple[str, int, int]] = []
    for i, t in enumerate(texts):
        if i >= len(boxes):
            break
        b = boxes[i]
        if hasattr(b, 'tolist'):
            b = b.tolist()
        try:
            x1, y1, x2, y2 = map(int, b)
        except Exception:
            continue
        out.append((str(t), (x1 + x2) // 2, (y1 + y2) // 2))
    return out


def _ultraflo_boxes_from_paddle(img: "Image.Image") -> float:
    """Read the BOXES column from an ULTRA FLO commercial-invoice table.

    Uses PaddleOCR bounding boxes to locate the 'BOXES' header, then picks the
    first numeric cell below it in the same column.
    """
    if _get_paddle is None:
        return 0.0
    try:
        import numpy as np
    except Exception:
        return 0.0
    ocr = _get_paddle()
    if ocr is None:
        return 0.0
    try:
        arr = np.array(img.convert('RGB'))
        res = ocr.predict(arr)
    except AttributeError:
        try:
            res = ocr.ocr(np.array(img.convert('RGB')))
        except Exception:
            return 0.0
    except Exception:
        return 0.0

    items = _paddle_text_boxes(res)
    header_keywords = {
        'client': 'client', 'variety': 'variety', 'farm': 'farm',
        'length': 'length', 'cm': 'cm', 'boxes': 'boxes', 'stems': 'stems',
        'unit': 'unit', 'price': 'price', 'total': 'total', 'eur': 'eur',
    }
    col_x: dict[str, list[int]] = {}
    header_y: Optional[int] = None
    for t, cx, cy in items:
        c = re.sub(r'[^a-z0-9]', '', t.lower())
        for kw, label in header_keywords.items():
            if kw in c:
                col_x.setdefault(label, []).append(cx)
                if label == 'boxes' and header_y is None:
                    header_y = cy
                break
    if 'boxes' not in col_x or header_y is None:
        return 0.0
    boxes_x = float(np.median(col_x['boxes']))
    candidates: list[tuple[int, int]] = []
    for t, cx, cy in items:
        if not re.match(r'^\d+$', t.strip()):
            continue
        if abs(cx - boxes_x) > 80:
            continue
        if cy <= header_y + 10:
            continue
        candidates.append((cy, int(t.strip())))
    if not candidates:
        return 0.0
    candidates.sort()
    return float(candidates[0][1])


def _form_a_invoice_boxes(block: CertBlock, pdf_path: str = '') -> float:
    """Sum boxes from ULTRA FLO invoice pages.

    Tries common Kenya invoice table patterns (Milele-like and ULTRA FLO-like).
    Falls back to PaddleOCR on the invoice page image if the Tesseract-based
    text pipeline cannot read the boxes column.
    """
    boxes = 0.0
    # Invoice pages may be attached to ``block.invoice_pages`` or remain in
    # ``block.pages`` for ULTRA FLO Form A scans.
    for page in block.pages + block.invoice_pages:
        # Milele-style: Roses 35-80 2 195 390
        for line in page.text.split('\n'):
            m = _KE_INV_TABLE_LINE.match(line.strip())
            if m:
                try:
                    boxes += float(m.group('boxes'))
                except (ValueError, TypeError):
                    pass
        # ULTRA FLO-style total row: "Total Flowers 3 2400 120"
        if boxes == 0:
            m = re.search(
                r'Total\s+Flowers\s+(?P<boxes>\d+)\s+(?P<stems>\d+)',
                page.text, re.IGNORECASE,
            )
            if m:
                try:
                    boxes += float(m.group('boxes'))
                except (ValueError, TypeError):
                    pass
    # PaddleOCR fallback for scan-only ULTRA FLO invoice tables.
    # ULTRA FLO invoice pages are sometimes kept inside ``block.pages`` (the
    # cert face page and the invoice are not split).  Skip cert face pages to
    # avoid burning Paddle time on pages that cannot contain the invoice table.
    if boxes == 0 and pdf_path and fitz is not None and Image is not None and _get_paddle is not None:
        for page in block.pages + block.invoice_pages:
            text = page.text.upper()
            if 'COMMERCIAL INVOICE' not in text and 'INVOICE:' not in text and 'BOXES' not in text:
                continue
            img = _render_page_image(pdf_path, page.idx, dpi=300)
            if img is None:
                continue
            # Crop to the commercial-invoice table area before PaddleOCR.
            # ULTRA FLO invoices keep the table in a fixed upper-middle band.
            w, h = img.size
            table_img = img.crop((
                min(w, int(35 * 300 / 72)),
                min(h, int(190 * 300 / 72)),
                min(w, int(565 * 300 / 72)),
                min(h, int(300 * 300 / 72)),
            ))
            paddle_boxes = _ultraflo_boxes_from_paddle(table_img)
            if paddle_boxes:
                boxes = paddle_boxes
                break
    return boxes


# === Публичный API ===

def parse_certificate(path: str) -> CertificateData:
    """Распарсить сертификат. При ошибке возвращает CertificateData с пустыми полями."""
    d = CertificateData(source=os.path.basename(path))
    d.country = _detect_country(d.source)

    if pdfplumber is None:
        return d

    # === OCR: build per-page lookup dict (before PDF open) ===
    # Вызываем OCR один раз; результаты используем и для нового ocr_by_page,
    # и для старого заполнения d.eur1_numbers / d.eur1_by_master / d.cert_number.
    _skip_ocr = os.environ.get('SKIP_OCR', '').strip() in ('1', 'true', 'yes', 'on')
    ocr_by_page: dict[int, dict] = {}
    ocr_res = None
    dian_res = None

    # Kenya ETHAM files are mostly raster scans; the slow PaddleOCR-based
    # ocr_eur1_numbers() takes 5-8 minutes on them.  Use the fast Tesseract
    # fallback inside the Kenya branch instead.
    _use_paddle = d.country != 'Kenya'

    if _use_paddle and ocr_eur1_numbers is not None and is_ocr_available() and not _skip_ocr:
        try:
            ocr_res = ocr_eur1_numbers(path)
            for pn, nr in zip(ocr_res.page_numbers, ocr_res.numbers):
                ocr_by_page.setdefault(pn, {}).setdefault('eur1', []).append(nr)
            if hasattr(ocr_res, 'awb_by_page') and ocr_res.awb_by_page:  # type: ignore[attr-defined]
                for pn, awb in ocr_res.awb_by_page.items():  # type: ignore[attr-defined]
                    ocr_by_page.setdefault(pn, {})['awb'] = awb
        except Exception:
            ocr_res = None

    if _use_paddle and ocr_dian_numbers is not None and is_ocr_available() and not _skip_ocr:
        try:
            dian_res = ocr_dian_numbers(path)
            for pn, nr in zip(dian_res.page_numbers, dian_res.numbers):
                ocr_by_page.setdefault(pn, {}).setdefault('dian', []).append(nr)
        except Exception:
            dian_res = None

    # === Single PDF open: extract text AND build page metadata ===
    try:
        with pdfplumber.open(path) as pdf:
            d.raw_pages = len(pdf.pages)
            all_text = ""
            page_texts: list[str] = []   # постраничный текст — для позиционной связки DIAN
            page_metas: list[CertPageMeta] = []
            for idx, page in enumerate(pdf.pages):
                text = page.extract_text() or ""
                page_texts.append(text)
                all_text += text + "\n"
                # Build page metadata with OCR enrichment (reuse already extracted text)
                meta = _extract_page_meta(page, idx, ocr_by_page, text=text)
                page_metas.append(meta)
    except Exception:
        return d

    # Determine country also from text layer (for files without country prefix).
    if not d.country:
        d.country = _detect_country_from_text(all_text)

    # === Fast Tesseract fallback for Kenya scan pages ===
    # ETHAM invoice pages are often raster images with no text layer.  OCR them
    # in bulk with Tesseract (~0.8 s/page) and merge the text back into the
    # metadata pipeline so AWB/EUR1/tables are detected the same way as for
    # text-layer PDFs.
    if not _skip_ocr and d.country == 'Kenya' and pytesseract is not None and fitz is not None:
        scan_indices = [
            idx for idx, meta in enumerate(page_metas)
            if len(meta.text.strip()) < 80
        ]
        if scan_indices:
            try:
                ocr_texts = _ocr_pages(path, scan_indices)
                for idx, new_text in ocr_texts.items():
                    if not new_text.strip():
                        continue
                    page_metas[idx].text = new_text
                    page_texts[idx] = new_text
                    # Re-extract metadata from the OCR text.
                    page_metas[idx] = _extract_page_meta(
                        None, idx, ocr_by_page, text=new_text,
                    )
                    # Rebuild all_text with the OCR text.
                    all_text = "\n".join(page_texts)

                # Re-OCR Form A pages with suspicious Reference No. at higher DPI.
                # Tesseract at 300 DPI sometimes reads '5' as '9' in red stamps.
                retry_indices = [
                    idx for idx in scan_indices
                    if page_metas[idx].form_a_ref_number
                    and page_metas[idx].form_a_ref_number.startswith('9')
                    and len(page_metas[idx].form_a_ref_number) == 6
                ]
                if retry_indices:
                    try:
                        retry_texts = _ocr_pages(path, retry_indices, dpi=400)
                        for idx, new_text in retry_texts.items():
                            if not new_text.strip():
                                continue
                            page_metas[idx].text = new_text
                            page_texts[idx] = new_text
                            page_metas[idx] = _extract_page_meta(
                                None, idx, ocr_by_page, text=new_text,
                            )
                            all_text = "\n".join(page_texts)
                    except Exception:
                        pass
            except Exception:
                pass

    # === Fast Tesseract fallback for Ecuador scan pages ===
    # Some Ecuador certificates include raster EUR.1 faces and commercial
    # master-invoice pages with no text layer.  OCR those pages and extract
    # summary items so every master invoice gets boxes/stems.
    if not _skip_ocr and d.country == 'Ecuador' and pytesseract is not None and fitz is not None:
        _valid_masters_ecu = set(ocr_res.eur1_by_master.keys()) if ocr_res else set()
        _ocr_master_by_page_ecu = {
            (p - 1): m for p, m in (getattr(ocr_res, 'master_by_page', {}) or {}).items()
            if m in _valid_masters_ecu
        }
        _scan_indices_ecu = [
            idx for idx, meta in enumerate(page_metas)
            if len(meta.text.strip()) < 80
        ]
        if _scan_indices_ecu:
            try:
                _ocr_texts_ecu = _ocr_pages(path, _scan_indices_ecu)
                # Propagate master across pages the same way the item loop does.
                _page_masters: dict[int, str] = {}
                _current_master_ecu = ''
                for meta in page_metas:
                    tm = _ecu_master_from_page_text(meta.text)
                    if tm in _valid_masters_ecu:
                        _current_master_ecu = tm
                    elif meta.idx in _ocr_master_by_page_ecu:
                        _current_master_ecu = _ocr_master_by_page_ecu[meta.idx]
                    _page_masters[meta.idx] = tm or _current_master_ecu

                _scan_seen: set[tuple] = set()
                for idx, new_text in _ocr_texts_ecu.items():
                    if not new_text.strip():
                        continue
                    master = _page_masters.get(idx, '')
                    if not master:
                        continue
                    _scan_items = _parse_ecu_scan_summary(new_text, master)

                    # For commercial-invoice scan pages the per-row format often
                    # has the price/total split across lines, causing INVOICE_LINE
                    # to miss most rows.  Re-OCR with PSM 6 (uniform block) which
                    # keeps the table compact and exposes the grand-total line
                    # "17.25 10105 4160.40".  Use the PSM-6 result when it gives
                    # a better (higher) stem total than PSM-4.
                    # Only do the extra OCR pass for pages that are known master-
                    # invoice pages (from the seeded cache) AND look like a commercial
                    # invoice (contain COMMERCIAL header in PSM-4 text) —
                    # NOT for EUR.1 certificate face pages.
                    _psm6_totals: tuple[float, int] | None = None
                    _psm4_stems = sum(it.stems for it in _scan_items)
                    _is_invoice_page = (
                        idx in _ocr_master_by_page_ecu
                        and 'COMMERCIAL' in new_text.upper()
                    )
                    if _is_invoice_page:
                        try:
                            _psm6_text = _ocr_pages(path, [idx], psm=6).get(idx, '')
                            if _psm6_text.strip():
                                _psm6_totals = _extract_ecu_scan_totals(_psm6_text)
                                # Also try per-row parse on PSM-6 text
                                _psm6_items = _parse_ecu_scan_summary(_psm6_text, master)
                                _psm6_stems = sum(it.stems for it in _psm6_items)
                                if _psm6_stems > _psm4_stems:
                                    _scan_items = _psm6_items
                                    _psm4_stems = _psm6_stems
                        except Exception:
                            _psm6_text = ''

                    if _scan_items:
                        # If PSM-6 gives a larger aggregate total than the sum of
                        # per-row items, replace the item list with a single
                        # aggregate entry so boxes/stems are accurate.
                        if _psm6_totals:
                            _total_boxes, _total_stems = _psm6_totals
                            if _total_stems > _psm4_stems:
                                # Keep per-row variety breakdown but fix totals
                                # by scaling — or just emit a single aggregate.
                                _scan_items = [CertItem(
                                    flower=_scan_items[0].flower if len(_scan_items) == 1 else 'FLOWERS',
                                    boxes=_total_boxes,
                                    stems=_total_stems,
                                    hs_code=_scan_items[0].hs_code if len(_scan_items) == 1 else '',
                                    invoice_nr=master,
                                    master_invoice=master,
                                )]
                        for it in _scan_items:
                            # Ensure invoice_nr is populated for scan-derived items
                            if not it.invoice_nr:
                                it.invoice_nr = master
                            key = (it.master_invoice, it.invoice_nr, it.flower, it.boxes, it.stems)
                            if key in _scan_seen:
                                continue
                            _scan_seen.add(key)
                            d.items.append(it)
                            if it.master_invoice and it.master_invoice not in d.master_invoices:
                                d.master_invoices.append(it.master_invoice)
                    else:
                        # No per-row items — use PSM-6 aggregate totals if available,
                        # otherwise fall back to PSM-4 totals.
                        totals = _psm6_totals or _extract_ecu_scan_totals(new_text)
                        if totals:
                            boxes, stems = totals
                            it = CertItem(
                                flower='FLOWERS',
                                boxes=boxes,
                                stems=stems,
                                invoice_nr=master,
                                master_invoice=master,
                            )
                            key = (it.master_invoice, it.invoice_nr, it.flower, it.boxes, it.stems)
                            if key not in _scan_seen:
                                _scan_seen.add(key)
                                d.items.append(it)
                                if master not in d.master_invoices:
                                    d.master_invoices.append(master)
            except Exception:
                pass

    # === Ecuador: extract tables with PyMuPDF for cleaner farm/client split ===
    # pdfplumber merges table cells into one line; PyMuPDF find_tables() keeps
    # columns separate and gives us the correct farm and marking.
    _is_ecu = d.country == 'Ecuador' or 'MASTER INVOICE' in all_text.upper()
    if _is_ecu and fitz is not None:
        try:
            ecu_tables_by_page = _extract_ecu_tables_fitz(path)
            for idx, tables in ecu_tables_by_page.items():
                if 0 <= idx < len(page_metas):
                    page_metas[idx].ecu_table_rows = [
                        row for table in tables for row in table
                    ]
        except Exception:
            pass

    # Если текстовый слой страницы не содержит master invoice (номер — картинкой),
    # используем карту страница → master из OCR-связки EUR1 ↔ master.
    # Master-страница обычно предшествует страницам с таблицей товаров, поэтому
    # распространяем найденный master на следующие страницы до появления нового.
    _valid_masters: set[str] = set()
    ocr_master_by_page: dict[int, str] = {}
    if ocr_res is not None:
        _valid_masters = set(ocr_res.eur1_by_master.keys())
        # OCR-страницы нумеруются с 1, page_metas — с 0.
        ocr_master_by_page = {
            (p - 1): m for p, m in (getattr(ocr_res, 'master_by_page', {}) or {}).items()
            if m in _valid_masters
        }

    # Шапочные поля (эквадорский формат)
    for nr in _extract_ecu_master_invoices(all_text):
        if nr not in d.master_invoices:
            d.master_invoices.append(nr)

    awb_m = _ECU_AWB.search(all_text)
    if awb_m:
        d.awb = _normalize_awb(awb_m.group(1))

    # Собираем ВСЕ AWB-ссылки из текста (для построения AWB→country map
    # в manifest_builder: last4 цифр → префикс папки → страна)
    for m in _GENERAL_AWB.finditer(all_text):
        awb_raw = re.sub(r'\s+', ' ', m.group(1))
        if awb_raw not in d.all_awbs:
            d.all_awbs.append(awb_raw)
    if not d.awb and d.all_awbs:
        d.awb = d.all_awbs[0]

    date_m = _ECU_DATE.search(all_text)
    if date_m:
        d.date = date_m.group(1)

    exp_m = _ECU_EXPORTER.search(all_text)
    if exp_m:
        d.exporter = exp_m.group(1).strip()

    # Сертификатные номера — несколько паттернов для разных форматов EUR1
    for pat in _EUR1_PATTERNS:
        for m in pat.finditer(all_text):
            nr = _normalize_eur1(m.group(1).strip())
            # Tesseract on Kenya scans reads leading "1" as "4".
            if nr and d.country == 'Kenya' and len(nr) == 8 and nr.startswith('B4') and nr[1:].isdigit():
                nr = 'B1' + nr[2:]
            if nr and nr not in d.eur1_numbers:
                d.eur1_numbers.append(nr)
    # DIAN-номера из текстового слоя (если есть)
    for m in _DIAN_PATTERN.finditer(all_text):
        nr = m.group(1).strip()
        if nr and nr not in d.dian_numbers:
            d.dian_numbers.append(nr)
    if d.eur1_numbers:
        d.cert_number = f'EUR1-{d.eur1_numbers[0]}'
    elif d.dian_numbers:
        d.cert_number = f'DIAN-{d.dian_numbers[0]}'

    # Consignee — простая эвристика: строка после "Consignee:"
    cons_m = re.search(r'Consignee:\s*([^\n]+)', all_text)
    if cons_m:
        d.consignee = re.sub(r'\s+AWB:.*$', '', cons_m.group(1)).strip()

    # === Colombia proforma header ===
    # PROFORMA IFC-NNNNNN — номер проформы (= инвойс для Colombia)
    prof_m = _COL_PROFORMA_NR.search(all_text)
    _col_proforma_nr = ''
    if prof_m:
        _col_proforma_nr = prof_m.group(1).upper().replace(' ', '')
        if _col_proforma_nr not in d.master_invoices:
            d.master_invoices.append(_col_proforma_nr)
    if not d.date:
        m = _COL_DATE_RE.search(all_text)
        if m:
            d.date = m.group(1)
    if not d.exporter:
        m = _COL_EXPORTER_2.search(all_text)
        if m:
            d.exporter = re.sub(r'\s+', ' ', m.group(1)).strip()

    # === Colombia/Kenya-fallback для шапки ===
    # Если Ecuador-регекспы ничего не нашли — пробуем Colombia-формат
    if d.country in ('Colombia', 'Kenya') or not d.awb:
        if not d.awb:
            m = _COL_AWB.search(all_text)
            if m:
                d.awb = m.group(1)
        if not d.exporter:
            m = _COL_EXPORTER.search(all_text)
            if m:
                d.exporter = re.sub(r'\s+', ' ', m.group(1)).strip()
        if not d.date:
            # Приоритет — английская дата с городом (BOGOTA, July 01, 2026 → берём дату)
            m = _COL_DATE_EN.search(all_text)
            if m and m.group(1).split()[0] in (
                'January','February','March','April','May','June','July',
                'August','September','October','November','December'):
                d.date = m.group(1)
            else:
                m = _COL_DATE_ISO.search(all_text)
                if m:
                    d.date = m.group(1)
        if not d.consignee:
            m = re.search(r'^2\.\s*Consignee[^\n]*\n([A-Z][A-Z\.\s&,\-\d]+?)\s*\n',
                          all_text, re.MULTILINE)
            if m:
                d.consignee = re.sub(r'\s+', ' ', m.group(1)).strip()

    # === Позиции (Ecuador) ===
    if d.country == 'Ecuador' or (d.country == '' and 'MASTER INVOICE' in all_text.upper()):
        seen: set[tuple] = set()
        _current_ocr_master = ''
        for meta in page_metas:
            _txt_master = _ecu_master_from_page_text(meta.text)
            if _txt_master in _valid_masters:
                _current_ocr_master = _txt_master
            elif meta.idx in ocr_master_by_page:
                _current_ocr_master = ocr_master_by_page[meta.idx]
            _page_master = _txt_master or _current_ocr_master
            for item in _parse_ecu_page(meta, _page_master):
                key = (item.master_invoice, item.invoice_nr, item.flower, item.boxes, item.stems)
                if key in seen:
                    continue
                seen.add(key)
                d.items.append(item)
                if item.master_invoice and item.master_invoice not in d.master_invoices:
                    d.master_invoices.append(item.master_invoice)

        # Prefer EUR.1 face-page aggregates (authoritative certificate totals)
        # and merge them with commercial-invoice rows (boxes, farm, invoice#).
        d.items = _merge_ecu_face_and_invoice(d.items)

    # === Colombia proforma table parser ===
    if not d.items and (d.country == 'Colombia' or _col_proforma_nr):
        # Парсим строки таблицы: «26 6,5 CARNATIONS T C.O 10400 0,10 1040,00»
        _col_flower_map = {
            'CARNATIONS': 'DIANTHUS', 'CARNATION': 'DIANTHUS',
            'ROSES': 'ROSA', 'ROSE': 'ROSA',
            'CRISANTEMO': 'CHRYSANTHEMUM', 'CHRYSANTHEMUM': 'CHRYSANTHEMUM',
            'POMPON': 'CHRYSANTHEMUM',
        }
        for line in all_text.split('\n'):
            m = _COL_TABLE_LINE.match(line.strip())
            if not m:
                continue
            flower_raw = m.group('flower').upper()
            flower = _col_flower_map.get(flower_raw, flower_raw)
            if flower not in _FLOWERS and flower_raw not in _col_flower_map:
                continue
            try:
                pieces = float(m.group(1))
                stems = int(m.group('stems'))
                d.items.append(CertItem(
                    flower=flower,
                    boxes=pieces,
                    stems=stems,
                    master_invoice=_col_proforma_nr,
                    invoice_nr=_col_proforma_nr,
                ))
            except (ValueError, TypeError):
                continue
        # Если строки не распарсились — берём total boxes из «TOTAL BOXES PIEZAS NN,NN»
        if not d.items:
            tot_m = _COL_TOTAL_BOXES.search(all_text)
            if tot_m:
                try:
                    d.total_boxes = float(tot_m.group(1).replace(',', '.'))
                except (ValueError, TypeError):
                    pass

    # === Kenya invoice parser ===
    if not d.items and d.country == 'Kenya':

        # --- Приоритет 1: таблица VARIETY в Master Invoice (Milele-формат) ---
        # Каждая пара страниц: сертификат (cert_nr + invoice_nr + итог) + инвойс (таблица)
        # Алгоритм: двигаемся по тексту, запоминаем cert_nr и invoice_nr,
        # затем парсим строки таблицы _KE_INV_TABLE_LINE.
        _lines = all_text.split('\n')
        _cur_cert = ''
        _cur_inv  = ''
        _in_variety_table = False
        for line in _lines:
            s = line.strip()
            if not s:
                continue
            # Номер бланка сертификата стоит первым на своей строке (6 цифр)
            cert_nr_m = _KE_CERT_NR_LINE.match(s)
            if cert_nr_m:
                _cur_cert = cert_nr_m.group(1)
                _in_variety_table = False
                continue
            # Номер инвойса
            inv_m = _KE_INVOICE_NR.search(s)
            if inv_m:
                _ke_inv_candidate = inv_m.group(1)
                # Не путаем номер сертификата с номером инвойса
                if _ke_inv_candidate != _cur_cert:
                    _cur_inv = _ke_inv_candidate
                    if _cur_inv not in d.master_invoices:
                        d.master_invoices.append(_cur_inv)
                _in_variety_table = False
                continue
            # Заголовок таблицы инвойса — включаем режим парсинга строк
            if re.search(r'VARIETY.*BOXES.*STEMS|P\.?\s*RATE.*TT\s*STEMS', s, re.IGNORECASE):
                _in_variety_table = True
                continue
            # Строка таблицы
            m = _KE_INV_TABLE_LINE.match(s)
            if m:
                flower_raw = m.group('flower').upper()
                flower = _KE_FLOWER_MAP.get(flower_raw, flower_raw)
                grade = (m.group('grade') or '').strip()
                try:
                    boxes = float(m.group('boxes'))
                    stems = int(m.group('stems'))
                    d.items.append(CertItem(
                        flower=flower,
                        farm=grade,            # ростовка храним в farm до отдельного поля
                        boxes=boxes,
                        stems=stems,
                        cert_number=_cur_cert,
                        master_invoice=_cur_inv,
                        invoice_nr=_cur_inv,
                    ))
                except (ValueError, TypeError):
                    pass
                continue

        # --- Приоритет 2: «ROSES 390 STEMS» в тексте сертификата (старый формат) ---
        if not d.items:
            _ke_inv = ''
            ke_inv_m = _KE_INVOICE_NR.search(all_text)
            if ke_inv_m:
                _ke_inv = ke_inv_m.group(1)
                if _ke_inv not in d.master_invoices:
                    d.master_invoices.append(_ke_inv)
            for line in all_text.split('\n'):
                m = _KE_FLOWER_LINE.match(line.strip())
                if not m:
                    continue
                flower_raw = m.group('flower').upper()
                flower = _KE_FLOWER_MAP.get(flower_raw, flower_raw)
                try:
                    stems = int(m.group('stems'))
                    d.items.append(CertItem(
                        flower=flower,
                        stems=stems,
                        master_invoice=_ke_inv,
                        invoice_nr=_ke_inv,
                    ))
                except (ValueError, TypeError):
                    continue

    # === Старый fallback: PCS-агрегат (если новые парсеры не нашли) ===
    if not d.items:
        # `1 21 PCS FRESH CUT FLOWERS ALSTROEMERIA 3360.0000 PRO 457`
        agg = re.search(
            r'\b(\d{1,4})\s+PCS\s+.*?(ROSA|ALSTROEMERIA|GYPSOPHILA|HYPERICUM|'
            r'CLAVEL|DIANTHUS|MATHIOLA|CHRYSANTHEMUM|LILIUM|ORNITOGALO|EUCALYPTUS)'
            r'\s+(\d+(?:\.\d+)?)',
            all_text, re.IGNORECASE
        )
        if agg:
            try:
                d.total_boxes = float(agg.group(1))
                d.total_stems = int(float(agg.group(3)))
                d.items.append(CertItem(
                    flower=agg.group(2).upper(),
                    boxes=d.total_boxes,
                    stems=d.total_stems,
                ))
            except (ValueError, TypeError):
                pass

    # Агрегаты (после всех парсеров — Colombia/Kenya добавляют items позже)
    if not d.total_boxes:
        d.total_boxes = sum(it.boxes for it in d.items)
    if not d.total_stems:
        d.total_stems = sum(it.stems for it in d.items)

    # === Backward-compatible EUR1 population from OCR ===
    # Sertif Ec/Co/Ke содержат сканы EUR1-сертификатов; на лицевой стороне — номер EUR1,
    # на обороте — MASTER INVOICE N° NNNNNN. OCR-модуль строит пары.
    if ocr_res is not None:
        try:
            for nr in ocr_res.numbers:
                nr_norm = _normalize_eur1(nr)
                if nr_norm and nr_norm not in d.eur1_numbers:
                    d.eur1_numbers.append(nr_norm)
            # Копируем карту пар и попутно расширяем master_invoices теми, что нашёл OCR.
            # Значения храним с префиксом «EUR1-» — florunner_writer пишет их в H как есть.
            for mi, eur1 in ocr_res.eur1_by_master.items():
                eur1_clean = eur1.upper().lstrip('EUR1').lstrip('-').strip()
                d.eur1_by_master[mi] = f'EUR1-{eur1_clean}'
                if mi not in d.master_invoices:
                    d.master_invoices.append(mi)
            # Если cert_number ещё не найден по тексту — берём первый OCR-номер
            if not d.cert_number and d.eur1_numbers:
                d.cert_number = f'EUR1-{d.eur1_numbers[0]}'

            # === Kenya: позиционная связка EUR.1 → cert_number ===
            # PDF-структура: [N стр. сканов EUR.1] → [серт. 1] → [инв. 1] → [серт. 2] → ...
            # EUR1-номера OCR даёт в порядке страниц (ранняя страница = первый EUR.1).
            # Сертификаты (по номеру страницы появления) сортируем аналогично.
            # Дополнительная связка (если количество EUR1 == количество кенийских сертификатов).
            if d.country == 'Kenya' and ocr_res.numbers and d.items:
                # Собираем уникальные cert_number из items в порядке появления
                _seen_certs: list[str] = []
                for _it in d.items:
                    if _it.cert_number and _it.cert_number not in _seen_certs:
                        _seen_certs.append(_it.cert_number)
                # EUR1-номера в порядке страниц (ocr_res.page_numbers уже отсортированы)
                _eur1_ordered = list(ocr_res.numbers)
                # Позиционная связка: cert[i] → eur1[i]
                for _i, _cert in enumerate(_seen_certs):
                    if _i < len(_eur1_ordered):
                        d.eur1_by_cert[_cert] = _eur1_ordered[_i]
                        # Также обновляем eur1_by_master через invoice_nr — если нет уже
                        _inv = d.items[_i].invoice_nr if _i < len(d.items) else ''
                        if _inv and _inv not in d.eur1_by_master:
                            d.eur1_by_master[_inv] = f'EUR1-{_eur1_ordered[_i]}'

            # === Проверка AWB: EUR.1 AWB ≡ Master Invoice AWB ===
            # OCR читает AWB с оборота EUR.1-скана;
            # Master Invoice AWB — из текстового слоя (d.awb).
            # Если AWB на EUR.1 есть, но отличается — сохраняем в awb_mismatches.
            if hasattr(ocr_res, 'awb_by_page') and ocr_res.awb_by_page:  # type: ignore[attr-defined]
                _inv_awb = re.sub(r'\s+', '', d.awb or '').strip('-')
                for _pn, _eur1_awb in ocr_res.awb_by_page.items():  # type: ignore[attr-defined]
                    _eur1_awb_norm = re.sub(r'\s+', '', _eur1_awb or '').strip('-')
                    if _eur1_awb_norm and _inv_awb and _eur1_awb_norm != _inv_awb:
                        d.awb_mismatches.append({
                            'page': _pn,
                            'eur1_awb': _eur1_awb,
                            'invoice_awb': d.awb,
                        })
        except Exception:
            pass

    # === Backward-compatible DIAN population from OCR (Колумбия) ===
    # Номер DIAN есть только на скане сертификата происхождения; связка со
    # счётом — позиционная: за сканом сертификата следует его проформа
    # (страница с «PROFORMA IFC-…» в текстовом слое). Связка по NIT не
    # годится: у двух сертификатов рейса бывает один экспортёр (727MHF).
    if dian_res is not None:
        # Страницы проформ из текстового слоя (если пусто — DIAN-OCR не нужен)
        proforma_pages: list[tuple[int, str]] = []
        for pi, ptxt in enumerate(page_texts, 1):
            pm = re.search(r'PROFORMA\s+([A-Z]{2,4}-?\d+)', ptxt, re.IGNORECASE)
            if pm:
                proforma_pages.append((pi, pm.group(1).upper()))
        if proforma_pages:
            try:
                for nr in dian_res.numbers:
                    if nr not in d.dian_numbers:
                        d.dian_numbers.append(nr)
                dian_pairs = sorted(zip(dian_res.page_numbers, dian_res.numbers))
                for p_page, proforma in proforma_pages:
                    # последний DIAN-скан ПЕРЕД страницей проформы
                    dian = None
                    for d_page, d_nr in dian_pairs:
                        if d_page < p_page:
                            dian = d_nr
                        else:
                            break
                    if dian is None and dian_pairs:
                        dian = dian_pairs[0][1]
                    if not dian:
                        continue
                    d.eur1_by_master.setdefault(proforma, f'DIAN-{dian}')
                    if proforma not in d.master_invoices:
                        d.master_invoices.append(proforma)
                    if not d.cert_number:
                        d.cert_number = f'DIAN-{dian}'
            except Exception:
                pass

    # === Build awb_groups hierarchical structure ===
    # For Kenya: rebuild the flat items list as one aggregate per certificate.
    _kenya_flat = d.country == 'Kenya'
    if _kenya_flat:
        d.items.clear()
        d.certificates.clear()

    if page_metas:
        awb_groups_map = _group_pages_by_awb(page_metas)
        for awb, pages in sorted(
            awb_groups_map.items(),
            key=lambda kv: kv[1][0].idx if kv[1] else 0,
        ):
            if awb == 'unknown':
                continue
            blocks = _detect_cert_blocks(pages)
            blocks = _attach_invoices(blocks, pages)
            blocks = _merge_empty_eur1_with_next(blocks)
            blocks = _apply_eur1_by_master(blocks, d.eur1_by_master)
            blocks = _remove_empty_ocr_only_blocks(blocks)
            blocks = _redistribute_orphan_pages(blocks, pages)

            # Exporter heuristic: most common exporter name on cert pages
            exporter = ''
            for block in blocks:
                for p in block.pages:
                    if 'EXPORTS LTD' in p.text.upper() or 'FLOWERS LIMITED' in p.text.upper():
                        m = re.search(
                            r'([A-Z][A-Z\s]+(?:EXPORTS?|FLOWERS?)\s+(?:LTD|LIMITED))',
                            p.text, re.IGNORECASE,
                        )
                        if m:
                            exporter = m.group(1).strip()
                            break
                if exporter:
                    break

            certs = []
            for idx, block in enumerate(blocks, 1):
                items = _items_from_block(block)
                # Get invoice_nr from invoice_pages first, then fall back to cert pages
                invoice_nr = ''
                if block.invoice_pages and block.invoice_pages[0].invoice_numbers:
                    invoice_nr = block.invoice_pages[0].invoice_numbers[0]
                else:
                    for p in block.pages:
                        if p.invoice_numbers:
                            invoice_nr = p.invoice_numbers[0]
                            break
                total_boxes = sum(it.boxes for it in items)
                total_stems = sum(it.stems for it in items)
                flower = 'ROSA'

                # Kenya ULTRA FLO Form A: boxes/stems from cert face page, invoice boxes as fallback.
                if block.cert_type == 'FORM_A':
                    form_a_boxes, form_a_stems, form_a_flower = _form_a_cert_aggregate(block)
                    if form_a_stems:
                        total_stems = form_a_stems
                    if form_a_flower:
                        flower = form_a_flower
                    if form_a_boxes:
                        total_boxes = form_a_boxes
                    else:
                        form_a_invoice_boxes = _form_a_invoice_boxes(block, path)
                        if form_a_invoice_boxes:
                            total_boxes = form_a_invoice_boxes

                # ETHAM EUR.1 face page states the aggregate boxes/stems; use it
                # to correct OCR misreads in the per-farm table.
                elif block.cert_type == 'EUR1' and items:
                    cert_text = '\n'.join(p.text for p in block.pages)
                    agg = _KE_ETHAM_AGG.search(cert_text)
                    if agg:
                        try:
                            agg_boxes = float(agg.group(1))
                            agg_stems = int(agg.group(2))
                            if total_stems == 0 or abs(agg_stems - total_stems) / max(agg_stems, 1) < 0.15:
                                total_boxes = agg_boxes
                                total_stems = agg_stems
                        except (ValueError, TypeError):
                            pass
                    if items:
                        flower = items[0].flower

                elif items:
                    flower = items[0].flower

                if _kenya_flat:
                    # One aggregate item per certificate (no per-farm/per-grade rows).
                    aggregate_item = {
                        'flower': flower,
                        'farm': '',
                        'marking': '',
                        'invoice_nr': invoice_nr,
                        'cert_number': block.cert_number,
                        'master_invoice': invoice_nr,
                        'boxes': total_boxes,
                        'stems': total_stems,
                        'weight_kg': 0.0,
                    }
                    cert_items = [aggregate_item]
                else:
                    cert_items = [{
                        'flower': it.flower,
                        'farm': it.farm,
                        'marking': it.marking,
                        'invoice_nr': it.invoice_nr,
                        'cert_number': it.cert_number,
                        'master_invoice': it.master_invoice,
                        'boxes': it.boxes,
                        'stems': it.stems,
                        'weight_kg': it.weight_kg,
                    } for it in items]

                certs.append({
                    'cert_index': idx,
                    'cert_type': block.cert_type,
                    'cert_number': block.cert_number,
                    'invoice_nr': invoice_nr,
                    'master_invoices': [invoice_nr] if invoice_nr else [],
                    'page_indices': [p.idx for p in block.pages + block.invoice_pages],
                    'items': cert_items,
                    'total_boxes': total_boxes,
                    'total_stems': total_stems,
                })

                if _kenya_flat:
                    d.certificates.append({
                        'country': d.country,
                        'cert_type': block.cert_type,
                        'cert_number': block.cert_number,
                        'invoice_nr': invoice_nr,
                        'flower': flower,
                        'boxes': total_boxes,
                        'stems': total_stems,
                        'awb': awb,
                        'exporter': exporter,
                    })
                    d.items.append(CertItem(
                        flower=flower,
                        farm='',
                        boxes=total_boxes,
                        stems=int(total_stems),
                        cert_number=block.cert_number,
                        invoice_nr=invoice_nr,
                        master_invoice=invoice_nr,
                    ))

            d.awb_groups.append({
                'awb': awb,
                'exporter': exporter,
                'awb_mismatches': [],
                'certs': certs,
            })

        # Distribute awb_mismatches by page number into each group
        for group in d.awb_groups:
            group_page_indices = {
                idx for cert in group['certs']
                for idx in cert.get('page_indices', [])
            }
            group['awb_mismatches'] = [
                mm for mm in d.awb_mismatches
                if mm.get('page') in group_page_indices
            ]

    # Kenya: totals come from the per-certificate aggregates (flat list).
    if d.country == 'Kenya' and d.items:
        d.total_boxes = sum(it.boxes for it in d.items)
        d.total_stems = sum(it.stems for it in d.items)

    return d


# === Утилиты аудита ===

def summarize(d: CertificateData) -> str:
    lines = [
        f'Файл:              {d.source}',
        f'Страна:            {d.country}',
        f'Страниц:           {d.raw_pages}',
        f'Cert #:            {d.cert_number}',
        f'EUR1-номера:       {", ".join(d.eur1_numbers) if d.eur1_numbers else "-"}',
        f'DIAN-номера:       {", ".join(d.dian_numbers) if d.dian_numbers else "-"}',
        f'Master invoices:   {", ".join(d.master_invoices) if d.master_invoices else "-"}',
        f'AWB:               {d.awb or "-"}',
        f'Date:              {d.date or "-"}',
        f'Exporter:          {d.exporter or "-"}',
        f'Consignee:         {d.consignee or "-"}',
        f'Позиций:           {len(d.items)}',
        f'Всего боксов:      {d.total_boxes}',
        f'Всего стеблей:     {d.total_stems}',
    ]
    return '\n'.join(lines)


# ============================================================
# Разбивка PDF на блоки (один блок = один сертификат)
# ============================================================

# Заголовки, которые сигнализируют о начале нового сертификата на странице
_CERT_HEADER_PATS = [
    re.compile(r'\bMOVEMENT\s+CERTIFICATE\b', re.IGNORECASE),
    re.compile(r'\bCERTIFICATE\s+OF\s+ORIGIN\b', re.IGNORECASE),
    re.compile(r'\bPHYTOSANITARY\s+CERTIFICATE\b', re.IGNORECASE),
    re.compile(r'\bEUR\.?\s*1\b', re.IGNORECASE),
    # DIAN-сертификат Колумбии
    re.compile(r'\bCERTIFICADO\s+DE\s+ORIGEN\b', re.IGNORECASE),
    # Кения / COMESA
    re.compile(r'\bCOMESA\b', re.IGNORECASE),
    # Кенийский фито-сертификат (Milele): страница начинается с голого номера бланка (5-7 цифр),
    # за ним следует название кенийского экспортёра
    # Добавляем паттерн для страниц, где первая строка — только цифры + следующая FLOWERS LIMITED
    re.compile(r'^\d{5,7}\n[A-Z].+?FLOWERS\s+(?:LIMITED|LTD)', re.IGNORECASE | re.MULTILINE),
]


def split_cert_blocks(pdf_path: str) -> list[str]:
    """
    Разбить PDF сертификата на текстовые блоки, по одному на каждый
    физический сертификат внутри файла.

    Алгоритм (вариант A+B+C):
    A. Читаем текст страниц через pdfplumber.
    B. Если заголовок (_CERT_HEADER_PATS) нашлись — делим по ним.
    C. Фоллбэк для полностью сканированных PDF (без текста):
       если OCR доступен — делим по страницам, где OCR нашёл EUR1/DIAN.
       если OCR недоступен — возвращаем один блок.

    Возвращает: list[str] — список текстовых блоков.
    Каждый блок содержит конкатенированный текст страниц одного сертификата.
    При ошибке возвращает [""]
    """
    if pdfplumber is None:
        return [""]

    try:
        page_texts: list[str] = []
        with pdfplumber.open(pdf_path) as pdf:
            for page in pdf.pages:
                page_texts.append(page.extract_text() or "")
    except Exception:
        return [""]

    if not page_texts:
        return [""]

    all_text = "\n".join(page_texts)

    # --- Вариант B: делим по текстовым заголовкам ---
    header_pages: list[int] = []
    for i, txt in enumerate(page_texts):
        for pat in _CERT_HEADER_PATS:
            if pat.search(txt):
                header_pages.append(i)
                break

    if header_pages:
        blocks: list[str] = []
        current_pages: list[str] = []
        for i, txt in enumerate(page_texts):
            if i in header_pages:
                if current_pages:
                    blocks.append("\n".join(current_pages))
                current_pages = [txt]
            else:
                current_pages.append(txt)
        if current_pages:
            blocks.append("\n".join(current_pages))
        if blocks:
            return blocks

    # --- Вариант C: фоллбэк для полностью сканированных PDF ---
    # Если весь PDF без текста (скан) — пробуем OCR
    is_scanonly = all(len(t.strip()) < 80 for t in page_texts)
    skip_ocr = os.environ.get('SKIP_OCR', '').strip() in ('1', 'true', 'yes', 'on')
    if is_scanonly and not skip_ocr and ocr_eur1_numbers is not None and is_ocr_available():
        try:
            ocr_res = ocr_eur1_numbers(pdf_path)
            # page_numbers — страницы (1-based) где нашёл номер EUR1
            eur1_pages = sorted(set(ocr_res.page_numbers))
            if len(eur1_pages) > 1:
                # Каждая страница EUR1 = начало нового блока (0-indexed)
                header_set = set(p - 1 for p in eur1_pages)  # переводим в 0-based
                blocks = []
                current_pages = []
                for i, txt in enumerate(page_texts):
                    if i in header_set:
                        if current_pages:
                            blocks.append("\n".join(current_pages))
                        current_pages = [txt]
                    else:
                        current_pages.append(txt)
                if current_pages:
                    blocks.append("\n".join(current_pages))
                if len(blocks) > 1:
                    return blocks
        except Exception:
            pass

    # Фоллбэк: весь PDF как один блок
    return [all_text]


def split_cert_blocks_by_awb(pdf_path: str) -> dict[str, list[str]]:
    """Split certificate PDF by AWB, then by certificate inside each AWB.

    Returns {normalized_awb: [cert_block_text, ...]}.
    Empty/unknown AWB pages are grouped under key 'UNKNOWN'.
    """
    if pdfplumber is None:
        return {}

    ocr_by_page: dict[int, dict] = {}
    _skip_ocr = os.environ.get('SKIP_OCR', '').strip() in ('1', 'true', 'yes', 'on')

    # Fast country hint: Ecuador is handled by the per-AWB fallback below,
    # so we can skip the very heavy EUR1/DIAN multi-DPI OCR and the full-page
    # Tesseract fallback. The text layer already contains AWB and master invoice.
    _country_hint = _detect_country(os.path.basename(pdf_path))
    if not _country_hint and fitz is not None:
        try:
            with fitz.open(pdf_path) as doc:
                sample = ''
                for i, page in enumerate(doc):
                    sample += (page.get_text('text') or '')[:400]
                    if i >= 3 or len(sample) > 1500:
                        break
            _country_hint = _detect_country_from_text(sample)
        except Exception:
            pass
    if _country_hint == 'Ecuador':
        _skip_ocr = True

    if ocr_eur1_numbers is not None and is_ocr_available() and not _skip_ocr:
        try:
            ocr_res = ocr_eur1_numbers(pdf_path)
            for pn, nr in zip(ocr_res.page_numbers, ocr_res.numbers):
                ocr_by_page.setdefault(pn, {}).setdefault('eur1', []).append(nr)
            if hasattr(ocr_res, 'awb_by_page') and ocr_res.awb_by_page:
                for pn, awb in ocr_res.awb_by_page.items():
                    ocr_by_page.setdefault(pn, {})['awb'] = awb
        except Exception:
            pass
    if ocr_dian_numbers is not None and is_ocr_available() and not _skip_ocr:
        try:
            dian_res = ocr_dian_numbers(pdf_path)
            for pn, nr in zip(dian_res.page_numbers, dian_res.numbers):
                ocr_by_page.setdefault(pn, {}).setdefault('dian', []).append(nr)
        except Exception:
            pass

    # Extract text layer first; then OCR scan-only pages so that
    # _extract_page_meta can find FORM_A Reference No., PHYTO bare numbers,
    # EUR1 numbers and AWB even when pdfplumber sees nothing.
    _OCR_TEXT_THRESHOLD = 80
    page_texts: list[str] = []
    try:
        with pdfplumber.open(pdf_path) as pdf:
            for page in pdf.pages:
                page_texts.append(page.extract_text() or '')
    except Exception:
        return {}

    scan_indices = [
        i for i, txt in enumerate(page_texts)
        if len(txt.strip()) < _OCR_TEXT_THRESHOLD
    ]
    if (scan_indices and not _skip_ocr and is_ocr_available()
            and fitz is not None and Image is not None and pytesseract is not None):
        try:
            # 150 dpi is ~3x faster than 300 dpi and still enough for AWB/cert
            # boundary detection. Full-page Kenya parsing keeps 300 dpi below.
            ocr_map = _ocr_pages(pdf_path, scan_indices, dpi=150)
            for idx, ocr_text in ocr_map.items():
                if ocr_text and ocr_text.strip():
                    page_texts[idx] = (
                        page_texts[idx].strip()
                        + '\n\n[OCR]\n'
                        + ocr_text.strip()
                    ).strip()
        except Exception:
            pass

    page_metas: list[CertPageMeta] = []
    for idx, text in enumerate(page_texts):
        page_metas.append(_extract_page_meta(None, idx, ocr_by_page, text=text))

    groups = _group_pages_by_awb(page_metas)

    # Country hint for Ecuador-specific fallback (each AWB = one cert bundle)
    country = _detect_country(os.path.basename(pdf_path))
    if not country:
        sample = ' '.join(t[:500] for t in page_texts[:5])
        country = _detect_country_from_text(sample)

    result: dict[str, list[str]] = {}
    for awb, pages in groups.items():
        blocks = _detect_cert_blocks(pages)
        # Ecuador fallback: if _detect_cert_blocks could not read EUR1 numbers from
        # the scan-only face page, still treat every AWB group as one certificate.
        if country == 'Ecuador' and not blocks and pages:
            blocks = [CertBlock(cert_type='EUR1', cert_number='', pages=pages)]
        blocks = _attach_invoices(blocks, pages)
        blocks = _merge_empty_eur1_with_next(blocks)
        blocks = _remove_empty_ocr_only_blocks(blocks)
        blocks = _redistribute_orphan_pages(blocks, pages)
        if blocks:
            result[awb] = ['\n'.join(p.text for p in (b.pages + b.invoice_pages)) for b in blocks]
    return result


if __name__ == '__main__':
    import sys
    try:
        sys.stdout.reconfigure(encoding='utf-8')
    except Exception:
        pass
    trip = sys.argv[1] if len(sys.argv) > 1 else \
        r'c:\Users\0\Desktop\Задача по логистике\03.07.2026\MTL 700'
    for f in sorted(os.listdir(trip)):
        if not f.lower().endswith('.pdf'):
            continue
        if not re.search(r'sertif|certif|сертиф', f, re.IGNORECASE):
            continue
        p = os.path.join(trip, f)
        blocks = split_cert_blocks(p)
        d = parse_certificate(p)
        print('=' * 70)
        print(f'\nФайл: {f}  |  блоков: {len(blocks)}')
        print(summarize(d))
        if d.items:
            print(f'--- Первые 5 позиций ---')
            for it in d.items[:5]:
                print(f'  {it.flower:12s} | {it.marking[:16]:16s} | inv={it.invoice_nr:>6s} | '
                      f'kg={it.weight_kg:5.1f} | box={it.boxes:5.2f} | stems={it.stems:5d} | '
                      f'{it.farm[:40]}')