gui/scan_tab.py - main - Paff/declarant

gui/scan_tab.py
615 строк26 KB
1pash1985-gif
Декларант - веб-версия для Timeweb Cloud
15 июл 2026, 09:36
15 июл 2026, 09:36a6b08b8
# -*- coding: utf-8 -*-
"""Вкладка 'Загрузка' — выбор папки рейса, сканирование файлов."""
from __future__ import annotations

import os
import sys
from pathlib import Path

from PyQt6.QtCore import Qt, QThread, pyqtSignal
from PyQt6.QtWidgets import (
    QWidget, QVBoxLayout, QHBoxLayout, QPushButton, QLabel,
    QFileDialog, QTreeWidget, QTreeWidgetItem, QProgressBar, QGroupBox,
    QTextEdit, QSplitter, QMessageBox,
)


def _find_trip_root(path: str) -> str | None:
    """Walk up from `path` looking for a directory that contains typical trip files
    (europa/import template, master date-file, or a folder named 'Инвойсы').

    Returns the trip root path if found, otherwise None.
    """
    p = Path(path)
    if not p.exists():
        return None

    # Walk up at most 6 levels
    candidates: list[Path] = []
    cur = p if p.is_dir() else p.parent
    for _ in range(6):
        candidates.append(cur)
        if cur.parent == cur:
            break
        cur = cur.parent

    for c in candidates:
        try:
            names = [f.name.lower() for f in c.iterdir()]
        except OSError:
            continue
        # Trip root signature: has europa/import template OR 'Инвойсы' subfolder
        has_europa = any(
            n.endswith('.xlsm') and ('европа' in n or 'europa' in n) for n in names
        )
        has_import = any(
            n.endswith(('.xls', '.xlsx')) and ('импорт' in n or 'import' in n) for n in names
        )
        has_florunner = any('florunner' in n for n in names)
        has_invoices_folder = any(n in ('инвойсы', 'invoices', 'invoice') for n in names)
        if has_europa or has_import or has_florunner or has_invoices_folder:
            return str(c)
    return None


class ScanWorker(QThread):
    """Background thread: parse trip folder."""
    progress = pyqtSignal(str)
    log_line = pyqtSignal(str)
    finished = pyqtSignal(dict)

    def __init__(self, trip_path: str):
        super().__init__()
        self.trip_path = trip_path

    def run(self):
        import re
        sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
        from packing_parser import (
            parse_packing, normalize_marking, _folder_hints,
        )
        from corrections import load_corrections

        self.progress.emit('Загрузка corrections...')
        corr = load_corrections()

        # --- Discover packing files (mirror parse_folder logic, but per-file feedback) ---
        root_p = Path(self.trip_path)
        # Skip: autofilled outputs, florunner-spec, europa/import templates, and master files
        # (master starts with date like '03.07.2026-700.xlsx'; invoices with date inside are OK)
        skip_pat = re.compile(
            r'_autofilled|florunner|Таблица|Табл\.|^\d{2}\.\d{2}\.\d{4}',
            re.IGNORECASE,
        )
        only_meta_pat = re.compile(r'pro\s*forma|proforma', re.IGNORECASE)

        candidates: list[Path] = []
        for f in sorted(root_p.rglob('*.xls*')):
            ext = f.suffix.lower()
            if ext not in ('.xls', '.xlsx', '.xlsm'):
                continue
            if skip_pat.search(f.name):
                continue
            if only_meta_pat.search(f.name):
                continue
            candidates.append(f)

        total = len(candidates)
        self.progress.emit(f'Найдено packing-файлов: {total}')
        self.log_line.emit(f'--- Начало парсинга: {total} файлов ---')

        sheets = []
        errors = 0
        for idx, f in enumerate(candidates, 1):
            rel = str(f.relative_to(root_p))
            self.progress.emit(f'[{idx}/{total}] {f.name}')
            try:
                sheet = parse_packing(str(f))
            except Exception as e:
                self.log_line.emit(f'  [✗] {rel}: Ошибка парсинга: {e}')
                errors += 1
                continue

            # Restore country/marking/awb from folder hints
            h = _folder_hints(str(f))
            if not sheet.country and h['country']:
                sheet.country = h['country']
            if h['client']:
                cur = (sheet.marking or '').strip().upper()
                bogus = cur in ('', 'INFORMATION', 'CLIENT', 'CUSTOMER',
                                'INVOICE', 'ORDER', 'NUMBER', 'PACKING')
                if bogus:
                    sheet.marking = normalize_marking(h['client'])
            if not sheet.awb and h['awb']:
                sheet.awb = h['awb']
            for it in sheet.items:
                if not it.country and sheet.country:
                    it.country = sheet.country
                if h['client'] and (it.marking or '').strip().upper() in (
                        '', 'INFORMATION', 'CLIENT', 'CUSTOMER', 'INVOICE',
                        'ORDER', 'NUMBER', 'PACKING'):
                    it.marking = sheet.marking

            if not sheet.items:
                self.log_line.emit(f'  [∅] {rel}: пусто (нет items)')
                continue

            n_items = len(sheet.items)
            n_stems = sum(it.stems for it in sheet.items)
            mark = sheet.marking or '?'
            country = sheet.country or '?'
            self.log_line.emit(
                f'  [✓] {rel}\n'
                f'      marking={mark!r} country={country!r} items={n_items} stems={n_stems}'
            )
            sheets.append(sheet)

        self.log_line.emit(f'--- Окончано (packing): успешно {len(sheets)}, ошибок {errors} ---')

        # === Парсинг PDF-инвойсов ===
        pdf_invoices = []
        pdf_results = {}
        try:
            from pdf_parser import FolderScanner
            from flower_db import FlowerDatabase
            self.progress.emit('Сканирование PDF-инвойсов...')
            self.log_line.emit('')
            self.log_line.emit('--- PDF-инвойсы ---')
            flower_db = FlowerDatabase()
            scanner = FolderScanner(flower_db)

            # Scan Invoices subfolder(s), not the whole trip root (avoid templates)
            # FolderScanner goes 1-level deep, so we also recurse into country dirs
            invoice_dirs = []
            for name in os.listdir(self.trip_path):
                fp = os.path.join(self.trip_path, name)
                nl = name.lower()
                if os.path.isdir(fp) and nl in ('инвойсы', 'invoices', 'invoice'):
                    invoice_dirs.append(fp)
                    # Also add country subfolders (e.g. 'KJN444 Columbia', '444ecu')
                    for sub in os.listdir(fp):
                        sub_fp = os.path.join(fp, sub)
                        if os.path.isdir(sub_fp):
                            invoice_dirs.append(sub_fp)
            if not invoice_dirs:
                # Fallback: if no Invoices folder, scan root for PDFs only
                invoice_dirs = [self.trip_path]

            for inv_dir in invoice_dirs:
                results_part = scanner.scan_directory(inv_dir)
                for client, invs in results_part.items():
                    for inv in invs:
                        pdf_invoices.append(inv)
                        fname = os.path.basename(inv.file_path)
                        n_items = len(inv.items)
                        n_boxes = inv.total_boxes
                        n_stems = inv.total_stems
                        if n_items > 0:
                            self.log_line.emit(
                                f'  [✓] {fname}\n'
                                f'      клиент={client!r} позиций={n_items} '
                                f'коробок={n_boxes} стеблей={n_stems}'
                            )
                        else:
                            self.log_line.emit(f'  [∅] {fname}: не распознано')
                    if client not in pdf_results:
                        pdf_results[client] = []
                    pdf_results[client].extend(invs)

            total_pdf_items = sum(len(inv.items) for inv in pdf_invoices)
            total_pdf_boxes = sum(inv.total_boxes for inv in pdf_invoices)
            self.log_line.emit(
                f'--- PDF итого: {len(pdf_invoices)} инвойсов, '
                f'{total_pdf_items} позиций, {total_pdf_boxes} коробок ---'
            )
        except ImportError as e:
            self.log_line.emit(f'  [⚠] PDF-парсер недоступен: {e}')
        except Exception as e:
            self.log_line.emit(f'  [✗] Ошибка PDF-парсера: {e}')

        # --- Применение corrections к маркировкам ---
        self.progress.emit('Применение corrections...')
        applied = 0
        for s in sheets:
            # 1) filename → marking override
            if s.source:
                fname_mark = corr.resolve_marking_from_filename(Path(s.source).name)
                if fname_mark:
                    s.marking = fname_mark
                    applied += 1
            # 2) marking alias (rename or skip)
            resolved = corr.resolve_marking(s.marking or '')
            if resolved is None:
                s.marking = ''
                applied += 1
            elif resolved != (s.marking or ''):
                s.marking = resolved
                applied += 1
            # 3) variety → family corrections
            for it in s.items:
                fam = corr.resolve_family(it.variety)
                if fam and not it.flower:
                    it.flower = fam
        if applied:
            self.log_line.emit(f'Применено corrections: {applied}')

        # Detect europa template
        europa_tpl = None
        europa_cols = []
        for f in os.listdir(self.trip_path):
            if f.lower().endswith('.xlsm') and 'европа' in f.lower():
                europa_tpl = os.path.join(self.trip_path, f)
                break
        if europa_tpl:
            self.progress.emit('Индексация шаблона europa...')
            try:
                from openpyxl import load_workbook
                wb = load_workbook(europa_tpl, read_only=True, data_only=True)
                ws = wb['Таблица']
                for c in range(6, ws.max_column + 1):
                    v = ws.cell(row=1, column=c).value
                    if v:
                        europa_cols.append(str(v).strip())
                wb.close()
            except Exception:
                pass

        # Build scan_data
        self.progress.emit('Анализ результатов...')
        nl_sheets = [s for s in sheets if (s.country or '').lower() == 'netherlands']
        other_sheets = [s for s in sheets if (s.country or '').lower() != 'netherlands']

        # Markings info
        markings_info = []
        from europa_writer import _mark_candidates, _prefix_match, _index_cols
        col_idx = {c.upper(): i for i, c in enumerate(europa_cols)}
        col_idx_real = {c.upper(): c for c in europa_cols}

        for s in nl_sheets:
            matched_col = ''
            status = 'OK'

            # 1) Check corrections.marking_to_column first (highest priority)
            corr_col = corr.resolve_column(s.marking or '')
            if corr_col and corr_col.upper() in col_idx:
                matched_col = col_idx_real.get(corr_col.upper(), corr_col)
                status = 'LEARNED'
            else:
                # 2) Try standard match
                for cand in _mark_candidates(s.marking or ''):
                    if cand in col_idx:
                        matched_col = col_idx_real.get(cand, cand)
                        break
                if not matched_col:
                    # 3) Try prefix
                    real_col_idx = {c.upper(): i for i, c in enumerate(europa_cols)}
                    pm = _prefix_match(s.marking or '', real_col_idx)
                    if pm is not None:
                        matched_col = europa_cols[pm] if pm < len(europa_cols) else ''
                        status = 'PREFIX'
            if not matched_col:
                status = 'FAIL'

            markings_info.append({
                'file': Path(s.source).name if s.source else '',
                'marking': s.marking or '',
                'column': matched_col,
                'status': status,
                'items': len(s.items),
                'stems': sum(it.stems for it in s.items),
            })

        # Unclassified varieties
        unclassified = set()
        for s in sheets:
            for it in s.items:
                if not it.flower:
                    unclassified.add(it.variety)

        # Files found
        files_info = []
        for root, dirs, files in os.walk(self.trip_path):
            for f in files:
                fl = f.lower()
                if fl.endswith(('.xls', '.xlsx', '.xlsm', '.pdf', '.rar', '.zip')):
                    rel = os.path.relpath(os.path.join(root, f), self.trip_path)
                    ftype = 'packing'
                    if 'европа' in fl or 'europa' in fl:
                        ftype = 'template_europa'
                    elif 'импорт' in fl or 'import' in fl:
                        ftype = 'template_import'
                    elif 'florunner' in fl or 'specification' in fl:
                        ftype = 'template_florunner'
                    elif fl.endswith('.pdf'):
                        ftype = 'certificate'
                    elif fl.endswith(('.rar', '.zip')):
                        ftype = 'archive'
                    files_info.append({'path': rel, 'type': ftype})

        scan_data = {
            'trip_path': self.trip_path,
            'sheets': sheets,
            'nl_sheets': nl_sheets,
            'other_sheets': other_sheets,
            'markings_info': markings_info,
            'europa_cols': europa_cols,
            'europa_tpl': europa_tpl,
            'unclassified': sorted(unclassified)[:100],
            'files_info': files_info,
            'total_items': sum(len(s.items) for s in sheets),
            'total_stems': sum(it.stems for s in sheets for it in s.items),
            # PDF invoice results
            'pdf_invoices': pdf_invoices,
            'pdf_results': pdf_results,
            'total_pdf_items': sum(len(inv.items) for inv in pdf_invoices),
            'total_pdf_boxes': sum(inv.total_boxes for inv in pdf_invoices),
        }
        self.finished.emit(scan_data)


class ScanTab(QWidget):
    scan_finished = pyqtSignal(dict)

    def __init__(self, parent=None):
        super().__init__(parent)
        self.trip_path = ''
        self._worker: ScanWorker | None = None
        self._setup_ui()

    def _setup_ui(self):
        layout = QVBoxLayout(self)

        # --- Top: folder selection ---
        grp_folder = QGroupBox('1. Выберите папку рейса')
        h = QHBoxLayout(grp_folder)
        self.lbl_path = QLabel('Папка не выбрана')
        self.lbl_path.setStyleSheet('font-size: 13px; color: #555;')
        self.btn_browse = QPushButton('📁  Выбрать папку...')
        self.btn_browse.setFixedWidth(170)
        self.btn_browse.clicked.connect(self._browse)
        h.addWidget(self.lbl_path, 1)
        h.addWidget(self.btn_browse)
        layout.addWidget(grp_folder)

        # --- Explanation ---
        self.lbl_explain = QLabel(
            '  Программа найдёт packing-листы, шаблоны (Europa/Import/Florunner), '
            'сертификаты и архивы.\n'
            '  После сканирования — вкладка "Верификация" покажет результат для проверки.'
        )
        self.lbl_explain.setStyleSheet('color: #666; font-size: 11px; margin: 2px 4px;')
        self.lbl_explain.setWordWrap(True)
        layout.addWidget(self.lbl_explain)

        # --- Split file tree and parse log vertically ---
        splitter = QSplitter(Qt.Orientation.Vertical)

        # File tree
        grp_files = QGroupBox('2. Обнаруженные файлы в папке')
        v = QVBoxLayout(grp_files)
        self.tree = QTreeWidget()
        self.tree.setHeaderLabels(['Имя файла / папка', 'Тип', 'Размер'])
        self.tree.setColumnWidth(0, 450)
        self.tree.setColumnWidth(1, 150)
        self.tree.setColumnWidth(2, 80)
        self.tree.setAlternatingRowColors(True)
        v.addWidget(self.tree)
        self.lbl_tree_count = QLabel('')
        self.lbl_tree_count.setStyleSheet('color: #555; font-size: 11px;')
        v.addWidget(self.lbl_tree_count)
        splitter.addWidget(grp_files)

        # Parse log
        grp_log = QGroupBox('3. Журнал парсинга — пофайлово')
        vl = QVBoxLayout(grp_log)
        self.log_view = QTextEdit()
        self.log_view.setReadOnly(True)
        self.log_view.setStyleSheet(
            'QTextEdit { font-family: "Consolas", "Courier New", monospace; '
            'font-size: 11px; background-color: #1e1e1e; color: #d4d4d4; }'
        )
        self.log_view.setPlaceholderText(
            'Журнал парсинга появится здесь после нажатия '
            '«Сканировать и распознать»...'
        )
        vl.addWidget(self.log_view)
        splitter.addWidget(grp_log)

        splitter.setStretchFactor(0, 3)
        splitter.setStretchFactor(1, 2)
        layout.addWidget(splitter, 1)

        # --- Bottom: scan button + progress ---
        h_bot = QHBoxLayout()
        self.btn_scan = QPushButton('🔍  Сканировать и распознать')
        self.btn_scan.setFixedHeight(40)
        self.btn_scan.setEnabled(False)
        self.btn_scan.setToolTip('Парсинг packing-листов, определение маркировок и классификация цветов')
        self.btn_scan.clicked.connect(self._start_scan)
        self.progress = QProgressBar()
        self.progress.setRange(0, 0)  # indeterminate
        self.progress.setVisible(False)
        self.lbl_status = QLabel('')
        self.lbl_status.setStyleSheet('font-size: 12px;')
        h_bot.addWidget(self.btn_scan)
        h_bot.addWidget(self.progress, 1)
        h_bot.addWidget(self.lbl_status)
        layout.addLayout(h_bot)

    def _browse(self):
        # Determine sensible starting directory: last used > user's Desktop > home
        start_dir = self.trip_path
        if not start_dir or not os.path.isdir(start_dir):
            desktop = Path.home() / 'Desktop'
            if not desktop.is_dir():
                alt = Path.home() / 'Рабочий стол'
                desktop = alt if alt.is_dir() else Path.home()
            start_dir = str(desktop)

        dlg = QFileDialog(
            self,
            'Выберите папку рейса (можно кликнуть любой файл — возьмётся его папка)',
            start_dir,
        )
        # AnyFile mode: user can click either a folder OR a file (we take its parent)
        dlg.setFileMode(QFileDialog.FileMode.AnyFile)
        # Force Qt's own dialog — native Windows dialog can't show files in Directory mode
        dlg.setOption(QFileDialog.Option.DontUseNativeDialog, True)
        dlg.setOption(QFileDialog.Option.DontConfirmOverwrite, True)
        dlg.setNameFilter('Все файлы (*.*)')
        dlg.setViewMode(QFileDialog.ViewMode.List)
        dlg.setLabelText(QFileDialog.DialogLabel.Accept, 'Выбрать')
        dlg.setLabelText(QFileDialog.DialogLabel.FileName, 'Папка / файл:')

        # Add Desktop / home as quick-jump sidebar shortcuts
        try:
            from PyQt6.QtCore import QUrl
            urls = dlg.sidebarUrls()
            for extra in [Path.home() / 'Desktop', Path.home() / 'Рабочий стол', Path.home()]:
                if extra.is_dir():
                    u = QUrl.fromLocalFile(str(extra))
                    if u not in urls:
                        urls.append(u)
            dlg.setSidebarUrls(urls)
        except Exception:
            pass

        if dlg.exec():
            sel = dlg.selectedFiles()
            if not sel:
                return
            path = sel[0]
            # If user selected a file instead of folder, use its parent
            p = Path(path)
            if not p.exists():
                # User typed a non-existent name in the filename field — use dialog directory
                path = dlg.directory().absolutePath()
            elif not p.is_dir():
                path = str(p.parent)

            # --- Auto-detect: if user selected a subfolder inside a trip, offer to switch ---
            trip_root = _find_trip_root(path)
            if trip_root and os.path.normcase(trip_root) != os.path.normcase(path):
                reply = QMessageBox.question(
                    self,
                    'Обнаружена папка рейса',
                    (
                        f'Вы выбрали подпапку:\n   {path}\n\n'
                        f'Похоже, корневая папка рейса — это:\n   {trip_root}\n\n'
                        f'Использовать корневую папку рейса?'
                    ),
                    QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No,
                    QMessageBox.StandardButton.Yes,
                )
                if reply == QMessageBox.StandardButton.Yes:
                    path = trip_root

            self.trip_path = path
            self.lbl_path.setText(path)
            self.lbl_path.setStyleSheet('font-size: 13px; color: #000;')
            self.btn_scan.setEnabled(True)
            self._populate_tree(path)

    def _populate_tree(self, path: str):
        self.tree.clear()
        type_labels = {
            'template_europa': '📊 Шаблон Europa',
            'template_import': '📊 Шаблон Import',
            'template_florunner': '📊 Шаблон Florunner',
            'packing': '📦 Packing-list',
            'invoice_pdf': '🧾 Инвойс (PDF)',
            'certificate': '📄 Сертификат',
            'archive': '📁 Архив',
            'master': '📋 Master',
            'pdf_other': '📄 PDF документ',
        }

        total_files = 0
        counts: dict[str, int] = {}
        for root, dirs, files in os.walk(path):
            for f in sorted(files):
                fl = f.lower()
                if not fl.endswith(('.xls', '.xlsx', '.xlsm', '.pdf', '.rar', '.zip')):
                    continue
                rel = os.path.relpath(os.path.join(root, f), path)

                # Determine type
                ftype = 'packing'
                if 'европа' in fl or 'europa' in fl:
                    ftype = 'template_europa'
                elif 'импорт' in fl or 'import' in fl:
                    ftype = 'template_import'
                elif 'florunner' in fl or 'specification' in fl:
                    ftype = 'template_florunner'
                elif fl.endswith('.pdf'):
                    # Distinguish invoices from certificates by filename
                    if 'invoice' in fl or 'инвойс' in fl:
                        ftype = 'invoice_pdf'
                    elif 'sertif' in fl or 'certif' in fl or 'сертиф' in fl:
                        ftype = 'certificate'
                    else:
                        ftype = 'pdf_other'
                elif fl.endswith(('.rar', '.zip')):
                    ftype = 'archive'

                # File size
                full_path = os.path.join(root, f)
                try:
                    size_kb = os.path.getsize(full_path) / 1024
                    size_str = f'{size_kb:.0f} KB' if size_kb < 1024 else f'{size_kb/1024:.1f} MB'
                except OSError:
                    size_str = ''

                item = QTreeWidgetItem([rel, type_labels.get(ftype, ftype), size_str])
                self.tree.addTopLevelItem(item)
                total_files += 1
                counts[ftype] = counts.get(ftype, 0) + 1

        # Build summary line with counts by type
        order = [('packing', 'packing'), ('invoice_pdf', 'инвойсы'),
                 ('certificate', 'сертификаты'), ('template_europa', 'europa'),
                 ('template_import', 'import'), ('template_florunner', 'florunner'),
                 ('archive', 'архивы'), ('pdf_other', 'другие PDF')]
        parts = [f'{label}: {counts[key]}' for key, label in order if counts.get(key)]
        summary = '   |   '.join(parts) if parts else 'файлов не найдено'
        self.lbl_tree_count.setText(f'Всего файлов: {total_files}   |   {summary}')

    def _start_scan(self):
        if not self.trip_path:
            return
        self.btn_scan.setEnabled(False)
        self.progress.setVisible(True)
        self.lbl_status.setText('Сканирование...')
        # Clear previous log
        self.log_view.clear()

        self._worker = ScanWorker(self.trip_path)
        self._worker.progress.connect(lambda msg: self.lbl_status.setText(msg))
        self._worker.log_line.connect(self._append_log)
        self._worker.finished.connect(self._on_scan_done)
        self._worker.start()

    def _append_log(self, line: str):
        self.log_view.append(line)
        # Auto-scroll to bottom
        sb = self.log_view.verticalScrollBar()
        sb.setValue(sb.maximum())

    def _on_scan_done(self, scan_data: dict):
        self.progress.setVisible(False)
        self.btn_scan.setEnabled(True)
        n = scan_data.get('total_items', 0)
        stems = scan_data.get('total_stems', 0)
        n_sheets = len(scan_data.get('sheets', []))
        pdf_n = len(scan_data.get('pdf_invoices', []))
        pdf_items = scan_data.get('total_pdf_items', 0)
        pdf_boxes = scan_data.get('total_pdf_boxes', 0)

        parts = [f'{n_sheets} packing-файлов, {n} позиций, {stems} стеблей']
        if pdf_n:
            parts.append(f'{pdf_n} PDF-инвойсов, {pdf_items} поз., {pdf_boxes} кор.')
        self.lbl_status.setText(f'Готово: {" | ".join(parts)}')
        self._append_log(
            f'\n=== Итого: {" | ".join(parts)} ==='
        )
        self.scan_finished.emit(scan_data)